diff --git a/functionsystem/tests/CMakeLists.txt b/functionsystem/tests/CMakeLists.txt index 9b0e83fdef845bbb68e42554a15344905841535e..b485278438de3a1fa76e2dde097e94827d8bee87 100644 --- a/functionsystem/tests/CMakeLists.txt +++ b/functionsystem/tests/CMakeLists.txt @@ -16,7 +16,8 @@ if (BUILD_LLT) add_definitions(-DJSON_HAS_CPP_14) message(STATUS "LLT is enabled") enable_testing() - + get_property(META_STORE_CLIENT_INCLUDE_DIR GLOBAL PROPERTY "META_STORE_CLIENT_INCLUDE_DIR") + include_directories(${META_STORE_CLIENT_INCLUDE_DIR}) add_subdirectory(unit) add_subdirectory(integration) endif () diff --git a/functionsystem/tests/integration/common/resource_view/view_utils.h b/functionsystem/tests/integration/common/resource_view/view_utils.h index f63975ebb9d4ab1fedecdb17055cafdeca93233d..e1d3608ecaa488e3151fd039723429249dbedd6b 100644 --- a/functionsystem/tests/integration/common/resource_view/view_utils.h +++ b/functionsystem/tests/integration/common/resource_view/view_utils.h @@ -21,7 +21,7 @@ #include "async/uuid_generator.hpp" #include "common/resource_view/resource_tool.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" namespace functionsystem::test::view_utils { diff --git a/functionsystem/tests/integration/function_master_test.cpp b/functionsystem/tests/integration/function_master_test.cpp deleted file mode 100644 index 4f763a99d7793f79093e179aa53e7641ff1061d8..0000000000000000000000000000000000000000 --- a/functionsystem/tests/integration/function_master_test.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include - -#include "common/constants/actor_name.h" -#include "heartbeat/ping_pong_driver.h" -#include "meta_storage_accessor/meta_storage_accessor.h" -#include "common/resource_view/view_utils.h" -#include "common/scheduler_topology/sched_tree.h" -#include "stubs/etcd_service/etcd_service_driver.h" -#include "utils.h" -#include "utils/future_test_helper.h" -#include "utils/port_helper.h" - -namespace functionsystem::test { -const std::string NODE_ID = "it_function_proxy"; // NOLINT -const std::string PROCESS_IP = "127.0.0.1:5656"; // NOLINT -const std::string SYS_FUNC_CUSTOM_ARGS = ""; // NOLINT - -const std::string SYSTEM_FUNC_CONFIG_PATH = "/home/sn/function/config"; // NOLINT -const std::string SYSTEM_FUNC_CONFIG_FILE = "system-function-config.json"; // NOLINT - -const std::string DEFAULT_ELECTION_MODE = "standalone"; // NOLINE -const std::string DEFAULT_SCHEDULE_PLUGINS = - "[\"Label\", \"ResourceSelector\", \"Default\", \"Heterogeneous\"]"; // NOLINE - -class LocalSchedulerMockActor : public litebus::ActorBase { -public: - explicit LocalSchedulerMockActor(const std::string &name) : litebus::ActorBase(name) - { - globalActorAid_.SetProtocol(litebus::BUS_TCP); - globalActorAid_.SetName(LOCAL_SCHED_MGR_ACTOR_NAME); - globalActorAid_.SetUrl(PROCESS_IP); - } - - ~LocalSchedulerMockActor() override = default; - - litebus::Future RegisterToGlobal() - { - YRLOG_ERROR("register to global_scheduler"); - - messages::Register reg; - reg.set_name(NODE_ID); - reg.set_address(GetAID().UnfixUrl()); - - *reg.mutable_resource() = view_utils::Get1DResourceUnit(NODE_ID); - - Send(globalActorAid_, "Register", reg.SerializeAsString()); - - return registered_.GetFuture(); - } - - litebus::Future RegisterToDomain() - { - YRLOG_INFO("register to domain_scheduler"); - - messages::Register reg; - reg.set_name(NODE_ID); - reg.set_address(GetAID().UnfixUrl()); - - resource_view::ResourceUnit localView = view_utils::Get1DResourceUnit(NODE_ID); - resource_view::ResourceUnit agentUnit = view_utils::Get1DResourceUnit("test-agent"); - agentUnit.set_ownerid(NODE_ID); - auto fragment = localView.mutable_fragment(); - (*fragment)[agentUnit.id()] = agentUnit; - (*reg.mutable_resources())[0] = localView; - (*reg.mutable_resources())[1] = view_utils::Get1DResourceUnit(NODE_ID); - - Send(domainActorAid_, "Register", reg.SerializeAsString()); - - return Status::OK(); - } - - void Registered(const litebus::AID &from, std::string &&name, std::string &&msg) - { - messages::Registered registered; - (void)registered.ParseFromString(msg); - - if (from.Name() == globalActorAid_.Name()) { - // registered message from global scheduler - OnRegisterToGlobal(registered); - } else if (from.Name() == domainActorAid_.Name()) { - // registered message from domain scheduler - onRegisterToDomain(registered); - } else { - YRLOG_WARN("get unexpected name of: {}", from.Name()); - } - } - - void Schedule(const litebus::AID &from, std::string &&, std::string &&msg) - { - scheduleRequest_ = std::make_shared(); - scheduleRequest_->ParseFromString(msg); - YRLOG_INFO("receive a schedule request({})", scheduleRequest_->ShortDebugString()); - - messages::ScheduleResponse response; - response.set_code(StatusCode::SUCCESS); - response.set_message("succeed to init runtime"); - response.set_requestid(scheduleRequest_->requestid()); - response.set_instanceid(scheduleRequest_->instance().instanceid()); - - Send(from, "ResponseSchedule", std::move(response.SerializeAsString())); - } - -private: - void OnRegisterToGlobal(const messages::Registered ®istered) - { - if (registered.code() != int32_t(StatusCode::SUCCESS)) { - YRLOG_ERROR("failed to register to global scheduler, errCode: {}, errMsg: {}", registered.code(), - registered.message()); - } else { - auto leader = registered.topo().leader(); - YRLOG_INFO("succeed to register to global scheduler, obtain a domain scheduler(name: {}, address: {})", - leader.name(), leader.address()); - domainActorAid_.SetName(leader.name() + DOMAIN_UNDERLAYER_SCHED_MGR_ACTOR_NAME_POSTFIX); - domainActorAid_.SetUrl(leader.address()); - StartPingPong(); - - std::this_thread::sleep_for(std::chrono::seconds(10)); - RegisterToDomain(); - } - } - - void onRegisterToDomain(const messages::Registered ®istered) - { - if (registered.code() != int32_t(StatusCode::SUCCESS)) { - YRLOG_ERROR("failed to register to domain scheduler, errCode: {}, errMsg: {}", registered.code(), - registered.message()); - } else { - YRLOG_INFO("succeed to register to domain scheduler({})", domainActorAid_.Name()); - registered_.SetValue(Status::OK()); - } - } - - void StartPingPong() - { - if (pingPongDriver_ != nullptr) { - YRLOG_INFO("ping pong server has started."); - return; - } - YRLOG_INFO("start a ping pong receiving message from domain scheduler"); - pingPongDriver_ = std::make_shared( - NODE_ID, 5000, [aid(GetAID())](const litebus::AID &, HeartbeatConnection type) { - YRLOG_ERROR("timeout to connect domain scheduler."); - }); - } - -protected: - void Init() override - { - Receive("Registered", &LocalSchedulerMockActor::Registered); - Receive("Schedule", &LocalSchedulerMockActor::Schedule); - } - -public: - std::shared_ptr pingPongDriver_; - std::shared_ptr scheduleRequest_; - litebus::Promise registered_; - -private: - litebus::AID globalActorAid_, domainActorAid_; -}; - -class FunctionMasterTest : public ::testing::Test { -public: - [[maybe_unused]] void SetUp() override - { - metaStoreServerPort_ = GetPortEnv("META_STORE_SERVER_PORT", 60000); - etcdSrvDriver_ = std::make_unique(); - etcdSrvDriver_->StartServer("127.0.0.1:" + std::to_string(metaStoreServerPort_)); - - auto binPathEnv = litebus::os::GetEnv("BIN_PATH"); - ASSERT_FALSE(binPathEnv.IsNone()); - binDir_ = binPathEnv.Get(); - - if (!litebus::os::ExistPath(SYSTEM_FUNC_CONFIG_PATH)) { - litebus::os::Mkdir(SYSTEM_FUNC_CONFIG_PATH); - } - StartFunctionMaster(); - } - - [[maybe_unused]] void TearDown() override - { - YRLOG_INFO("stop function_master process"); - KillProcess(process_.Get()->GetPid(), 2); - - etcdSrvDriver_->StopServer(); - - DeleteSystemFunctionConfigFile(); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - } - - void StartFunctionMaster() - { - YRLOG_INFO("start function_master process"); - const std::string path = binDir_ + "/function_master"; - const std::vector args = { - "", - "--node_id=it", // do not modify - "--ip=" + PROCESS_IP, - "--meta_store_address=127.0.0.1:" + std::to_string(metaStoreServerPort_), - "--sys_func_retry_period=5000", - "--sys_func_custom_args=" + SYS_FUNC_CUSTOM_ARGS, - R"(--log_config={"filepath": "/home/yr/log","level": "DEBUG","rolling": {"maxsize": )" - R"(100,"maxfiles": 1}, "alsologtostderr": true})", - "--election_mode=" + DEFAULT_ELECTION_MODE, - "--schedule_plugins=" + DEFAULT_SCHEDULE_PLUGINS - }; - - process_ = CreateProcess(path, args); - ASSERT_TRUE(process_.IsOK()); - } - - void WriteSystemFunctionConfigFile(const std::string &content) - { - if (!litebus::os::ExistPath(SYSTEM_FUNC_CONFIG_PATH)) { - litebus::os::Mkdir(SYSTEM_FUNC_CONFIG_PATH); - } - auto filePath = SYSTEM_FUNC_CONFIG_PATH + "/" + SYSTEM_FUNC_CONFIG_FILE; - - std::ofstream outfile; - outfile.open(filePath.c_str()); - outfile << content << std::endl; - outfile.close(); - } - - void DeleteSystemFunctionConfigFile() - { - auto file = litebus::os::Join(SYSTEM_FUNC_CONFIG_PATH, SYSTEM_FUNC_CONFIG_FILE); - if (litebus::os::ExistPath(file)) { - litebus::os::Rm(file); - } - } - -protected: - std::string binDir_; - uint16_t metaStoreServerPort_; - litebus::Try> process_; // NOLINT - std::unique_ptr etcdSrvDriver_; -}; - -TEST_F(FunctionMasterTest, StartTest) // NOLINT -{ - auto actor = std::make_shared("LocalScheduler"); - litebus::AID aid = litebus::Spawn(actor); - - auto status = litebus::Async(aid, &LocalSchedulerMockActor::RegisterToGlobal).Get(); - EXPECT_EQ(status.IsOk(), true); - litebus::Terminate(aid); - litebus::Await(aid); -} -} // namespace functionsystem::test diff --git a/functionsystem/tests/integration/function_proxy_test.cpp b/functionsystem/tests/integration/function_proxy_test.cpp deleted file mode 100644 index 75db291ba4932d1031f527191d7ef513838e4088..0000000000000000000000000000000000000000 --- a/functionsystem/tests/integration/function_proxy_test.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include - -#include "logs/logging.h" -#include "stubs/etcd_service/etcd_service_driver.h" -#include "utils.h" // for CreateProcess -#include "utils/port_helper.h" - -namespace functionsystem::test::function_proxy { -const std::string HOST_IP = "127.0.0.1"; // NOLINT -const std::string DATA_SYSTEM_PORT = "5400"; // NOLINT -const std::string LOG_CONFIG = // NOLINT - R"(--log_config={"filepath": "/home/yr/log","level": "DEBUG","rolling": {"maxsize": 100,"maxfiles": 1}})"; - -const std::string MASTER_NODE_ID = "it_function_master_long_123455656_lsrjt-34211"; // NOLINT -const std::string MASTER_ADDRESS = "127.0.0.1:5500"; // NOLINT - -const std::string PROXY_NODE_ID = "it_function_proxy"; // NOLINT -const std::string PROXY_ADDRESS = "127.0.0.1:5600"; // NOLINT -const std::string PROXY_GRPC_PORT = "5601"; // NOLINT - -const std::string AGENT_NODE_ID = "it_function_agent"; // NOLINT -const std::string AGENT_PORT = "5700"; // NOLINT -const std::string AGENT_ADDRESS = "127.0.0.1:" + AGENT_PORT; // NOLINT - -const std::string MANAGER_NODE_ID = "it_runtime_manager"; // NOLINT -const std::string MANAGER_PORT = "5800"; // NOLINT - -const std::string RUNTIME_INITIAL_PORT = "500"; // NOLINT -const std::string RUNTIME_PORT_COUNT = "2000"; // NOLINT - -const std::string ACCESSOR_NODE_ID = "it_function_accessor"; // NOLINT -const std::string ACCESSOR_PORT = "5900"; // NOLINT -const std::string ACCESSOR_GRPC_PORT = "5901"; // NOLINT - -class FunctionProxyTest : public ::testing::Test { -public: - [[maybe_unused]] void SetUp() override - { - metaStoreServerPort_ = GetPortEnv("META_STORE_SERVER_PORT", 60000); - } - - [[maybe_unused]] void TearDown() override - { - } - - void StartFunctionMaster() - { - YRLOG_INFO("start function_master process"); - const std::string path = binDir_ + "/function_master"; - const std::vector args = { "", - "--node_id=" + MASTER_NODE_ID, // do not modify - "--ip=" + MASTER_ADDRESS, - "--meta_store_address=127.0.0.1:" - + std::to_string(metaStoreServerPort_), - "--sys_func_retry_period=5000", - "--sys_func_custom_args=", - LOG_CONFIG }; - - masterProcess_ = CreateProcess(path, args); - ASSERT_TRUE(masterProcess_.IsOK()); - } - - void StartFunctionProxy() - { - YRLOG_INFO("start function_proxy process"); - const std::string path = binDir_ + "/function_proxy"; - const std::vector args = { "", - "--address=" + PROXY_ADDRESS, - "--meta_store_address=127.0.0.1:" - + std::to_string(metaStoreServerPort_), - "--services_path=", - "--lib_path=", - "--node_id=" + PROXY_NODE_ID, - "--ip=" + HOST_IP, - "--grpc_listen_port=" + PROXY_GRPC_PORT, - "--runtime_heartbeat_enable=false", - "--runtime_max_heartbeat_timeout_times=5", - "--runtime_heartbeat_timeout_ms=5000", - "--global_scheduler_address=" + MASTER_ADDRESS, - "--cache_storage_host=" + HOST_IP, - "--cache_storage_port=" + DATA_SYSTEM_PORT, - "--enable_trace=false", - LOG_CONFIG }; - - proxyProcess_ = CreateProcess(path, args); - ASSERT_TRUE(proxyProcess_.IsOK()); - } - - void StartFunctionAgent() - { - YRLOG_INFO("start function_agent process"); - const std::string path = binDir_ + "/function_agent"; - const std::vector args = { "", - "--node_id=" + AGENT_NODE_ID, - "--ip=" + HOST_IP, - "--agent_listen_port=" + AGENT_PORT, - "--local_scheduler_address=" + PROXY_ADDRESS, - "--access_key=", - "--secret_key=", - "--s3_endpoint=", - LOG_CONFIG }; - - agentProcess_ = CreateProcess(path, args); - ASSERT_TRUE(agentProcess_.IsOK()); - } - - void StartFunctionAccessor() - { - YRLOG_INFO("start function_accessor process"); - const std::string path = binDir_ + "/function_accessor"; - const std::vector args = { "", - "--node_id=" + ACCESSOR_NODE_ID, - "--ip=" + HOST_IP, - "--http_listen_port=" + ACCESSOR_PORT, - "--grpc_listen_port=" + ACCESSOR_GRPC_PORT, - "--select_scheduler_policy=TopKRandom", - "--min_instance_memory_size=128", - "--min_instance_cpu_size=300", - "--meta_store_address=127.0.0.1:" - + std::to_string(metaStoreServerPort_), - "--enable_trace=false", - LOG_CONFIG }; - - accessorProcess_ = CreateProcess(path, args); - ASSERT_TRUE(accessorProcess_.IsOK()); - } - -protected: - std::string binDir_; - uint16_t metaStoreServerPort_; - - litebus::Try> masterProcess_; // NOLINT - litebus::Try> proxyProcess_; // NOLINT - litebus::Try> agentProcess_; // NOLINT - litebus::Try> managerProcess_; // NOLINT - litebus::Try> accessorProcess_; // NOLINT - - std::unique_ptr etcdSrvDriver_; -}; - -TEST_F(FunctionProxyTest, StartTest) // NOLINT -{ - ASSERT_TRUE(true); -} -} // namespace functionsystem::test::function_proxy diff --git a/functionsystem/tests/integration/main.cpp b/functionsystem/tests/integration/main.cpp index 65b29246b165b91c05330ce79b9a8476c37814f4..37c8114266b47e6e4f1bd9beeb0b0f4ed73f2541 100644 --- a/functionsystem/tests/integration/main.cpp +++ b/functionsystem/tests/integration/main.cpp @@ -19,8 +19,8 @@ #include #include -#include "logs/logging.h" -#include "status/status.h" +#include "common/logs/logging.h" +#include "common/status/status.h" #include "logs/sdk/log_param_parser.h" #include "utils/port_helper.h" @@ -60,10 +60,7 @@ int main(int argc, char **argv) int port = functionsystem::test::FindAvailablePort(); litebus::os::SetEnv("LITEBUS_PORT", std::to_string(port)); - std::cout << "port: " << port << std::endl; - int metaStoreServerPort = functionsystem::test::FindAvailablePort(); - litebus::os::SetEnv("META_STORE_SERVER_PORT", std::to_string(metaStoreServerPort)); - std::cout << "metaStoreServerPort: " << metaStoreServerPort << std::endl; + std::cout << "Start LITEBUS on net port: " << port << std::endl; auto res = litebus::Initialize("tcp://127.0.0.1:" + std::to_string(port), "", "udp://127.0.0.1:" + std::to_string(port), ""); if (res != BUS_OK) { diff --git a/functionsystem/tests/integration/mocks/mock_function_agent_service_actor.h b/functionsystem/tests/integration/mocks/mock_function_agent_service_actor.h index 1fd35f2fd1f23965f7e39723fb6d5411a380a072..063a00d1d2571fde38fb552e9ec9fd7a2714823a 100644 --- a/functionsystem/tests/integration/mocks/mock_function_agent_service_actor.h +++ b/functionsystem/tests/integration/mocks/mock_function_agent_service_actor.h @@ -22,9 +22,9 @@ #include #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" #include "common/register/register_helper.h" -#include "status/status.h" +#include "common/status/status.h" namespace functionsystem::test { @@ -57,8 +57,10 @@ public: return; } runtimeManagerAID_ = litebus::AID(req.name(), req.address()); - registerHelper_->SetHeartbeatObserveDriver(req.name(), req.address(), 12000, - [](const litebus::AID &) { YRLOG_WARN("heartbeat timeouts"); }); + registerHelper_->SetHeartbeatObserveDriver( + req.name(), req.address(), 12000, [](const litebus::AID &, HeartbeatConnection) { + YRLOG_WARN("heartbeat timeouts"); + }, MOCK_AGENT_SERVICE_NAME); // send Registered message back to runtime_manager rsp.set_code((int32_t)StatusCode::SUCCESS); registerHelper_->SendRegistered(req.name(), req.address(), rsp.SerializeAsString()); @@ -96,23 +98,27 @@ public: Send(from, "UpdateRuntimeStatusResponse", rsp.SerializeAsString()); } - bool StartInstance(const messages::StartInstanceRequest &request) + int StartInstance(const messages::StartInstanceRequest &request) { - YRLOG_INFO("send StartInstance request to {}", std::string(runtimeManagerAID_)); - Send(runtimeManagerAID_, "StartInstance", request.SerializeAsString()); - return true; + YRLOG_INFO("Send StartInstance request to {}", std::string(runtimeManagerAID_)); + return Send(runtimeManagerAID_, "StartInstance", request.SerializeAsString()); } + void StartInstanceResponse(const litebus::AID &from, std::string && /* name */, std::string &&msg) { + YRLOG_INFO("Receive StartInstance response from {}", std::string(from)); startInstanceResponseMsg_.SetValue(msg); } - void StopInstance(const messages::StopInstanceRequest &request) + int StopInstance(const messages::StopInstanceRequest &request) { - Send(runtimeManagerAID_, "StopInstance", request.SerializeAsString()); + YRLOG_INFO("Send StopInstance request to {}", std::string(runtimeManagerAID_)); + return Send(runtimeManagerAID_, "StopInstance", request.SerializeAsString()); } + void StopInstanceResponse(const litebus::AID &from, std::string && /* name */, std::string &&msg) { + YRLOG_INFO("Receive StopInstance response from {}", std::string(from)); stopInstanceResponseMsg_.SetValue(msg); } diff --git a/functionsystem/tests/integration/mocks/mock_posix_stream_handler.h b/functionsystem/tests/integration/mocks/mock_posix_stream_handler.h index 6a2de9fe25c6780aa5b068ebaa30c99cc3d3cc77..6da226e45062fd89c83a07b57e94451d2c303883 100644 --- a/functionsystem/tests/integration/mocks/mock_posix_stream_handler.h +++ b/functionsystem/tests/integration/mocks/mock_posix_stream_handler.h @@ -21,7 +21,7 @@ #include -#include "proto/pb/posix_pb.h" +#include "common/proto/pb/posix_pb.h" namespace functionsystem::test { diff --git a/functionsystem/tests/integration/runtime_manager_test.cpp b/functionsystem/tests/integration/runtime_manager_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9ab607c1445b8020292df3a736000854d3ca355a --- /dev/null +++ b/functionsystem/tests/integration/runtime_manager_test.cpp @@ -0,0 +1,395 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "common/constants/constants.h" +#include "common/logs/logging.h" +#include "common/utils/files.h" +#include "mocks/mock_function_agent_service_actor.h" +#include "utils.h" +#include "utils/port_helper.h" +#include "common/utils/exec_utils.h" +#include "utils/future_test_helper.h" + +namespace functionsystem::test { + +const std::string RUNTIME_MANAGER_NODE_ID = "dggpsmk100001-8008"; +const std::string RUNTIME_MANAGER_IP = "127.0.0.1"; +const std::string RUNTIME_MANAGER_RUNTIME_INITIAL_PORT = "500"; +const std::string RUNTIME_MANAGER_PORT_NUM = "2000"; +const std::string RUNTIME_MANAGER_IS_NEW_RUNTIME_PATH = "false"; +const std::string RUNTIME_MANAGER_PYTHON_DEPENDENCY_PATH = ""; +const std::string RUNTIME_MANAGER_RUNTIME_DIR = "/tmp"; +const std::string RUNTIME_MANAGER_RUNTIME_LOGS_DIR = ""; +const std::string RUNTIME_MANAGER_RUNTIME_LD_LIBRARY_PATH = "/tmp"; +const std::string RUNTIME_MANAGER_LOG_CONFIG = + R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})"; +const std::string RUNTIME_MANAGER_PROC_METRICS_CPU = "2000"; +const std::string RUNTIME_MANAGER_PROC_METRICS_MEMORY = "2000"; +const std::string testDeployDir = "/tmp/layer/func/bucket-test-log1/yr-test-integration-runtime-manager"; +const std::string funcObj = testDeployDir + "/" + "funcObj"; +const std::string KILL_PROCESS_TIMEOUT_SECONDS = "1"; +const std::string TEST_MONITOR_DISK_PATH = "/diskMonitorTestDir"; +const std::string TEST_TENANT_ID = "tenant001"; + +class RuntimeManagerTest : public testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + if (!litebus::os::ExistPath("/tmp/cpp/bin")) { + litebus::os::Mkdir("/tmp/cpp/bin"); + } + + auto fd = open("/tmp/cpp/bin/runtime", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + + std::ofstream outfile; + outfile.open("/tmp/cpp/bin/runtime"); + outfile << "sleep 2" << std::endl; + outfile.close(); + } + + void SetUp() override + { + auto outputPath = litebus::os::GetEnv("BIN_PATH"); + binPath_ = outputPath.Get() + "/runtime_manager"; + litebus::os::SetEnv("YR_BARE_MENTAL", "1"); + (void)litebus::os::Mkdir(testDeployDir); + (void)TouchFile(funcObj); + (void)system( + "echo \"testDeployDir in integration runtime_executor_test\"" + "> /tmp/layer/func/bucket-test-log1/yr-test-integration-runtime-executor/funcObj"); + litebus::os::Rmdir(TEST_MONITOR_DISK_PATH); + } + + void TearDown() override + { + (void)litebus::os::Rmdir(testDeployDir); + } + + std::shared_ptr StartFunctionAgent() + { + auto functionAgent = std::make_shared(); + litebus::Spawn(functionAgent); + return functionAgent; + } + + static std::shared_ptr StartRuntimeManager( + const std::string &binPath, const std::shared_ptr &functionAgent) + { + YRLOG_INFO("start runtime manager"); + const std::vector args = { " ", + "--node_id=" + RUNTIME_MANAGER_NODE_ID, + "--ip=" + RUNTIME_MANAGER_IP, + "--host_ip=" + RUNTIME_MANAGER_IP, + "--port=" + std::to_string(FindAvailablePort()), + "--runtime_initial_port=" + RUNTIME_MANAGER_RUNTIME_INITIAL_PORT, + "--port_num=" + RUNTIME_MANAGER_PORT_NUM, + "--runtime_dir=" + RUNTIME_MANAGER_RUNTIME_DIR, + "--agent_address=" + functionAgent->GetAID().Url(), + "--runtime_ld_library_path=" + RUNTIME_MANAGER_RUNTIME_LD_LIBRARY_PATH, + "--proc_metrics_cpu=" + RUNTIME_MANAGER_PROC_METRICS_CPU, + "--proc_metrics_memory=" + RUNTIME_MANAGER_PROC_METRICS_MEMORY, + "--kill_process_timeout_seconds=" + KILL_PROCESS_TIMEOUT_SECONDS, + "--disk_usage_monitor_path=" + TEST_MONITOR_DISK_PATH, + "--disk_usage_limit=1", + "--disk_usage_monitor_duration=200", + "--disk_usage_monitor_notify_failure_enable=true", + RUNTIME_MANAGER_LOG_CONFIG }; + auto runtimeProcess = CreateProcess(binPath, args); + EXPECT_TRUE(runtimeProcess.IsOK()); + return runtimeProcess.Get(); + } + + messages::StartInstanceRequest GenStartInstanceRequest() + { + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("cpp"); + auto userEnvs = runtimeConfig->mutable_userenvs(); + userEnvs->insert({ "user_env1", "user_env1_value" }); + userEnvs->insert({ "user_env2", "user_env2_value" }); + + auto deployConfig = runtimeInfo->mutable_deploymentconfig(); + deployConfig->set_objectid("test_objectID"); + deployConfig->set_bucketid("test_bucketID"); + deployConfig->set_deploydir(testDeployDir); + deployConfig->set_storagetype("s3"); + return startRequest; + } + + void PrepareWorkingDir() + { + (void)litebus::os::Mkdir(unzipedAppWorkingDir_); + const std::string entrypointPath = unzipedAppWorkingDir_ + "script.py"; + (void)litebus::os::Rm(entrypointPath); + TouchFile(entrypointPath); + std::ofstream outfile; + outfile.open(entrypointPath); + outfile << "import sys" << std::endl; + outfile << "import os" << std::endl; + outfile << "import time" << std::endl; + outfile << R"(print("Python executable path:", sys.executable))" << std::endl; + outfile << R"(print("Python module search path (sys.path):", sys.path))" << std::endl; // print PYTHONPATH + outfile << R"(print("Environment Variables:"))" << std::endl; + outfile << R"(for key, value in os.environ.items():)" << std::endl; + outfile << R"( print(f"{key}={value}"))" << std::endl << std::endl; + outfile << R"(time.sleep(60))" << std::endl; // sleep 60s + outfile.close(); + } + + void DestroyWorkingDir() + { + (void)litebus::os::Rmdir(unzipedAppWorkingDir_); + } + + messages::StartInstanceRequest GenStartJobRequest() + { + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("posix-custom-runtime"); + auto userEnvs = runtimeConfig->mutable_userenvs(); + userEnvs->insert({ "user_env1", "user_env1_value" }); + userEnvs->insert({ "user_env2", "user_env2_value" }); + runtimeInfo->mutable_runtimeconfig()->mutable_posixenvs()->insert( + { "LD_LIBRARY_PATH", "${LD_LIBRARY_PATH}:/opt/buildtools/python3.9/lib/" }); + // both UNZIPPED_WORKING_DIR and YR_WORKING_DIR are required. + runtimeInfo->mutable_runtimeconfig()->mutable_posixenvs()->insert({"UNZIPPED_WORKING_DIR", unzipedAppWorkingDir_}); + runtimeInfo->mutable_runtimeconfig()->mutable_posixenvs()->insert({"YR_WORKING_DIR", workingDirFile_}); + runtimeInfo->mutable_runtimeconfig()->mutable_posixenvs()->insert({YR_TENANT_ID, TEST_TENANT_ID}); + runtimeInfo->mutable_runtimeconfig()->set_entryfile("python3 script.py"); // ray job entrypoint + + auto deployConfig = runtimeInfo->mutable_deploymentconfig(); + deployConfig->set_objectid("test_objectID"); + deployConfig->set_bucketid("test_bucketID"); + deployConfig->set_deploydir(testDeployDir); + deployConfig->set_storagetype("s3"); + return startRequest; + } + + messages::StopInstanceRequest GenStopInstanceRequest() + { + messages::StopInstanceRequest stopRequest; + stopRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + stopRequest.set_requestid("test_requestID"); + stopRequest.set_runtimeid("test_runtimeID"); + stopRequest.set_traceid("test_traceID"); + return stopRequest; + } + + std::string binPath_; + std::string unzipedAppWorkingDir_ = "/home/sn/function/package/xxxst/working_dir/yyy"; + std::string workingDirFile_ = "file:///home/sn/function/package/file.zip"; +}; + +TEST_F(RuntimeManagerTest, StartInstance) +{ + auto functionAgent = StartFunctionAgent(); + auto runtimeProcess = StartRuntimeManager(binPath_, functionAgent); + auto startRequest = GenStartInstanceRequest(); + + // If the resource is not null, runtime-manager initialization is complete. + EXPECT_AWAIT_TRUE([&]() -> bool { return functionAgent->resource_ != nullptr; }); + litebus::Async(functionAgent->GetAID(), &MockFunctionAgentServiceActor::StartInstance, startRequest); + + auto startInstanceResponseMsg = functionAgent->startInstanceResponseMsg_.GetFuture(); + EXPECT_AWAIT_READY(startInstanceResponseMsg); // wait response + messages::StartInstanceResponse res; + res.ParseFromString(startInstanceResponseMsg.Get()); + EXPECT_TRUE(res.code() == StatusCode::SUCCESS); + + auto updateInstanceStatusMsg = functionAgent->updateInstanceStatusMsg_.GetFuture(); + EXPECT_AWAIT_READY(updateInstanceStatusMsg); // wait response + messages::UpdateInstanceStatusRequest req; + req.ParseFromString(updateInstanceStatusMsg.Get()); + EXPECT_TRUE(req.instancestatusinfo().status() == 0); + EXPECT_TRUE(req.instancestatusinfo().instanceid() == "test_instanceID"); + + litebus::Terminate(functionAgent->GetAID()); + litebus::Await(functionAgent->GetAID()); + KillProcess(runtimeProcess->GetPid(), SIGKILL); + (void)runtimeProcess->GetStatus().Get(); +} + +TEST_F(RuntimeManagerTest, StopInstance) +{ + auto functionAgent = StartFunctionAgent(); + auto runtimeProcess = StartRuntimeManager(binPath_, functionAgent); + + litebus::AID aid = functionAgent->GetAID(); + + auto startRequest = GenStartInstanceRequest(); + EXPECT_GT(litebus::Async(aid, &MockFunctionAgentServiceActor::StartInstance, startRequest).Get(), 0); + + auto startInstanceResponseMsg = functionAgent->startInstanceResponseMsg_.GetFuture(); + messages::StartInstanceResponse startInstanceResponse; + startInstanceResponse.ParseFromString(startInstanceResponseMsg.Get()); + EXPECT_EQ(startInstanceResponse.code(), StatusCode::SUCCESS); + auto resRuntimeId = startInstanceResponse.startruntimeinstanceresponse().runtimeid(); + + auto stopRequest = GenStopInstanceRequest(); + stopRequest.set_runtimeid(resRuntimeId); + EXPECT_GT(litebus::Async(aid, &MockFunctionAgentServiceActor::StopInstance, stopRequest).Get(), 0); + + auto stopInstanceResponseMsg = functionAgent->stopInstanceResponseMsg_.GetFuture(); + messages::StopInstanceResponse stopInstanceResponse; + stopInstanceResponse.ParseFromString(stopInstanceResponseMsg.Get()); + EXPECT_EQ(stopInstanceResponse.code(), StatusCode::SUCCESS); + EXPECT_EQ(stopInstanceResponse.message(), "stop instance success"); + EXPECT_EQ(stopInstanceResponse.runtimeid(), resRuntimeId); + EXPECT_EQ(stopInstanceResponse.requestid(), "test_requestID"); + + litebus::Terminate(functionAgent->GetAID()); + litebus::Await(functionAgent->GetAID()); + KillProcess(runtimeProcess->GetPid(), SIGKILL); + (void)runtimeProcess->GetStatus().Get(); +} + +void KillChildProcesses(pid_t parentPid) +{ + namespace fs = std::filesystem; + std::vector childPids; + + for (const auto& entry : fs::directory_iterator("/proc")) { + if (!entry.is_directory()) continue; + + const std::string pidDir = entry.path().filename().string(); + if (!std::all_of(pidDir.begin(), pidDir.end(), ::isdigit)) continue; + + pid_t pid = std::stoi(pidDir); + std::string statPath = entry.path().string() + "/stat"; + + std::ifstream statFile(statPath); + if (!statFile.is_open()) continue; + + std::string statContent; + std::getline(statFile, statContent); + statFile.close(); + + // Get the ppid (parent process ID) from the stat file + size_t startPos = statContent.find(')') + 2; + size_t spacePos = statContent.find(' ', startPos); + std::istringstream iss(statContent.substr(spacePos + 1)); + int ppid; + iss >> ppid; + + if (ppid == parentPid) { + childPids.push_back(pid); + } + } + + // Kill child process + for (pid_t childPid : childPids) { + if (kill(childPid, SIGKILL) == 0) { + std::cout << "Killed child process: " << childPid << std::endl; + } else { + perror(("Failed to kill child process: " + std::to_string(childPid)).c_str()); + } + } +} + +/* +TEST_F(RuntimeManagerTest, RayJobStartAndKill) +{ + PrepareWorkingDir(); + + auto functionAgent = StartFunctionAgent(); + auto runtimeProcess = StartRuntimeManager(binPath_, functionAgent); + auto startRequest = GenStartJobRequest(); + + litebus::Async(functionAgent->GetAID(), &MockFunctionAgentServiceActor::StartInstance, startRequest); + + auto startInstanceResponseMsg = functionAgent->startInstanceResponseMsg_.GetFuture(); + messages::StartInstanceResponse res; + res.ParseFromString(startInstanceResponseMsg.Get()); + EXPECT_TRUE(res.code() == StatusCode::SUCCESS); + auto resRuntimeId = res.startruntimeinstanceresponse().runtimeid(); + + YRLOG_DEBUG("kill job"); + KillChildProcesses(runtimeProcess->GetPid()); + + YRLOG_DEBUG("receive update job status msg"); + auto updateInstanceStatusMsg = functionAgent->updateInstanceStatusMsg_.GetFuture(); + messages::UpdateInstanceStatusRequest req; + req.ParseFromString(updateInstanceStatusMsg.Get()); + EXPECT_EQ(req.instancestatusinfo().status(), 9); // SIGKILL + EXPECT_TRUE(req.instancestatusinfo().instanceid() == "test_instanceID"); + + litebus::Terminate(functionAgent->GetAID()); + litebus::Await(functionAgent->GetAID()); + KillProcess(runtimeProcess->GetPid(), SIGKILL); + runtimeProcess->GetStatus().Get(); + DestroyWorkingDir(); +} + */ + +TEST_F(RuntimeManagerTest, StartInstanceWithDiskMonitor) +{ + litebus::os::Mkdir(TEST_MONITOR_DISK_PATH); + + auto functionAgent = StartFunctionAgent(); + auto runtimeProcess = StartRuntimeManager(binPath_, functionAgent); + + // disk usage execeed limit must caused by instance + auto startRequest = GenStartInstanceRequest(); + litebus::Async(functionAgent->GetAID(), &MockFunctionAgentServiceActor::StartInstance, startRequest); + auto startInstanceResponseMsg = functionAgent->startInstanceResponseMsg_.GetFuture(); + messages::StartInstanceResponse res; + res.ParseFromString(startInstanceResponseMsg.Get()); + EXPECT_TRUE(res.code() == StatusCode::SUCCESS); + + // write into file + ExecuteCommand("dd if=/dev/zero of=" + TEST_MONITOR_DISK_PATH + "/test.txt bs=2M count=1"); + auto updateInstanceStatusMsg = functionAgent->updateInstanceStatusMsg_.GetFuture(); + messages::UpdateInstanceStatusRequest req; + req.ParseFromString(updateInstanceStatusMsg.Get()); + EXPECT_AWAIT_TRUE([&]() -> bool { + return req.instancestatusinfo().status() == static_cast(INSTANCE_DISK_USAGE_EXCEED_LIMIT); + }); + + litebus::Terminate(functionAgent->GetAID()); + litebus::Await(functionAgent->GetAID()); + KillProcess(runtimeProcess->GetPid(), SIGKILL); + litebus::os::Mkdir(TEST_MONITOR_DISK_PATH); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/integration/stubs/etcd_service/etcd_service_driver.cpp b/functionsystem/tests/integration/stubs/etcd_service/etcd_service_driver.cpp index d16230226186ea24e1e395b824edbd0c3b2bfc4e..4bcd7b98376d21d488e1e2f01bc4536e1cdffef7 100644 --- a/functionsystem/tests/integration/stubs/etcd_service/etcd_service_driver.cpp +++ b/functionsystem/tests/integration/stubs/etcd_service/etcd_service_driver.cpp @@ -19,7 +19,7 @@ #include #include "async/async.hpp" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "grpcpp/server_builder.h" #include "stubs/etcd_service/etcd_kv_service.h" #include "stubs/etcd_service/etcd_lease_service.h" @@ -69,7 +69,7 @@ void EtcdServiceDriver::StartServer(const std::string &address) void EtcdServiceDriver::StopServer() { if (server_ != nullptr) { - server_->Shutdown(); + server_->Shutdown(std::chrono::system_clock::now()); server_ = nullptr; } diff --git a/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_service.cpp b/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_service.cpp index 3b0ea67c0307e23dc173ac38c6e0c2bc47322d6c..85779d8321471e0d445c0285bab48b7ac0220fc8 100644 --- a/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_service.cpp +++ b/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_service.cpp @@ -17,7 +17,7 @@ #include "etcd_watch_service.h" #include "async/async.hpp" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "etcd_watch_srv_actor.h" namespace functionsystem::meta_store::test { diff --git a/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_srv_actor.cpp b/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_srv_actor.cpp index d8ccff34a0c5aeb8dfa8eaa82b277ba1b37fc2cd..47114f8dce7f79fccf5d29c84c5aed580d7f01f3 100644 --- a/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_srv_actor.cpp +++ b/functionsystem/tests/integration/stubs/etcd_service/etcd_watch_srv_actor.cpp @@ -16,7 +16,7 @@ #include "etcd_watch_srv_actor.h" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "meta_store_common.h" namespace functionsystem::meta_store::test { diff --git a/functionsystem/tests/integration/stubs/local_sched_srv_stub/local_sched_srv_stub.h b/functionsystem/tests/integration/stubs/local_sched_srv_stub/local_sched_srv_stub.h index 210ce2a35f1ce6c7fc4c395a40f29d73e9d67416..3b966cfa3e2bb1189dc06e023667e566f3a05738 100644 --- a/functionsystem/tests/integration/stubs/local_sched_srv_stub/local_sched_srv_stub.h +++ b/functionsystem/tests/integration/stubs/local_sched_srv_stub/local_sched_srv_stub.h @@ -17,8 +17,8 @@ #ifndef TEST_INTEGRATION_STUBS_LOCAL_SCHEDULER_SERVICE_STUB_H #define TEST_INTEGRATION_STUBS_LOCAL_SCHEDULER_SERVICE_STUB_H -#include "proto/pb/posix_pb.h" -#include "rpc/stream/posix/control_client.h" +#include "common/proto/pb/posix_pb.h" +#include "common/rpc/stream/posix/control_client.h" namespace functionsystem::test { diff --git a/functionsystem/tests/integration/utils.h b/functionsystem/tests/integration/utils.h index f43f11547943425e6c538fc9f74d92971d6a48aa..1a1997ec49f29dd6836744f2404b5dbb9cf7295d 100644 --- a/functionsystem/tests/integration/utils.h +++ b/functionsystem/tests/integration/utils.h @@ -25,8 +25,12 @@ inline litebus::Try> CreateProcess(const std::str const std::vector &args) { litebus::Try> process = litebus::Exec::CreateExec( - path, args, litebus::None(), litebus::ExecIO::CreateFDIO(STDIN_FILENO), - litebus::ExecIO::CreateFDIO(STDOUT_FILENO), litebus::ExecIO::CreateFDIO(STDERR_FILENO)); + path, args, + litebus::None(), + litebus::ExecIO::CreateFDIO(STDIN_FILENO), + litebus::ExecIO::CreateFDIO(STDOUT_FILENO), + litebus::ExecIO::CreateFDIO(STDERR_FILENO) + ); // Need to check process is alive. sleep(1); @@ -38,7 +42,6 @@ inline void KillProcess(pid_t pid, int sig) ::kill(pid, sig); // Need to check process has been killed. } - -} // namespace functionsystem::test +} // namespace functionsystem::test #endif // TEST_INTEGRATION_UTILS_H diff --git a/functionsystem/tests/integration/utils/port_helper.h b/functionsystem/tests/integration/utils/port_helper.h index 96bc75a62c9a83d05d5528aaec2e31278719d725..077edbc37924c198ea69914912bc56f531b8ad8b 100644 --- a/functionsystem/tests/integration/utils/port_helper.h +++ b/functionsystem/tests/integration/utils/port_helper.h @@ -57,7 +57,7 @@ inline int FindAvailablePort() { std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution<> dist(1024, 65535); + std::uniform_int_distribution<> dist(10240, 65535); int sock; struct sockaddr_in addr; diff --git a/functionsystem/tests/unit/CMakeLists.txt b/functionsystem/tests/unit/CMakeLists.txt index 23ac9409522df23b0370b4b7db8a669d0ab071eb..0f90836c183c22d3f341e4d4d9fab093d1345e86 100644 --- a/functionsystem/tests/unit/CMakeLists.txt +++ b/functionsystem/tests/unit/CMakeLists.txt @@ -35,8 +35,12 @@ get_property(SRC_LOCAL_SCHEDULER_INCLUDE_DIR GLOBAL PROPERTY "SRC_LOCAL_SCHEDULE get_property(SRC_FUNCTION_PROXY_INCLUDE_DIR GLOBAL PROPERTY "SRC_FUNCTION_PROXY_INCLUDE_DIR") get_property(INSTANCE_MANAGER_DIR GLOBAL PROPERTY "INSTANCE_MANAGER_DIR") # instance_manager src get_property(INSTANCE_MANAGER_LIB GLOBAL PROPERTY "INSTANCE_MANAGER_LIB") # instance_manager lib +get_property(SYSTEM_FUNCTION_LOADER_DIR GLOBAL PROPERTY "SYSTEM_FUNCTION_LOADER_DIR") # system_function_loader src +get_property(SYSTEM_FUNCTION_LOADER_LIB GLOBAL PROPERTY "SYSTEM_FUNCTION_LOADER_LIB") # system_function_loader lib get_property(MASTER_COMMON_DIR GLOBAL PROPERTY "MASTER_COMMON_DIR") get_property(MASTER_COMMON_LIB GLOBAL PROPERTY "MASTER_COMMON_LIB") +get_property(SCALER_DIR GLOBAL PROPERTY "SCALER_DIR") # scaler src +get_property(SCALER_LIB GLOBAL PROPERTY "SCALER_LIB") # scaler lib get_property(GLOBAL_SCHEDULER_DIR GLOBAL PROPERTY "GLOBAL_SCHEDULER_DIR") get_property(GLOBAL_SCHEDULER_LIB GLOBAL PROPERTY "GLOBAL_SCHEDULER_LIB") get_property(RESOURCE_GROUP_MANAGER_DIR GLOBAL PROPERTY "RESOURCE_GROUP_MANAGER_DIR") @@ -59,8 +63,10 @@ target_include_directories(${UNIT_TEST_MODULE} PRIVATE ${SRC_FUNCTION_PROXY_INCLUDE_DIR} ${MASTER_COMMON_DIR} # function master common src ${INSTANCE_MANAGER_DIR} # instance_manager src + ${SYSTEM_FUNCTION_LOADER_DIR} # system_function_loader src ${RESOURCE_GROUP_MANAGER_DIR} ${FUNCTION_AGENT_DIR} # function_agent src + ${SCALER_DIR} # scaler src ${GLOBAL_SCHEDULER_DIR} ${SRC_RUNTIME_MANAGER_INCLUDE_DIR} ${SRC_DOMAIN_SCHEDULER_INCLUDE_DIR} diff --git a/functionsystem/tests/unit/common/CMakeLists.txt b/functionsystem/tests/unit/common/CMakeLists.txt index d709b70b239d7570a430ae51e6e44e3e0ca69dff..2b54717bbaea3f31089e4b52399dfcddf918baef 100644 --- a/functionsystem/tests/unit/common/CMakeLists.txt +++ b/functionsystem/tests/unit/common/CMakeLists.txt @@ -37,6 +37,7 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/yaml_tool YAML_TOOL_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/service_json SERVICE_JSON_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/http HTTP_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/metrics METRICS_TEST_SRCS) +aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/kube_client KUBE_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/common_flags COMMON_FLAGS_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/leader LEADER_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/explorer EXPLORER_SRCS) @@ -46,6 +47,7 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/observability OBSERVABILITY_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/file_monitor FILE_MONITOR_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/network NETWORK_TEST_SRCS) +aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/resource_lock RESOURCE_LOCK_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/aksk AKSK_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/profile PROFILE_TEST_SRCS) @@ -74,6 +76,7 @@ target_sources( ${TRACE_SRCS} ${HTTP_TEST_SRCS} ${METRICS_TEST_SRCS} + ${KUBE_TEST_SRCS} ${COMMON_FLAGS_SRCS} ${LEADER_SRCS} ${EXPLORER_SRCS} @@ -82,6 +85,7 @@ target_sources( ${OBSERVABILITY_TEST_SRCS} ${FILE_MONITOR_TEST_SRCS} ${NETWORK_TEST_SRCS} + ${RESOURCE_LOCK_SRCS} ${AKSK_TEST_SRCS} ${PROFILE_TEST_SRCS} ${SCHEDULE_PLUGIN_SRCS}) diff --git a/functionsystem/tests/unit/common/aksk/aksk_test.cpp b/functionsystem/tests/unit/common/aksk/aksk_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..33be3fbdb466a7485aff372e36d0c65d7125ec9d --- /dev/null +++ b/functionsystem/tests/unit/common/aksk/aksk_test.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/aksk/aksk_util.h" +#include "common/aksk/sign_request.h" +#include "common/utils/aksk_content.h" + +namespace functionsystem::test { + +class AKSKManagerTest : public ::testing::Test {}; + +TEST_F(AKSKManagerTest, SignAndVerifyHttpRequest_Test) +{ + KeyForAKSK key{ "ak", SensitiveValue("xxxxx"), SensitiveValue("defdf") }; + + std::string method = "GET"; + std::string path = "/getAAA"; + std::map initMap = { { "q1", "value1" }, { "q2", "value2" }, { "q3", "value3" } }; + std::shared_ptr> queries = + std::make_shared>(initMap); + + std::map headers = { { "h1", "value1" }, { "h2", "value2" }, { "h3", "value3" } }; + std::string body = R"({"aaa":"bbb"})"; + std::string ak = "ak"; + std::map out = SignHttpRequest(SignRequest(method, path, queries, headers, body), key); + headers[HEADER_TOKEN_KEY] = out[HEADER_TOKEN_KEY]; + headers[HEADER_SIGNED_HEADER_KEY] = out[HEADER_SIGNED_HEADER_KEY]; + EXPECT_EQ(out[HEADER_SIGNED_HEADER_KEY], "h1;h2;h3"); + bool verify = VerifyHttpRequest(SignRequest(method, path, queries, headers, body), key); + EXPECT_EQ(verify, true); + headers["aaa"] = "bbb"; + verify = VerifyHttpRequest(SignRequest(method, path, queries, headers, body), key); + EXPECT_EQ(verify, true); + headers["h1"] = "v1"; + verify = VerifyHttpRequest(SignRequest(method, path, queries, headers, body), key); + EXPECT_EQ(verify, false); +} + +TEST_F(AKSKManagerTest, SignAndVerifyActorMsg_Test) +{ + KeyForAKSK key{ "ak", SensitiveValue("xxxxxx"), SensitiveValue("defdf") }; + + std::string msgName = "func1"; + std::string msgBody = "{xxxx:xxxx}"; + std::string ak = "ak"; + + std::string actorOut = SignActorMsg(msgName, msgBody, key); + auto verify = VerifyActorMsg(msgName, msgBody, actorOut, key); + EXPECT_EQ(verify, true); + msgBody = "{xxxx:xxx}"; + verify = VerifyActorMsg(msgName, msgBody, actorOut, key); + EXPECT_EQ(verify, false); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/common_flags/common_flags_test.cpp b/functionsystem/tests/unit/common/common_flags/common_flags_test.cpp index ef1e605e2f4752a7a8f131629edf4445616d9e0e..00a1f4de4259afab43e27a37792ab86ec84d06e1 100644 --- a/functionsystem/tests/unit/common/common_flags/common_flags_test.cpp +++ b/functionsystem/tests/unit/common/common_flags/common_flags_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "common_flags/common_flags.h" +#include "common/common_flags/common_flags.h" #include #include diff --git a/functionsystem/tests/unit/common/etcd_service/etcd_service_driver.cpp b/functionsystem/tests/unit/common/etcd_service/etcd_service_driver.cpp index 139032b30915c490a0ca5f3905b660aa72d75a70..2e33cbb2f281aba0978c9856081390dd376cca0a 100644 --- a/functionsystem/tests/unit/common/etcd_service/etcd_service_driver.cpp +++ b/functionsystem/tests/unit/common/etcd_service/etcd_service_driver.cpp @@ -22,7 +22,7 @@ #include "common/etcd_service/etcd_kv_service.h" #include "common/etcd_service/etcd_lease_service.h" #include "common/etcd_service/etcd_watch_service.h" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "grpcpp/server_builder.h" namespace functionsystem::meta_store::test { @@ -68,7 +68,7 @@ void EtcdServiceDriver::StartServer(const std::string &address, const std::strin void EtcdServiceDriver::StopServer() { if (server_ != nullptr) { - server_->Shutdown(); + server_->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(5)); server_ = nullptr; } diff --git a/functionsystem/tests/unit/common/etcd_service/etcd_watch_service.cpp b/functionsystem/tests/unit/common/etcd_service/etcd_watch_service.cpp index 3b0ea67c0307e23dc173ac38c6e0c2bc47322d6c..85779d8321471e0d445c0285bab48b7ac0220fc8 100644 --- a/functionsystem/tests/unit/common/etcd_service/etcd_watch_service.cpp +++ b/functionsystem/tests/unit/common/etcd_service/etcd_watch_service.cpp @@ -17,7 +17,7 @@ #include "etcd_watch_service.h" #include "async/async.hpp" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "etcd_watch_srv_actor.h" namespace functionsystem::meta_store::test { diff --git a/functionsystem/tests/unit/common/etcd_service/etcd_watch_srv_actor.cpp b/functionsystem/tests/unit/common/etcd_service/etcd_watch_srv_actor.cpp index 7f41e44f85e7aa0dedf435d8b9e69ddca7e2a910..fb522e72f50a30a41bd960008b92a01d4c63a010 100644 --- a/functionsystem/tests/unit/common/etcd_service/etcd_watch_srv_actor.cpp +++ b/functionsystem/tests/unit/common/etcd_service/etcd_watch_srv_actor.cpp @@ -16,7 +16,7 @@ #include "etcd_watch_srv_actor.h" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "meta_store_common.h" namespace functionsystem::meta_store::test { diff --git a/functionsystem/tests/unit/common/explorer/explorer_test.cpp b/functionsystem/tests/unit/common/explorer/explorer_test.cpp index 5252c8fc3469fd3846ea83e9e1a8a03301043101..f61df22bb7acb51495e72c8356943cd78e9c7616 100644 --- a/functionsystem/tests/unit/common/explorer/explorer_test.cpp +++ b/functionsystem/tests/unit/common/explorer/explorer_test.cpp @@ -24,10 +24,12 @@ #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" #include "common/explorer/etcd_explorer_actor.h" +#include "common/explorer/k8s_explorer_actor.h" #include "common/explorer/txn_explorer_actor.h" -#include "logs/logging.h" -#include "metadata/metadata.h" +#include "common/logs/logging.h" +#include "common/metadata/metadata.h" #include "common/utils/generate_message.h" +#include "mocks/mock_kube_client.h" #include "mocks/mock_meta_store_client.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -69,7 +71,7 @@ TEST_F(ExplorerTest, StandaloneMode) // TEST 1: New leader info, and new standalone explorer explorer::LeaderInfo leaderInfo{ .name = DEFAULT_MASTER_ELECTION_KEY, .address = "123" }; explorer::ElectionInfo electionInfo{ .identity = "123", .mode = STANDALONE_MODE }; - (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_); + (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_, nullptr, "default"); LeaderInfo cachedLeaderInfo; // TEST 1.1: register callback @@ -83,7 +85,7 @@ TEST_F(ExplorerTest, EtcdElectionMode) { explorer::LeaderInfo leaderInfo{ .name = DEFAULT_MASTER_ELECTION_KEY, .address = "123" }; explorer::ElectionInfo electionInfo{ .identity = "123", .mode = ETCD_ELECTION_MODE }; - (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_); + (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_, nullptr, "default"); auto explorerActor = Explorer::GetInstance().GetExplorer(DEFAULT_MASTER_ELECTION_KEY); @@ -127,14 +129,39 @@ TEST_F(ExplorerTest, EtcdElectionMode) litebus::Await(explorerActor->GetAID()); } +TEST_F(ExplorerTest, K8sElectionMode) +{ + explorer::LeaderInfo leaderInfo{ .name = FUNCTION_MASTER_K8S_LEASE_NAME, .address = "123" }; + explorer::ElectionInfo electionInfo{ + .identity = "123", + .mode = K8S_ELECTION_MODE, + .electKeepAliveInterval = 30, + .electLeaseTTL = 300, + }; + auto mockClient1 = std::make_shared(); + (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_, mockClient1, "default"); + LeaderInfo cachedLeaderInfo; + // TEST 1: register callback + Explorer::GetInstance().AddLeaderChangedCallback( + "cbid", [&cachedLeaderInfo](const LeaderInfo &lf) { cachedLeaderInfo = lf; }); + auto lease = MockKubeClient::CreateLease("123"); + litebus::Promise> promise; + promise.SetValue(lease); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillRepeatedly(testing::Return(promise.GetFuture())); + auto explorerActor = Explorer::GetInstance().GetExplorer(FUNCTION_MASTER_K8S_LEASE_NAME); + litebus::Async(explorerActor->GetAID(), &K8sExplorerActor::FastPublish, LeaderInfo{}); + litebus::Async(explorerActor->GetAID(), &K8sExplorerActor::Observe); + ASSERT_AWAIT_TRUE([&]() -> bool { return cachedLeaderInfo.address == "123"; }); +} + TEST_F(ExplorerTest, TxnElectionMode) { auto watcher = std::make_shared([](int64_t watchID) {}); - EXPECT_CALL(*mockMetaClient_, GetAndWatch).WillOnce(testing::Return(watcher)); + EXPECT_CALL(*mockMetaClient_, GetAndWatch(testing::_,testing::_,testing::_,testing::_)).WillOnce(testing::Return(watcher)); explorer::ElectionInfo electionInfo{ .identity = "123", .mode = TXN_ELECTION_MODE }; explorer::LeaderInfo leaderInfo{ .name = DEFAULT_MASTER_ELECTION_KEY, .address = "123" }; - (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_); + (void)Explorer::CreateExplorer(electionInfo, leaderInfo, mockMetaClient_, nullptr, "default"); auto actor = Explorer::GetInstance().GetExplorer(DEFAULT_MASTER_ELECTION_KEY); auto *explorer = dynamic_cast(actor.get()); diff --git a/functionsystem/tests/unit/common/explorer/txn_explorer_test.cpp b/functionsystem/tests/unit/common/explorer/txn_explorer_test.cpp index 2880eac76a318fac5e0af148b5bca371853020b1..9520cc6233a8b964ec862821b3e59845f7756cd5 100644 --- a/functionsystem/tests/unit/common/explorer/txn_explorer_test.cpp +++ b/functionsystem/tests/unit/common/explorer/txn_explorer_test.cpp @@ -21,7 +21,7 @@ #include "common/constants/actor_name.h" #include "common/etcd_service/etcd_service_driver.h" #include "common/explorer/txn_explorer_actor.h" -#include "metadata/metadata.h" +#include "common/metadata/metadata.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -32,14 +32,14 @@ class TxnExplorerTest : public ::testing::Test { public: void SetUp() override { - EXPECT_AWAIT_READY(metaStoreClient_->Delete("/", { .prevKv = false, .prefix = true })); + EXPECT_AWAIT_READY(metaStoreClient_->Delete("/", { .prefix = true })); } void TearDown() override { } - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -48,7 +48,7 @@ public: MetaStoreClient::Create({ .etcdAddress = "127.0.0.1:" + std::to_string(metaStoreServerPort) }, {}, {}); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { metaStoreClient_ = nullptr; @@ -89,7 +89,9 @@ TEST_F(TxnExplorerTest, TxnElectionTest) EXPECT_AWAIT_READY(metaStoreClient_->Put(DEFAULT_MASTER_ELECTION_KEY, "127.0.0.1:81", {})); EXPECT_TRUE(actor->cachedLeaderInfo_.address == "127.0.0.1:80"); - auto result = litebus::Async(aid, &TxnExplorerActor::Sync); + auto getResponse = metaStoreClient_->Get(DEFAULT_MASTER_ELECTION_KEY, {}); + ASSERT_AWAIT_READY(getResponse); + auto result = litebus::Async(aid, &TxnExplorerActor::Sync, getResponse.Get()); ASSERT_AWAIT_READY(result); EXPECT_TRUE(result.Get().status.IsOk()); diff --git a/functionsystem/tests/unit/common/file_monitor/file_monitor_test.cpp b/functionsystem/tests/unit/common/file_monitor/file_monitor_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b9ebedf331d3dabba2d5ec8534e79d59f3034cf7 --- /dev/null +++ b/functionsystem/tests/unit/common/file_monitor/file_monitor_test.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "common/file_monitor/file_monitor.h" +#include "common/utils/exec_utils.h" +#include "common/utils/files.h" +#include "utils/os_utils.hpp" +#include "utils/future_test_helper.h" + +namespace functionsystem::test { + +using namespace std; +using namespace functionsystem; + +const std::string FILE_PATH = "/home/sn/function/config"; +const std::string FILE_NAME = "system-function-config.json"; + +const std::string content1 = R"( +{ + "0-system-faascontroller": { + "tenantID":"12345678901234561234567890123456", + "version":"001", + "memory": 666, + "cpu": 666, + "instanceNum": 1, + "schedulingOps": {}, + "createOptions": {}, + } +} +)"; + +const std::string content2 = R"( +{ + "0-system-faascontroller": { + "tenantID":"12345678901234561234567890123456", + "version":"002", + "memory": 666, + "cpu": 666, + "instanceNum": 1, + "schedulingOps": {}, + "createOptions": {}, + } +} +)"; + +class FileMonitorTest : public ::testing::Test { +protected: + std::shared_ptr monitor_{ nullptr }; +}; + +TEST_F(FileMonitorTest, AddWatchTest) +{ + if (!litebus::os::ExistPath(FILE_PATH)) { + litebus::os::Mkdir(FILE_PATH); + } + auto file = FILE_PATH + "/" + FILE_NAME; + EXPECT_TRUE(Write(file, content1)); + + std::function callback = + [](const std::string &path, const std::string &name, uint32_t mask) { + EXPECT_EQ(name, FILE_NAME); + if (mask & IN_MODIFY) { + YRLOG_INFO("callback: path({}), name({}), mask({})", path, name, mask); + } + }; + + monitor_ = std::make_shared(); + auto future = monitor_->Start(); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get(), true); + + future = monitor_->AddWatch(FILE_PATH, callback); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get(), true); + + std::thread modifyFile([file]() { + std::ofstream out; + out.open(file); + out.flush(); + out << content2; + out.close(); + }); + modifyFile.join(); + EXPECT_EQ(Read(file), content2); + + future = monitor_->RemoveWatch(FILE_PATH); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get(), true); + + monitor_->Stop(); + litebus::os::Rmdir(FILE_PATH); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/file_monitor/monitor_callback_actor_test.cpp b/functionsystem/tests/unit/common/file_monitor/monitor_callback_actor_test.cpp index 41ddf722384701190b401a6077b176f3fccacf0b..eecee7c4d9a7441f2ad3a27937981ace8c49aebc 100644 --- a/functionsystem/tests/unit/common/file_monitor/monitor_callback_actor_test.cpp +++ b/functionsystem/tests/unit/common/file_monitor/monitor_callback_actor_test.cpp @@ -14,12 +14,13 @@ * limitations under the License. */ +#include "common/file_monitor/monitor_callback_actor.h" + #include #include +#include "async/async.hpp" #include "utils/os_utils.hpp" -#include "status/status.h" -#include "common/file_monitor/monitor_callback_actor.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/common/heartbeat/heartbeat_test.cpp b/functionsystem/tests/unit/common/heartbeat/heartbeat_test.cpp index 546aedf7022ff39f52cead1f30e3bb644335539b..b6fd3dfb30e77f70c5283f3319d54156feac6663 100644 --- a/functionsystem/tests/unit/common/heartbeat/heartbeat_test.cpp +++ b/functionsystem/tests/unit/common/heartbeat/heartbeat_test.cpp @@ -14,183 +14,186 @@ * limitations under the License. */ -#include #include +#include #include -#include "heartbeat/heartbeat_observer.h" -#include "heartbeat/ping_pong_driver.h" -#include "heartbeat/heartbeat_observer_ctrl.h" -#include "ChildHeartbeatObserver.h" #include "utils/port_helper.h" +#define private public +#include "../../../integration/utils/future_test_helper.h" +#include "common/heartbeat/heartbeat_client.h" +#include "common/heartbeat/heartbeat_observer.h" + namespace functionsystem::test { class Heartbeat : public ::testing::Test {}; -TEST_F(Heartbeat, ObserverWithInvalidDst) +TEST_F(Heartbeat, ObserverWithInvalidActorName) { - HeartbeatObserveDriver heartbeatDriver("pinger", "invalid_dst", [](const litebus::AID &) {}); - auto ret = heartbeatDriver.Start(); - EXPECT_LE(ret, static_cast(StatusCode::CONN_ERROR)); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 50, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + ASSERT_AWAIT_TRUE([=]() -> bool { + return heartbeatClient->GetStatus().IsError(); + }); + heartbeatClient->Stop(); + litebus::Terminate(heartbeatClient->GetAID()); + litebus::Await(heartbeatClient); } -TEST_F(Heartbeat, ObserverWithInvalidActorName) +TEST_F(Heartbeat, ObserverSuccessWithoutAddNode) { - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); - HeartbeatObserveDriver heartbeatDriver("pinger", "invalid_dst@127.0.0.1:" + std::to_string(port), - [](const litebus::AID &) {}); - auto ret = heartbeatDriver.Start(); - EXPECT_LE(ret, 0); + auto observer = std::make_shared("Test"); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 100, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + ASSERT_AWAIT_TRUE([=]() -> bool { + return heartbeatClient->GetStatus().IsOk(); + }); + litebus::Terminate(heartbeatClient->GetAID()); } -TEST_F(Heartbeat, ObserverNormalExited) +TEST_F(Heartbeat, ObserverSuccessAddNode) { - litebus::Promise actorNamePromise; - litebus::Promise lostTypePromise; - - PingPongDriver pingpong("pinged", 1000, - [actorNamePromise, lostTypePromise](const litebus::AID &aid, HeartbeatConnection type) { - actorNamePromise.SetValue(aid.Name()); - lostTypePromise.SetValue(type); - }); - litebus::AID observer; - { - HeartbeatObserveDriver heartbeatDriver("pinger", pingpong.GetActorAID(), 5, 10, [](const litebus::AID &) {}); - auto ret = heartbeatDriver.Start(); - EXPECT_EQ(ret, 0); - observer = heartbeatDriver.GetActorAID(); - } - auto name = actorNamePromise.GetFuture().Get(1000); - EXPECT_EQ(name.IsSome(), true); - EXPECT_EQ(name.Get(), observer.Name()); - - auto type = lostTypePromise.GetFuture().Get(1000); - EXPECT_EQ(type.IsSome(), true); - EXPECT_EQ(type.Get(), HeartbeatConnection::EXITED); + auto observer = std::make_shared("Test"); + bool heartbeatLost = false; + observer->AddNewHeartbeatNode( + HEARTBEAT_CLIENT_BASENAME + "pinger", 300, + [&heartbeatLost](const litebus::AID &, HeartbeatConnection) { heartbeatLost = true; }); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 100, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + ASSERT_AWAIT_TRUE([=]() -> bool { + return heartbeatClient->GetStatus().IsOk(); + }); + litebus::Terminate(heartbeatClient->GetAID()); } -TEST_F(Heartbeat, ObserverDetectedRemoteExited) +TEST_F(Heartbeat, ObserverSuccessClientDied) { - litebus::Promise actorNamePromise; - - PingPongDriver pingpong("pinged", 1000, [](const litebus::AID &aid, HeartbeatConnection type) {}); - HeartbeatObserveDriver heartbeatDriver("pinger", pingpong.GetActorAID(), 5, 10, - [actorNamePromise](const litebus::AID &aid) { actorNamePromise.SetValue(aid.Name()); }); - auto ret = heartbeatDriver.Start(); - EXPECT_EQ(ret, 0); - - litebus::Terminate(pingpong.GetActorAID()); - litebus::Await(pingpong.GetActorAID()); - - auto pingAID = pingpong.GetActorAID(); - auto name = actorNamePromise.GetFuture().Get(1000); - EXPECT_EQ(name.IsSome(), true); - EXPECT_EQ(name.Get(), pingAID.Name()); + auto observer = std::make_shared("Test"); + bool heartbeatLost = false; + HeartbeatConnection reason = HeartbeatConnection::EXITED; + observer->AddNewHeartbeatNode(HEARTBEAT_CLIENT_BASENAME + "pinger", 300, + [&heartbeatLost, &reason](const litebus::AID &, HeartbeatConnection connection) { + heartbeatLost = true; + reason = connection; + }); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 100, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + litebus::Terminate(heartbeatClient->GetAID()); + litebus::Await(heartbeatClient->GetAID()); + ASSERT_AWAIT_TRUE([&heartbeatLost]() -> bool { + return heartbeatLost; + }); + EXPECT_EQ(reason, HeartbeatConnection::LOST); } -TEST_F(Heartbeat, ObserverDetectTimeOut) +TEST_F(Heartbeat, ObserverSuccessObserverDied) { - class NoResponsePingPong : public PingPongActor { - public: - NoResponsePingPong(std::string name) - : PingPongActor(name, 1000, [](const litebus::AID &, HeartbeatConnection) {}) - {} - ~NoResponsePingPong() override = default; - void Ping(const litebus::AID &from, std::string &&name, std::string &&msg) override - { - count_++; - } - int count_ = 0; - }; - - auto noResponse = std::make_shared("NoResponse"); - (void)litebus::Spawn(noResponse); - - int maxPingTimeoutNums = 5; - litebus::Promise timeoutActor; - HeartbeatObserveDriver heartbeatDriver("pinger", noResponse->GetAID(), maxPingTimeoutNums, 10, - [timeoutActor](const litebus::AID &actor) { timeoutActor.SetValue(actor.Name()); }); - auto ret = heartbeatDriver.Start(); - EXPECT_EQ(ret, 0); - - auto timeoutCallBackResult = timeoutActor.GetFuture().Get(100); - EXPECT_EQ(timeoutCallBackResult.IsSome(), true); - EXPECT_EQ(timeoutCallBackResult.Get(), noResponse->GetAID().Name()); - EXPECT_EQ(noResponse->count_, maxPingTimeoutNums); - litebus::Terminate(noResponse->GetAID()); - litebus::Await(noResponse->GetAID()); + auto observer = + std::make_shared(HEARTBEAT_OBSERVER_BASENAME + "Test", DEFAULT_PING_PONG_TIMEOUT); + litebus::Spawn(observer); + bool heartbeatLost = false; + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared( + "pinger", 2, 100, [&heartbeatLost](const litebus::AID &) { heartbeatLost = true; }); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + litebus::Terminate(observer->GetAID()); + litebus::Await(observer->GetAID()); + ASSERT_AWAIT_TRUE([&heartbeatLost]() -> bool { + return heartbeatLost; + }); + litebus::Terminate(heartbeatClient->GetAID()); + litebus::Await(heartbeatClient->GetAID()); } -TEST_F(Heartbeat, PingPongActorDetectTimeout) +TEST_F(Heartbeat, ObserverSuccessClinetExit) { - litebus::Promise actorNamePromise; - litebus::Promise lostTypePromise; - - // pingIntervalMs > pingpongTimeMs can simulating pingpong get ping request timeout. - uint32_t pingpongTimeMs = 100; - uint32_t pingIntervalMs = 1000; - PingPongDriver pingpong("pinged", pingpongTimeMs, - [actorNamePromise, lostTypePromise](const litebus::AID &aid, HeartbeatConnection type) { - actorNamePromise.SetValue(aid.Name()); - lostTypePromise.SetValue(type); - }); - HeartbeatObserveDriver heartbeatDriver("pinger", pingpong.GetActorAID(), 5, pingIntervalMs, - [](const litebus::AID &) {}); - auto ret = heartbeatDriver.Start(); - EXPECT_EQ(ret, 0); - auto observer = heartbeatDriver.GetActorAID(); - - auto name = actorNamePromise.GetFuture().Get(500); - EXPECT_EQ(name.IsSome(), true); - EXPECT_EQ(name.Get(), observer.Name()); - - auto type = lostTypePromise.GetFuture().Get(500); - EXPECT_EQ(type.IsSome(), true); - EXPECT_EQ(type.Get(), HeartbeatConnection::LOST); + auto observer = std::make_shared("Test"); + bool heartbeatLost = false; + HeartbeatConnection reason = HeartbeatConnection::LOST; + observer->AddNewHeartbeatNode(HEARTBEAT_CLIENT_BASENAME + "pinger", 300, + [&heartbeatLost, &reason](const litebus::AID &, HeartbeatConnection connection) { + heartbeatLost = true; + reason = connection; + }); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 100, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + heartbeatClient->Stop(); + ASSERT_AWAIT_TRUE([&heartbeatLost]() -> bool { + return heartbeatLost; + }); + EXPECT_EQ(reason, HeartbeatConnection::LOST); + litebus::Terminate(heartbeatClient->GetAID()); + litebus::Await(heartbeatClient->GetAID()); } -/** - * Feature: HeartbeatObserver. - * Description: To test HeartbeatObserver functions. - * Steps: - * 1. Init HeartbeatObserver. - * Expectation: make parameters and invoke method to cover the - * HeartbeatObserveDriver\HeartbeatObserver basic path codes - */ -TEST_F(Heartbeat, ObserverNormalStop) +TEST_F(Heartbeat, ObserverResetHandler) { - litebus::Promise actorNamePromise; - litebus::Promise lostTypePromise; - - PingPongDriver pingpong("pinged", 1000, - [actorNamePromise, lostTypePromise](const litebus::AID &aid, HeartbeatConnection type) { - actorNamePromise.SetValue(aid.Name()); - lostTypePromise.SetValue(type); - }); - - HeartbeatObserveDriver heartbeatDriver("pinger", pingpong.GetActorAID(), 5, 10, [](const litebus::AID &) {}); - auto ret = heartbeatDriver.Start(); - EXPECT_EQ(ret, 0); - // repeat start return 0 - EXPECT_EQ(heartbeatDriver.Start(), 0); - auto ret1 = heartbeatDriver.Stop(); - EXPECT_EQ(ret1, 1); - - ChildHeartbeatObserver childHeartbeatObserver("pinger", pingpong.GetActorAID(), [](const litebus::AID &) {}); - childHeartbeatObserver.Exited(pingpong.GetActorAID()); - - functionsystem::HeartbeatObserverCtrl heartbeatObserverCtrl(3, 100); - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); - litebus::Future statusFuture = heartbeatObserverCtrl.Add( - pingpong.GetActorAID(), "127.0.0.1:" + std::to_string(port), [](const litebus::AID &) {}); - EXPECT_EQ(statusFuture.Get().StatusCode(), StatusCode::SUCCESS); - - // repeated heartbeatObserverCtrl add action - litebus::Future statusFutureRepeated = heartbeatObserverCtrl.Add( - pingpong.GetActorAID(), "127.0.0.1:" + std::to_string(port), [](const litebus::AID &) {}); - EXPECT_EQ(statusFutureRepeated.Get().StatusCode(), StatusCode::SUCCESS); - - heartbeatObserverCtrl.Delete(pingpong.GetActorAID()); + auto observer = std::make_shared("Test"); + bool heartbeatLost = false; + HeartbeatConnection reason = HeartbeatConnection::LOST; + observer->AddNewHeartbeatNode(HEARTBEAT_CLIENT_BASENAME + "pinger", 300, + [&heartbeatLost, &reason](const litebus::AID &, HeartbeatConnection connection) { + heartbeatLost = true; + reason = connection; + }); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::AID aid(HEARTBEAT_OBSERVER_BASENAME + "Test", "127.0.0.1:" + std::to_string(port)); + aid.SetProtocol(litebus::BUS_TCP); + auto heartbeatClient = std::make_shared("pinger", 3, 100, [](const litebus::AID &) {}); + litebus::Spawn(heartbeatClient); + heartbeatClient->Start(aid); + EXPECT_EQ(aid, heartbeatClient->dst_); + EXPECT_TRUE(heartbeatClient->GetStatus().IsOk()); + heartbeatClient->Stop(); + for (auto i = 0; i < 5; i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + observer->ResetHeartbeatNode(HEARTBEAT_CLIENT_BASENAME + "pinger", 300, + [&heartbeatLost, &reason](const litebus::AID &, HeartbeatConnection connection) { + heartbeatLost = true; + reason = connection; + }); + } + EXPECT_FALSE(heartbeatLost); + ASSERT_AWAIT_TRUE([&heartbeatLost]() -> bool { + return heartbeatLost; + }); + litebus::Terminate(heartbeatClient->GetAID()); + litebus::Await(heartbeatClient->GetAID()); } -} // namespace functionsystem::test +} // namespace functionsystem::test diff --git a/functionsystem/tests/unit/common/http/http_test.cpp b/functionsystem/tests/unit/common/http/http_test.cpp index a165e0269ad729af044841016ff2d81b3220c954..c30b7436454884b1c0f6db471b1cb69bc4255771 100644 --- a/functionsystem/tests/unit/common/http/http_test.cpp +++ b/functionsystem/tests/unit/common/http/http_test.cpp @@ -19,8 +19,8 @@ #include -#include "http/api_router_register.h" -#include "http/http_server.h" +#include "common/http/api_router_register.h" +#include "common/http/http_server.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -50,14 +50,14 @@ public: class HttpTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { httpServer_->RegisterRoute(testActor_); litebus::Spawn(testActor_); litebus::Spawn(httpServer_); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { litebus::Terminate(httpServer_->GetAID()); litebus::Await(httpServer_->GetAID()); diff --git a/functionsystem/tests/unit/common/http/http_util_test.cpp b/functionsystem/tests/unit/common/http/http_util_test.cpp index d4c51beed6b6333eded22cd95b9fde71ce0b9606..f60a87d11c9fa2b791b0a54516331563cda482f3 100644 --- a/functionsystem/tests/unit/common/http/http_util_test.cpp +++ b/functionsystem/tests/unit/common/http/http_util_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "http/http_util.h" +#include "common/http/http_util.h" #include "gtest/gtest.h" diff --git a/functionsystem/tests/unit/common/kube_client/CMakeLists.txt b/functionsystem/tests/unit/common/kube_client/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0afc47fd2328d820dc98897cd5c138dd0e5a9144 --- /dev/null +++ b/functionsystem/tests/unit/common/kube_client/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. + +aux_source_directory(../../common/kube_client KUBE_CLIENT_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${KUBE_CLIENT_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/common/kube_client/kube_api_server.cpp b/functionsystem/tests/unit/common/kube_client/kube_api_server.cpp new file mode 100644 index 0000000000000000000000000000000000000000..18c2604d8dbe2adfdcc31b92b24d9eddfc53355b --- /dev/null +++ b/functionsystem/tests/unit/common/kube_client/kube_api_server.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "kube_api_server.h" + +namespace functionsystem::kube_client::test { +using namespace litebus; +using namespace litebus::http; +using namespace std; + +using litebus::http::HeaderMap; +using litebus::http::HttpConnect; +using litebus::http::Request; +using litebus::http::Response; +using litebus::http::ResponseCode; +using litebus::http::URL; + + +Future KubeApiServer::HandlePodRequest(const Request &request) +{ + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + if (request.method == "POST" || request.method == "PATCH") { + return Ok(request.body, ResponseBodyType::JSON); + } else if(request.method == "DELETE") { + string response = "{\"metadata\":{\"name\":\"function-agent-pod-0\"}}"; + return Ok(response, ResponseBodyType::JSON); + } else { + string response = "{\"kind\": \"list\" }"; + return Ok(response, ResponseBodyType::JSON); + } +} + +Future KubeApiServer::HandleNodeRequest(const Request &request) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(timeout_)); + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + if (request.method == "POST" || request.method == "PATCH") { + return Ok(request.body, ResponseBodyType::JSON); + } else { + string response = "{\"kind\": \"list\",\"items\":[],\"metadata\":{}}"; + return Ok(response, ResponseBodyType::JSON); + } +} + +Future KubeApiServer::HandleDeploymentRequest(const Request &request) +{ + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + if (request.method == "POST" || request.method == "PATCH") { + return Ok(request.body, ResponseBodyType::JSON); + } else if(request.method == "DELETE") { + string response = "{\"metadata\":{\"name\":\"function-agent-pool1\"}}"; + return Ok(response, ResponseBodyType::JSON); + } else { + string response = "{\"kind\": \"list\" }"; + return Ok(response, ResponseBodyType::JSON); + } +} + +Future KubeApiServer::HandleLeaseRequest(const litebus::http::Request &request) +{ + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + return Ok(request.body, ResponseBodyType::JSON); +} + +Future KubeApiServer::HandleHorizontalPodAutoscalerRequest(const Request &request) +{ + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + if (request.method == "POST" || request.method == "PATCH") { + return Ok(request.body, ResponseBodyType::JSON); + } else if(request.method == "DELETE") { + string response = "{\"metadata\":{\"name\":\"function-agent-pool1-hpa\"}}"; + return Ok(response, ResponseBodyType::JSON); + } else { + string response = "{\"kind\": \"list\" }"; + return Ok(response, ResponseBodyType::JSON); + } +} + +Future KubeApiServer::HandleNuwaRequest(const Request &request) +{ + if (!responseQueue_.empty()) { + auto resp = Response(responseQueue_.front().first, responseQueue_.front().second, ResponseBodyType::JSON); + responseQueue_.pop(); + return resp; + } + if (!responseOkQueue_.empty()) { + auto resp = responseOkQueue_.front(); + responseOkQueue_.pop(); + return resp; + } + if (request.method == "GET") { + string response = "{\"instanceId\": \"123\"}"; + return Ok(response, ResponseBodyType::JSON); + } + return Ok(request.body, ResponseBodyType::JSON); +} + +Future KubeApiServer::HandleRayRequest(const Request &request) +{ + if (!responseOkQueue_.empty()) { + auto resp = responseOkQueue_.front(); + responseOkQueue_.pop(); + return resp; + } + string response = R"({"spec": {"workerGroupSpecs": [ {"groupName": "group", "replicas": 1 }]}})"; + return Ok(response, ResponseBodyType::JSON); +} +} // namespace functionsystem::kube_client::test diff --git a/functionsystem/tests/unit/common/kube_client/kube_api_server.h b/functionsystem/tests/unit/common/kube_client/kube_api_server.h new file mode 100644 index 0000000000000000000000000000000000000000..a72662cef33dd120aac2c4f19fdc493c8a95d6fd --- /dev/null +++ b/functionsystem/tests/unit/common/kube_client/kube_api_server.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FUNCTIONSYSTEM_TEST_UNIT_KUBE_API_SERVER_H +#define FUNCTIONSYSTEM_TEST_UNIT_KUBE_API_SERVER_H + +#include +#include + +#include "async/future.hpp" +#include "async/async.hpp" +#include "litebus.hpp" +#include "httpd/http.hpp" +#include "httpd/http_connect.hpp" +#include "httpd/http_actor.hpp" + + +namespace functionsystem::kube_client::test { +using namespace litebus; +using namespace litebus::http; + +using litebus::http::HeaderMap; +using litebus::http::HttpConnect; +using litebus::http::Request; +using litebus::http::Response; +using litebus::http::ResponseCode; +using litebus::http::URL; + +class KubeApiServer : public litebus::http::HttpActor { +public: + explicit KubeApiServer(const std::string &name) : HttpActor(name) + { + } + + void SetResponseQueue(const std::queue> &responseQueue) + { + responseQueue_ = responseQueue; + } + + void SetResponse(const Response &response) + { + responseOkQueue_.push(response); + } + + std::queue> GetResponseQueue() + { + return responseQueue_; + } + + void SetTimeout(uint32_t timeout) + { + timeout_ = timeout; + } +protected: + virtual void Init() override + { + AddRoute("/api/v1/namespaces/default/pods", &KubeApiServer::HandlePodRequest); + AddRoute("/api/v1/nodes", &KubeApiServer::HandleNodeRequest); + AddRoute("/apis/apps/v1/namespaces/default/deployments", &KubeApiServer::HandleDeploymentRequest); + AddRoute("/apis/coordination.k8s.io/v1/namespaces/default/leases", &KubeApiServer::HandleLeaseRequest); + AddRoute("/apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers", &KubeApiServer::HandleHorizontalPodAutoscalerRequest); + AddRoute("/nuwa-runtime", &KubeApiServer::HandleNuwaRequest); + AddRoute("/apis/ray.io/v1/namespaces", &KubeApiServer::HandleRayRequest); + } + + Future HandlePodRequest(const Request &request); + + Future HandleNodeRequest(const Request &request); + + Future HandleDeploymentRequest(const Request &request); + + Future HandleLeaseRequest(const Request &request); + + Future HandleHorizontalPodAutoscalerRequest(const Request &request); + + Future HandleNuwaRequest(const Request &request); + + Future HandleRayRequest(const Request &request); + + std::queue> responseQueue_; + + std::queue responseOkQueue_; + + uint32_t timeout_{0}; +}; +} // namespace functionsystem::kube_client::test + +#endif //FUNCTIONSYSTEM_TEST_UNIT_KUBE_API_SERVER_H \ No newline at end of file diff --git a/functionsystem/tests/unit/common/kube_client/kube_client_api_test.cpp b/functionsystem/tests/unit/common/kube_client/kube_client_api_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c92636bdaab370030936d5f98b0ea658348b980f --- /dev/null +++ b/functionsystem/tests/unit/common/kube_client/kube_client_api_test.cpp @@ -0,0 +1,1139 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/kube_client/api/core_v1_api.h" +#include "common/kube_client/kube_client.h" +#include "kube_api_server.h" +#include "mocks/mock_kube_client.h" +#include "utils/future_test_helper.h" +#include "utils/port_helper.h" + +namespace functionsystem::kube_client::test { +using namespace functionsystem::test; +using namespace functionsystem::kube_client::model; +using namespace functionsystem::kube_client::api; + +class KubeClientApiTest : public ::testing::Test { +public: + void SetUpApiClient() + { + SslConfig sslConfig{ .clientCertFile = "", .clientKeyFile = "", .caCertFile = "", + .credential = "k8sClient", .isSkipTlsVerify = false }; + HealthMonitorParam param{ + .maxFailedTimes = 5, + .checkIntervalMs = 20, + }; + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + kubeClient_ = + KubeClient::CreateKubeClient("http://127.0.0.1:" + std::to_string(port) + "/k8s", sslConfig, true, param); + kubeClient_->SetK8sClientRetryTime(2); + kubeClient_->SetK8sClientCycMs(20); + kubeClient_->SetRetryCycMsUpper(10); + apiClient_ = std::make_shared(std::make_shared()); + } + + void SetUp() override + { + SetUpApiClient(); + apiServer = std::make_shared("k8s"); + litebus::Spawn(apiServer); + } + + void TearDown() override + { + auto healthMonitor = kubeClient_->GetHealthMonitor(); + if (healthMonitor != nullptr) { + litebus::Terminate(healthMonitor->GetAID()); + litebus::Await(healthMonitor->GetAID()); + } + kubeClient_ = nullptr; + litebus::Terminate(apiServer->GetAID()); + litebus::Await(apiServer->GetAID()); + } +protected: + std::shared_ptr apiClient_ = nullptr; + std::shared_ptr CoreV1Api_ = nullptr; + std::shared_ptr apiServer = nullptr; + std::shared_ptr kubeClient_ = nullptr; + + static std::shared_ptr CreatePatchNodeBody() + { + std::shared_ptr node = std::make_shared(); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("node001"); + node->SetMetadata(metadata); + std::shared_ptr newNode = std::make_shared(); + std::shared_ptr newMetadata = std::make_shared(); + newMetadata->SetName("node001"); + newNode->SetMetadata(newMetadata); + std::shared_ptr newNodeSpec = std::make_shared(); + newNodeSpec->SetTaints({}); + newNode->SetSpec(newNodeSpec); + nlohmann::json nodeDiffPatch = nlohmann::json::diff(node->ToJson(), newNode->ToJson()); + std::shared_ptr body = + std::make_shared(); + (void)body->FromJson(nodeDiffPatch); + return body; + } + + static std::shared_ptr CreatePatchNamespacedDeploymentBody() + { + std::shared_ptr deployment = std::make_shared(); + deployment->SetApiVersion("0"); + deployment->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-300m-300mi"); + deployment->SetMetadata(metadata); + std::shared_ptr newDeployment = std::make_shared(); + newDeployment->SetApiVersion("1"); + newDeployment->SetKind("1"); + std::shared_ptr newMetadata = std::make_shared(); + newMetadata->SetName("function-agent-300m-300mi"); + newDeployment->SetMetadata(newMetadata); + nlohmann::json podDiffPatch = nlohmann::json::diff(deployment->ToJson(), newDeployment->ToJson()); + std::shared_ptr body = + std::make_shared(); + (void)body->FromJson(podDiffPatch); + return body; + } + + static std::shared_ptr CreatePatchNamespacedPod() + { + std::shared_ptr pod = std::make_shared(); + pod->SetApiVersion("0"); + pod->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + pod->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + pod->SetSpec(Spec); + + std::shared_ptr newPod = std::make_shared(); + pod->SetApiVersion("1"); + pod->SetKind("1"); + std::shared_ptr newMetadata = std::make_shared(); + newMetadata->SetName("function-agent-pod-0"); + newPod->SetMetadata(newMetadata); + nlohmann::json podDiffPatch = nlohmann::json::diff(pod->ToJson(), newPod->ToJson()); + std::shared_ptr body = + std::make_shared(); + (void)body->FromJson(podDiffPatch); + return body; + } + + static std::shared_ptr CreateNamespacedPodBody() + { + std::shared_ptr body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + body->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + body->SetSpec(Spec); + return body; + } + + static std::shared_ptr CreateNamespacedDeploymentBody() + { + std::shared_ptr body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-300m-300mi"); + body->SetMetadata(metadata); + return body; + } + + static std::shared_ptr CreateNamespacedLeaseBody() + { + std::shared_ptr body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("Lease"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-master-lease"); + body->SetMetadata(metadata); + return body; + } + + void SetRetryResponseQueue (int time, ResponseCode code) { + std::queue> responseQueue; + for (int i = 0; i < time; i++) { + responseQueue.emplace(code, ""); + } + apiServer->SetResponseQueue(responseQueue); + } + + template + static void CheckFutureError(const litebus::Future &future, int32_t errorCode) + { + ASSERT_AWAIT_TRUE([future]() { return future.IsError(); }); + EXPECT_EQ(future.GetErrorCode(), errorCode); + } +}; + +const std::string DEFAULT_LIST_JSON_STR = R"( +{ + "kind": "List", + "metadata": { + "resourceVersion": "", + "selfLink": "" + }, + "apiVersion": "v1", + "items": [] +} +)"; + +const std::string DEFAULT_NODE_JSON_STR = R"( +{ + "apiVersion": "v1", + "kind": "Node", + "metadata": { + "name": "node-001" + }, + "spec": { + }, + "status": { + "addresses": [ + { + "address": "10.247.23.146", + "type": "InternalIP" + }] + } +} +)"; + +const std::string DEFAULT_POD_JSON_STR = R"( +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "pod-001", + "namespace": "default" + }, + "spec": { + } +} +)"; + +const std::string DEFAULT_DEPLOYMENT_JSON_STR = R"( +{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": { + "name": "deploy-001" + }, + "spec": { + } +} +)"; + +TEST_F(KubeClientApiTest, testRegisterK8sStatusChangeHandler) +{ + std::function callback = [=] {}; + auto result = kubeClient_->RegisterK8sStatusChangeHandler(K8sStatusChangeEvent::ON_MAX, "UT", callback); + EXPECT_FALSE(result); + result = kubeClient_->RegisterK8sStatusChangeHandler(K8sStatusChangeEvent::ON_RECOVER, "ON_RECOVER_subscriber1", callback); + EXPECT_TRUE(result); + result = kubeClient_->RegisterK8sStatusChangeHandler(K8sStatusChangeEvent::ON_RECOVER, "ON_RECOVER_subscriber2", callback); + EXPECT_TRUE(result); + result = kubeClient_->RegisterK8sStatusChangeHandler(K8sStatusChangeEvent::ON_FAIL, "ON_FAIL_subscriber3", callback); + EXPECT_TRUE(result); + auto k8sStatusChangeHandlerMap = kubeClient_->GetK8sStatusChangeHandlerMap(); + auto handlerMap = k8sStatusChangeHandlerMap.find(K8sStatusChangeEvent::ON_RECOVER)->second; + EXPECT_TRUE(k8sStatusChangeHandlerMap.find(K8sStatusChangeEvent::ON_RECOVER)->second.find("ON_RECOVER_subscriber1") != handlerMap.end()); + EXPECT_TRUE(k8sStatusChangeHandlerMap.find(K8sStatusChangeEvent::ON_RECOVER)->second.find("ON_RECOVER_subscriber2") != handlerMap.end()); + EXPECT_TRUE(k8sStatusChangeHandlerMap.find(K8sStatusChangeEvent::ON_FAIL)->second.find("ON_FAIL_subscriber3") != handlerMap.end()); +} + +TEST_F(KubeClientApiTest, testOnK8sRecoverCallback) +{ + bool isCallbackCalled = false; + std::function callback = [&] { + isCallbackCalled = true; + }; + auto result = kubeClient_->RegisterK8sStatusChangeHandler(K8sStatusChangeEvent::ON_RECOVER, + "ON_RECOVER_subscriber1", callback); + auto healthMonitor = kubeClient_->GetHealthMonitor(); + healthMonitor->RunHealthCallback(true); + EXPECT_TRUE(result); + EXPECT_TRUE(isCallbackCalled); +} + +TEST_F(KubeClientApiTest, testCreateNamespacedPod) +{ + std::string r_namespace = "default"; + std::shared_ptr body; + auto result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + body->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + body->SetSpec(Spec); + result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, CreatePodWithOwnerReferrence) +{ + kubeClient_->InitOwnerReference("default", "function-master"); + std::shared_ptr functionMasterDeploy = std::make_shared(); + std::shared_ptr masterMeta = std::make_shared(); + masterMeta->SetName("function-master"); + masterMeta->SetUid("master-000001"); + functionMasterDeploy->SetMetadata(masterMeta); + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto body = CreateNamespacedPodBody(); + auto result = kubeClient_->CreateNamespacedPod("default", body); + result.Get(); + EXPECT_TRUE(result.Get()->GetMetadata()->GetOwnerReferences().size() > 0); + EXPECT_EQ(result.Get()->GetMetadata()->GetOwnerReferences()[0]->GetUid(), "master-000001"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + body = CreateNamespacedPodBody(); + result = kubeClient_->CreateNamespacedPod("default", body); + result.Get(); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(KubeClientApiTest, testK8sNormal) +{ + auto healthMonitor = kubeClient_->GetHealthMonitor(); + healthMonitor->RunHealthCallback(true); + std::string r_namespace = "default"; + std::shared_ptr body; + body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + body->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + body->SetSpec(Spec); + auto result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, testK8sFuse) +{ + auto healthMonitor = kubeClient_->GetHealthMonitor(); + healthMonitor->RunHealthCallback(false); + std::string r_namespace = "default"; + std::shared_ptr body; + body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + body->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + body->SetSpec(Spec); + auto result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + EXPECT_EQ(result.GetErrorCode(), StatusCode::ERR_K8S_UNAVAILABLE); +} + +TEST_F(KubeClientApiTest, testK8sFuseThenRecover) +{ + std::string r_namespace = "default"; + std::shared_ptr body; + body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("0"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-agent-pod-0"); + body->SetMetadata(metadata); + std::shared_ptr Spec = std::make_shared(); + body->SetSpec(Spec); + + auto healthMonitor = kubeClient_->GetHealthMonitor(); + healthMonitor->RunHealthCallback(false); + auto result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + EXPECT_EQ(result.GetErrorCode(), StatusCode::ERR_K8S_UNAVAILABLE); + + healthMonitor->RunHealthCallback(true); + result = kubeClient_->CreateNamespacedPod(r_namespace, body); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, testDeleteNamespacedPod) +{ + std::string r_namespace = "default"; + auto result = kubeClient_->DeleteNamespacedPod("function-agent-pod-0", r_namespace); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, testPatchNamespacedPod) +{ + std::string r_namespace = "default"; + auto body = CreatePatchNamespacedPod(); + auto result = kubeClient_->PatchNamespacedPod("function-agent-pod-0", r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, testReadNamespacePod) +{ + std::string r_namespace = "default"; + nlohmann::json podJson = nlohmann::json::parse(DEFAULT_POD_JSON_STR); + std::shared_ptr pod1 = std::make_shared(); + pod1->FromJson(podJson); + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, pod1->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, pod1->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto result = kubeClient_->ReadNamespacedPod(r_namespace, "pod-001"); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "pod-001"); + result = kubeClient_->ReadNamespacedPod(r_namespace, "pod-001"); + result.Get(); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(KubeClientApiTest, testListNamespacedPod) +{ + std::string r_namespace = "default"; + kubeClient_->SetPageLimit(2); + nlohmann::json listJson = nlohmann::json::parse(DEFAULT_LIST_JSON_STR); + nlohmann::json podJson = nlohmann::json::parse(DEFAULT_POD_JSON_STR); + std::shared_ptr podListWithEmpty = std::make_shared(); + podListWithEmpty->FromJson(listJson); + + std::shared_ptr podListWithOne = std::make_shared(); + podListWithOne->FromJson(listJson); + std::shared_ptr pod1 = std::make_shared(); + pod1->FromJson(podJson); + podListWithOne->SetItems({pod1}); + + std::shared_ptr podListWithTwo = std::make_shared(); + podListWithTwo->FromJson(listJson); + std::shared_ptr pod2 = std::make_shared(); + pod2->FromJson(podJson); + podListWithTwo->SetItems({pod1, pod2}); + // list empty pod + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto result = kubeClient_->ListNamespacedPod(r_namespace); + result.Get(); + EXPECT_EQ(result.Get()->GetItems().size(), static_cast(0)); + // list one pod + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedPod(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), static_cast(1)); + // list two pod + podListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedPod(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), static_cast(2)); + // list pod with two page + podListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedPod(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 3); + // list pods with second page error + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, podListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedPod(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 2); + // list pod with 4 items + responseQueue = {}; + podListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + podListWithTwo->GetMetadata()->SetRContinue(""); + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, podListWithTwo->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedPod(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 4); + EXPECT_EQ(apiServer->GetResponseQueue().size(), 1); +} + +TEST_F(KubeClientApiTest, ListNamespacedDeployment) +{ + std::string r_namespace = "default"; + kubeClient_->SetPageLimit(2); + nlohmann::json listJson = nlohmann::json::parse(DEFAULT_LIST_JSON_STR); + nlohmann::json deployJson = nlohmann::json::parse(DEFAULT_DEPLOYMENT_JSON_STR); + std::shared_ptr deployListWithEmpty = std::make_shared(); + deployListWithEmpty->FromJson(listJson); + + std::shared_ptr deployListWithOne = std::make_shared(); + deployListWithOne->FromJson(listJson); + std::shared_ptr deploy1 = std::make_shared(); + deploy1->FromJson(deployJson); + deployListWithOne->SetItems({deploy1}); + + std::shared_ptr deployListWithTwo = std::make_shared(); + deployListWithTwo->FromJson(listJson); + std::shared_ptr deploy2 = std::make_shared(); + deploy2->FromJson(deployJson); + deployListWithTwo->SetItems({deploy1, deploy2}); + // list empty pod + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto result = kubeClient_->ListNamespacedDeployment(r_namespace); + result.Get(); + EXPECT_EQ(result.Get()->GetItems().size(), 0); + // list one pod + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedDeployment(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 1); + // list two pod + deployListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedDeployment(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 2); + // list pod with two page + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedDeployment(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 3); + // list pods with second page error + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, deployListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedDeployment(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 2); + // list pod with 4 items + responseQueue = {}; + deployListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + deployListWithTwo->GetMetadata()->SetRContinue(""); + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, deployListWithTwo->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNamespacedDeployment(r_namespace); + EXPECT_EQ(result.Get()->GetItems().size(), 4); + EXPECT_EQ(apiServer->GetResponseQueue().size(), 1); +} + +TEST_F(KubeClientApiTest, ListNamespacedHorizontalPodAutoscaler) +{ + auto result = kubeClient_->ListNamespacedHorizontalPodAutoscaler("default", false); + EXPECT_EQ(result.Get()->GetItems().size(), 0); +} + +TEST_F(KubeClientApiTest, CreateNamespacedHorizontalPodAutoscaler) +{ + std::string r_namespace = "default"; + std::shared_ptr body; + auto result = kubeClient_->CreateNamespacedHorizontalPodAutoscaler(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + + body = std::make_shared(); + body->SetApiVersion("autoscaling/v2beta2"); + body->SetKind("HorizontalPodAutoscaler"); + auto metaData = std::make_shared(); + metaData->SetName("function-agent-pool1-hpa"); + metaData->SetRNamespace(r_namespace); + body->SetMetadata(metaData); + result = kubeClient_->CreateNamespacedHorizontalPodAutoscaler(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsOK()); + std::cout << result.Get()->ToJson().dump() << std::endl; +} + +TEST_F(KubeClientApiTest, DeleteNamespacedHorizontalPodAutoscaler) +{ + std::string r_namespace = "default"; + auto result = kubeClient_->DeleteNamespacedHorizontalPodAutoscaler("function-agent-pool1-hpa", r_namespace, litebus::None(), litebus::None()); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, PatchNamespacedHorizontalPodAutoscaler) +{ + std::string r_namespace = "default"; + std::shared_ptr body; + auto result = kubeClient_->PatchNamespacedHorizontalPodAutoscaler("function-agent-pool1-hpa", r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, DeleteNamespacedDeployment) +{ + std::string r_namespace = "default"; + auto result = kubeClient_->DeleteNamespacedDeployment("function-agent-pool1", r_namespace); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pool1"); +} + +TEST_F(KubeClientApiTest, CreateNamespacedDeployment) +{ + std::string r_namespace = "default"; + std::shared_ptr body; + auto result = kubeClient_->CreateNamespacedDeployment(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + body = CreateNamespacedDeploymentBody(); + result = kubeClient_->CreateNamespacedDeployment(r_namespace, body); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-300m-300mi"); +} + +TEST_F(KubeClientApiTest, CreatePodPoolWithOwnerReferrence) +{ + kubeClient_->InitOwnerReference("default", "function-master"); + std::shared_ptr functionMasterDeploy = std::make_shared(); + std::shared_ptr masterMeta = std::make_shared(); + masterMeta->SetName("function-master"); + masterMeta->SetUid("master-000001"); + functionMasterDeploy->SetMetadata(masterMeta); + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto body = CreateNamespacedDeploymentBody(); + auto result = kubeClient_->CreateNamespacedDeployment("default", body); + result.Get(); + EXPECT_TRUE(result.Get()->GetMetadata()->GetOwnerReferences().size() > 0); + EXPECT_EQ(result.Get()->GetMetadata()->GetOwnerReferences()[0]->GetUid(), "master-000001"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + body = CreateNamespacedDeploymentBody(); + result = kubeClient_->CreateNamespacedDeployment("default", body); + result.Get(); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(KubeClientApiTest, PatchNamespacedDeployment) +{ + std::string r_namespace = "default"; + auto body = CreatePatchNamespacedDeploymentBody(); + auto result = kubeClient_->PatchNamespacedDeployment("function-agent-300m-300mi", r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, ListNode) +{ + kubeClient_->SetPageLimit(2); + nlohmann::json listJson = nlohmann::json::parse(DEFAULT_LIST_JSON_STR); + nlohmann::json nodeJson = nlohmann::json::parse(DEFAULT_NODE_JSON_STR); + std::shared_ptr nodeListWithEmpty = std::make_shared(); + nodeListWithEmpty->FromJson(listJson); + + std::shared_ptr nodeListWithOne = std::make_shared(); + nodeListWithOne->FromJson(listJson); + std::shared_ptr node1 = std::make_shared(); + node1->FromJson(nodeJson); + nodeListWithOne->SetItems({node1}); + + std::shared_ptr nodeListWithTwo = std::make_shared(); + nodeListWithTwo->FromJson(listJson); + std::shared_ptr node2 = std::make_shared(); + node2->FromJson(nodeJson); + nodeListWithTwo->SetItems({node1, node2}); + // list empty node + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto result = kubeClient_->ListNode(); + result.Get(); + EXPECT_EQ(result.Get()->GetItems().size(), 0); + // list one node + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNode(); + EXPECT_EQ(result.Get()->GetItems().size(), 1); + // list two node + nodeListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithEmpty->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNode(); + EXPECT_EQ(result.Get()->GetItems().size(), 2); + // list nodes with two page + nodeListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNode(); + EXPECT_EQ(result.Get()->GetItems().size(), 3); + // list nodes with second page error + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, nodeListWithOne->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNode(); + EXPECT_EQ(result.Get()->GetItems().size(), 2); + // list node with 4 items + responseQueue = {}; + nodeListWithTwo->GetMetadata()->SetRContinue("id-123"); + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + nodeListWithTwo->GetMetadata()->SetRContinue(""); + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + responseQueue.push(std::make_pair(ResponseCode::OK, nodeListWithTwo->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ListNode(); + EXPECT_EQ(result.Get()->GetItems().size(), 4); + EXPECT_EQ(apiServer->GetResponseQueue().size(), 1); +} + +TEST_F(KubeClientApiTest, PatchNode) +{ + auto body = CreatePatchNodeBody(); + auto result = kubeClient_->PatchNode("node001", body); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, WatchHandlerManagerTest) +{ + WatchHandlerManager watchHandlerManager; + watchHandlerManager.RegisterWatchHandler("", EventType::EVENT_TYPE_ADD, nullptr); + watchHandlerManager.GetWatchHandlerFunc("", EventType::EVENT_TYPE_ADD); + watchHandlerManager.GetEventTypeByStr(""); + + litebus::http::Response response; + apiClient_->HandleResponse("path", &response); +} + +TEST_F(KubeClientApiTest, HandleResponseTest) +{ + std::string etcdPod = + R"({"type":"ADDED","object":{"apiVersion":"v1","kind":"Pod","metadata":{"name":"etcd-master1"}}})"; + bool flag = false; + bool res = apiClient_->RegisterWatchHandler("V1Pod", EventType::EVENT_TYPE_ADD, + [&flag](const EventType &, const std::shared_ptr &) { + flag = true; + YRLOG_DEBUG("Call Watch Handler"); + }); + EXPECT_TRUE(res); + std::string podBody1 = etcdPod.substr(0, etcdPod.size() - 1); + litebus::http::Response response1(ResponseCode::OK, podBody1); + apiClient_->HandleResponse("path", &response1); + EXPECT_FALSE(flag); + std::string podBody2 = etcdPod.substr(etcdPod.size() - 1) + "\n"; + litebus::http::Response response2(ResponseCode::OK, podBody2); + apiClient_->HandleResponse("path", &response2); + EXPECT_TRUE(flag); +} + +TEST_F(KubeClientApiTest, LeaseTest) +{ + nlohmann::json podJson = nlohmann::json::parse(functionsystem::test::DEFAULT_LEASE_JSON_STR); + std::shared_ptr lease = std::make_shared(); + lease->FromJson(podJson); + + std::string r_namespace = "default"; + std::shared_ptr body; + auto result = kubeClient_->CreateNamespacedLease(r_namespace, body); + result.Get(); + EXPECT_TRUE(result.IsError()); + body = std::make_shared(); + body->SetApiVersion("0"); + body->SetKind("Lease"); + std::shared_ptr metadata = std::make_shared(); + metadata->SetName("function-master-lease"); + body->SetMetadata(metadata); + std::queue> responseQueue; + lease->GetSpec()->SetHolderIdentity("test"); + responseQueue.push(std::make_pair(ResponseCode::OK, lease->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->CreateNamespacedLease(r_namespace, body); + result.Get(); + EXPECT_EQ(result.Get()->GetSpec()->GetHolderIdentity(), "test"); + // read lease + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, lease->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ReadNamespacedLease("function-master-lease", r_namespace); + responseQueue.push(std::make_pair(ResponseCode::OK, lease->ToJson().dump().c_str())); + EXPECT_EQ(result.Get()->GetSpec()->GetHolderIdentity(), "test"); + // replace lease + lease->GetSpec()->SetHolderIdentity("test1"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::OK, lease->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + result = kubeClient_->ReplaceNamespacedLease("function-master-lease", r_namespace, lease); + EXPECT_EQ(result.Get()->GetSpec()->GetHolderIdentity(), "test1"); + +} + +TEST_F(KubeClientApiTest, CreateLeaseWithOwnerReferrence) +{ + kubeClient_->InitOwnerReference("default", "function-master"); + std::shared_ptr functionMasterDeploy = std::make_shared(); + std::shared_ptr masterMeta = std::make_shared(); + masterMeta->SetName("function-master"); + masterMeta->SetUid("master-000001"); + functionMasterDeploy->SetMetadata(masterMeta); + std::queue> responseQueue; + responseQueue.push(std::make_pair(ResponseCode::OK, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + auto body = CreateNamespacedLeaseBody(); + auto result = kubeClient_->CreateNamespacedLease("default", body); + result.Get(); + EXPECT_TRUE(result.Get()->GetMetadata()->GetOwnerReferences().size() > 0); + EXPECT_EQ(result.Get()->GetMetadata()->GetOwnerReferences()[0]->GetUid(), "master-000001"); + responseQueue = {}; + responseQueue.push(std::make_pair(ResponseCode::BAD_REQUEST, functionMasterDeploy->ToJson().dump().c_str())); + apiServer->SetResponseQueue(responseQueue); + body = CreateNamespacedLeaseBody(); + result = kubeClient_->CreateNamespacedLease("default", body); + result.Get(); + EXPECT_TRUE(result.IsError()); +} + +TEST_F(KubeClientApiTest, CreateNamespacedPodRetry) +{ + std::string r_namespace = "default"; + auto body = CreateNamespacedPodBody(); + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::INTERNAL_SERVER_ERROR); + auto result = kubeClient_->CreateNamespacedPod(r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::INTERNAL_SERVER_ERROR); + result = kubeClient_->CreateNamespacedPod(r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, ListNamespacedPodRetry) +{ + std::string r_namespace = "default"; + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::BAD_GATEWAY); + auto result = kubeClient_->DoListNamespacePod(r_namespace, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry 2 success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::BAD_GATEWAY); + result = kubeClient_->DoListNamespacePod(r_namespace, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetKind(), "list"); +} + +TEST_F(KubeClientApiTest, PatchNamespacedPodRetry) +{ + std::string r_namespace = "default"; + auto body = CreatePatchNamespacedPod(); + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::SERVICE_UNAVAILABLE); + auto result = kubeClient_->PatchNamespacedPod("function-agent-pod-0", r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::SERVICE_UNAVAILABLE); + result = kubeClient_->PatchNamespacedPod("function-agent-pod-0", r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, DeleteNamespacedPodRetry) +{ + std::string r_namespace = "default"; + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::GATEWAY_TIMEOUT); + auto result = kubeClient_->DeleteNamespacedPod("function-agent-pod-0", r_namespace); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::GATEWAY_TIMEOUT); + result = kubeClient_->DeleteNamespacedPod("function-agent-pod-0", r_namespace); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pod-0"); +} + +TEST_F(KubeClientApiTest, CreateNamespacedDeploymentRetry) +{ + std::string r_namespace = "default"; + auto body = CreateNamespacedDeploymentBody(); + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::INTERNAL_SERVER_ERROR); + auto result = kubeClient_->CreateNamespacedDeployment(r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::INTERNAL_SERVER_ERROR); + result = kubeClient_->CreateNamespacedDeployment(r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-300m-300mi"); +} + +TEST_F(KubeClientApiTest, ListNamespacedDeploymentRetry) +{ + std::string r_namespace = "default"; + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::NOT_IMPLEMENTED); + auto result = kubeClient_->DoListNamespaceDeployment(r_namespace, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry 2 success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::NOT_IMPLEMENTED); + result = kubeClient_->DoListNamespaceDeployment(r_namespace, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetKind(), "list"); +} + +TEST_F(KubeClientApiTest, PatchNamespacedDeploymentRetry) +{ + std::string r_namespace = "default"; + auto body = CreatePatchNamespacedDeploymentBody(); + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::GATEWAY_TIMEOUT); + auto result = kubeClient_->PatchNamespacedDeployment("function-agent-300m-300mi", r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::GATEWAY_TIMEOUT); + result = kubeClient_->PatchNamespacedDeployment("function-agent-300m-300mi", r_namespace, body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + +TEST_F(KubeClientApiTest, DeleteNamespacedDeploymentRetry) +{ + std::string r_namespace = "default"; + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::INTERNAL_SERVER_ERROR); + auto result = kubeClient_->DeleteNamespacedDeployment("function-agent-pool1", r_namespace); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::INTERNAL_SERVER_ERROR); + result = kubeClient_->DeleteNamespacedDeployment("function-agent-pool1", r_namespace); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetMetadata()->GetName(), "function-agent-pool1"); +} + +TEST_F(KubeClientApiTest, ListNodeRetry) +{ + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::TOO_MANY_REQUESTS); + auto result = kubeClient_->DoListNode({}, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry 2 success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::TOO_MANY_REQUESTS); + result = kubeClient_->DoListNode({}, {}, {}); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_EQ(result.Get()->GetItems().size(), 0); +} + +TEST_F(KubeClientApiTest, PatchNodeRetry) +{ + auto body = CreatePatchNodeBody(); + + // first request fail, retry 2 times fail + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime() + 1, ResponseCode::INTERNAL_SERVER_ERROR); + auto result = kubeClient_->PatchNode("node001", body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsError()); + + // first request fail, retry 1 time fail, retry success + SetRetryResponseQueue(kubeClient_->GetK8sClientRetryTime(), ResponseCode::INTERNAL_SERVER_ERROR); + result = kubeClient_->PatchNode("node001", body); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + result.Get(); + EXPECT_TRUE(result.IsOK()); +} + + +TEST_F(KubeClientApiTest, HealthMonitorTest) +{ + auto healthMonitor = kubeClient_->GetHealthMonitor(); + + auto cb = [=](bool healthy) { + YRLOG_INFO("transfer health status: {} to {}", !healthy, healthy); + ASSERT_EQ(healthMonitor->IsHealthy(), healthy); + }; + healthMonitor->SubscribeK8sHealth(cb); + + // 模拟å¥åº·ç›‘测2次åŽï¼Œè¿”回æˆåŠŸï¼Œæå‰é¢„置好å“应失败结果 + std::queue> responseQueue; + responseQueue.emplace(ResponseCode::INTERNAL_SERVER_ERROR, ""); + responseQueue.emplace(ResponseCode::INTERNAL_SERVER_ERROR, ""); + apiServer->SetResponseQueue(responseQueue); + // 模拟x次请求失败，在第6次失败时开始å¥åº·ç›‘测 + for (int i = 0; i < 7; i++) { + litebus::Async(healthMonitor->GetAID(), &HealthMonitor::Start); + } + // ç­‰å¾…ç›‘æµ‹å®Œæˆ + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + ASSERT_AWAIT_TRUE([&]() -> bool { return healthMonitor->IsHealthy(); }); + ASSERT_AWAIT_TRUE([&]() -> bool { + return healthMonitor->GetFailedTime() == static_cast(0); + }); + + // 模拟x次请求失败，在第6次失败时开始å¥åº·ç›‘测 + for (int i = 0; i < 7; i++) { + litebus::Async(healthMonitor->GetAID(), &HealthMonitor::Start); + } + // æž„é€ è¶…æ—¶è¯·æ±‚ï¼Œç­‰å¾…ç›‘æµ‹å®Œæˆ + ASSERT_AWAIT_TRUE([&]() -> bool { return healthMonitor->IsHealthy(); }); + ASSERT_EQ(healthMonitor->GetFailedTime(), static_cast(0)); + + litebus::Terminate(healthMonitor->GetAID()); + litebus::Await(healthMonitor->GetAID()); +} + +TEST_F(KubeClientApiTest, InvalidKubeConfig) +{ + SslConfig sslConfig{ + .clientCertFile = "", .clientKeyFile = "", .caCertFile = "", .credential = "k8sClient", .isSkipTlsVerify = false + }; + HealthMonitorParam param{ + .maxFailedTimes = 5, + .checkIntervalMs = 20, + }; + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + auto kubeClient = + KubeClient::CreateKubeClient("http://127.0.0.1:" + std::to_string(port) + "/k8s", sslConfig, false, param); + KubeClient::ClusterSslConfig("fake", "fake", true); + kubeClient->SetIsK8sHealthy(false); + + CheckFutureError(kubeClient->ListNamespacedHorizontalPodAutoscaler("default", false), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + std::string r_namespace = "default"; + std::shared_ptr body; + CheckFutureError(kubeClient->CreateNamespacedHorizontalPodAutoscaler(r_namespace, body), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + std::shared_ptr objBody; + CheckFutureError( + kubeClient->PatchNamespacedHorizontalPodAutoscaler("function-agent-pool1-hpa", r_namespace, objBody), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DeleteNamespacedHorizontalPodAutoscaler("function-agent-pool1-hpa", r_namespace, + litebus::None(), litebus::None()), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DoListNamespaceDeployment(r_namespace, {}, {}), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + std::shared_ptr deploymentBody; + CheckFutureError(kubeClient->CreateNamespacedDeployment(r_namespace, deploymentBody), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DeleteNamespacedDeployment("function-agent-pool1", r_namespace), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->PatchNamespacedDeployment("function-agent-300m-300mi", r_namespace, + CreatePatchNamespacedDeploymentBody()), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->ReadNamespacedDeployment("function-agent-300m-300mi", r_namespace), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DoListNamespacePod(r_namespace, {}, {}), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->PatchNamespacedPod("function-agent-pod-0", r_namespace, CreatePatchNamespacedPod()), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DeleteNamespacedPod("function-agent-pod-0", r_namespace), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->ReadNamespacedPod(r_namespace, "pod-001"), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->DoListNode({}, {}, {}), static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + std::shared_ptr leaseBody; + CheckFutureError(kubeClient->CreateNamespacedLease(r_namespace, leaseBody), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->ReadNamespacedLease("function-master-lease", r_namespace), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + std::shared_ptr lease = std::make_shared(); + CheckFutureError(kubeClient->ReplaceNamespacedLease("function-master-lease", r_namespace, lease), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); + + CheckFutureError(kubeClient->PatchNode("node001", CreatePatchNodeBody()), + static_cast(StatusCode::ERR_K8S_UNAVAILABLE)); +} +} // namespace functionsystem::kube_client::test diff --git a/functionsystem/tests/unit/common/kube_client/kube_client_model_test.cpp b/functionsystem/tests/unit/common/kube_client/kube_client_model_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..231dcb97c5289a2c62a0957cd159d3c30696f7da --- /dev/null +++ b/functionsystem/tests/unit/common/kube_client/kube_client_model_test.cpp @@ -0,0 +1,1629 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/kube_client/model/common/v1_capabilities.h" +#include "common/kube_client/model/common/v1_delete_options.h" +#include "common/kube_client/model/common/v1_env_var.h" +#include "common/kube_client/model/common/v1_env_var_source.h" +#include "common/kube_client/model/common/v1_exec_action.h" +#include "common/kube_client/model/common/v1_host_alias.h" +#include "common/kube_client/model/common/v1_key_to_path.h" +#include "common/kube_client/model/common/v1_label_selector.h" +#include "common/kube_client/model/common/v1_label_selector_requirement.h" +#include "common/kube_client/model/common/v1_lifecycle.h" +#include "common/kube_client/model/common/v1_lifecycle_handler.h" +#include "common/kube_client/model/common/v1_list_meta.h" +#include "common/kube_client/model/common/v1_local_object_reference.h" +#include "common/kube_client/model/common/v1_object_field_selector.h" +#include "common/kube_client/model/common/v1_object_meta.h" +#include "common/kube_client/model/common/v1_owner_reference.h" +#include "common/kube_client/model/common/v1_resource_field_selector.h" +#include "common/kube_client/model/common/v1_resource_requirements.h" +#include "common/kube_client/model/common/v1_security_context.h" +#include "common/kube_client/model/container/v1_container.h" +#include "common/kube_client/model/container/v1_container_port.h" +#include "common/kube_client/model/container/v1_container_state.h" +#include "common/kube_client/model/container/v1_container_status.h" +#include "common/kube_client/model/container/v1_probe.h" +#include "common/kube_client/model/container/v1_tcp_socket_action.h" +#include "common/kube_client/model/deployment/v1_deployment.h" +#include "common/kube_client/model/deployment/v1_deployment_list.h" +#include "common/kube_client/model/deployment/v1_deployment_spec.h" +#include "common/kube_client/model/lease/v1_lease.h" +#include "common/kube_client/model/lease/v1_lease_spec.h" +#include "common/kube_client/model/node/v1_node.h" +#include "common/kube_client/model/node/v1_node_address.h" +#include "common/kube_client/model/node/v1_node_list.h" +#include "common/kube_client/model/node/v1_node_spec.h" +#include "common/kube_client/model/node/v1_node_status.h" +#include "common/kube_client/model/node/v1_taint.h" +#include "common/kube_client/model/pod/v1_affinity.h" +#include "common/kube_client/model/pod/v1_pod.h" +#include "common/kube_client/model/pod/v1_pod_affinity.h" +#include "common/kube_client/model/pod/v1_pod_affinity_term.h" +#include "common/kube_client/model/pod/v1_pod_anti_affinity.h" +#include "common/kube_client/model/pod/v1_pod_list.h" +#include "common/kube_client/model/pod/v1_pod_security_context.h" +#include "common/kube_client/model/pod/v1_pod_spec.h" +#include "common/kube_client/model/pod/v1_pod_status.h" +#include "common/kube_client/model/pod/v1_pod_template_spec.h" +#include "common/kube_client/model/pod/v1_weighted_pod_affinity_term.h" +#include "common/kube_client/model/volume/v1_host_path_volume_source.h" +#include "common/kube_client/model/volume/v1_empty_dir_volume_source.h" +#include "common/kube_client/model/volume/v1_secret_volume_source.h" +#include "common/kube_client/model/volume/v1_volume.h" +#include "common/kube_client/model/volume/v1_volume_mount.h" + +namespace functionsystem::kube_client::test { + +using namespace functionsystem::kube_client::model; + +class KubeClientModelTest : public ::testing::Test { +public: + void SetUp() override + { + } + + void TearDown() override + { + } +}; + +TEST_F(KubeClientModelTest, V1CapabilitiesTest) +{ + std::shared_ptr V1Capabilities_ = std::make_shared(); + std::vector Add; + V1Capabilities_->SetAdd(Add); + std::vector Drop; + V1Capabilities_->SetDrop(Drop); + auto json = V1Capabilities_->ToJson(); + auto result = V1Capabilities_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Capabilities_->AddIsSet()); + EXPECT_TRUE(V1Capabilities_->DropIsSet()); +} + +TEST_F(KubeClientModelTest, V1AffinityTest) +{ + std::shared_ptr V1Affinity_ = std::make_shared(); + V1Affinity_->UnSetPodAffinity(); + V1Affinity_->UnSetPodAntiAffinity(); + std::shared_ptr PodAffinity = std::make_shared(); + V1Affinity_->SetPodAffinity(PodAffinity); + std::shared_ptr PodAntiAffinity = std::make_shared(); + V1Affinity_->SetPodAntiAffinity(PodAntiAffinity); + std::shared_ptr nodeAffinity = std::make_shared(); + std::shared_ptr nodeSelector = std::make_shared(); + std::shared_ptr nodeSelectorTerm = std::make_shared(); + std::shared_ptr nodeSelectorRequirement = std::make_shared(); + nodeSelectorRequirement->SetKey("key1"); + nodeSelectorRequirement->SetROperator("In"); + nodeSelectorRequirement->SetValues({"val1"}); + std::vector> nodeSelectorRequirements{nodeSelectorRequirement}; + nodeSelectorTerm->SetMatchExpressions(nodeSelectorRequirements); + std::vector> nodeSelectorTerms{nodeSelectorTerm}; + nodeSelector->SetNodeSelectorTerms(nodeSelectorTerms); + nodeAffinity->SetRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector); + V1Affinity_->SetNodeAffinity(nodeAffinity); + auto json = V1Affinity_->ToJson(); + auto result = V1Affinity_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Affinity_->PodAffinityIsSet()); + EXPECT_TRUE(V1Affinity_->PodAntiAffinityIsSet()); + + std::shared_ptr preferredSchedulingTerm = std::make_shared(); + preferredSchedulingTerm->SetPreference(nodeSelectorTerm); + preferredSchedulingTerm->SetWeight(1); + nodeAffinity->SetPreferredDuringSchedulingIgnoredDuringExecution({preferredSchedulingTerm}); + V1Affinity_->SetNodeAffinity(nodeAffinity); + json = V1Affinity_->ToJson(); + result = V1Affinity_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Affinity_->NodeAffinityIsSet()); + EXPECT_TRUE(V1Affinity_->GetNodeAffinity()->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_EQ(static_cast(1), + V1Affinity_->GetNodeAffinity()->GetPreferredDuringSchedulingIgnoredDuringExecution().size()); + EXPECT_EQ(static_cast(1), + V1Affinity_->GetNodeAffinity()->GetPreferredDuringSchedulingIgnoredDuringExecution().front()->GetWeight()); + EXPECT_TRUE(V1Affinity_->GetNodeAffinity()->GetPreferredDuringSchedulingIgnoredDuringExecution().front()->GetPreference()->MatchExpressionsIsSet()); + EXPECT_EQ(static_cast(1), V1Affinity_->GetNodeAffinity()->GetPreferredDuringSchedulingIgnoredDuringExecution().front()->GetPreference()->GetMatchExpressions().size()); + EXPECT_EQ("key1", V1Affinity_->GetNodeAffinity()->GetPreferredDuringSchedulingIgnoredDuringExecution().front()->GetPreference()->GetMatchExpressions().front()->GetKey()); +} + +TEST_F(KubeClientModelTest, V1ConfigMapVolumeSourceTest) +{ + std::shared_ptr V1ConfigMapVolumeSource_ = std::make_shared(); + V1ConfigMapVolumeSource_->UnsetOptional(); + V1ConfigMapVolumeSource_->UnsetDefaultMode(); + V1ConfigMapVolumeSource_->UnsetName(); + V1ConfigMapVolumeSource_->UnsetItems(); + V1ConfigMapVolumeSource_->SetDefaultMode(0); + std::vector> Items; + V1ConfigMapVolumeSource_->SetItems(Items); + V1ConfigMapVolumeSource_->SetName("0"); + V1ConfigMapVolumeSource_->SetOptional(0); + auto json = V1ConfigMapVolumeSource_->ToJson(); + auto result = V1ConfigMapVolumeSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ConfigMapVolumeSource_->DefaultModeIsSet()); + EXPECT_EQ(0, V1ConfigMapVolumeSource_->GetDefaultMode()); + EXPECT_TRUE(V1ConfigMapVolumeSource_->ItemsIsSet()); + EXPECT_TRUE(V1ConfigMapVolumeSource_->NameIsSet()); + EXPECT_EQ("0", V1ConfigMapVolumeSource_->GetName()); + EXPECT_TRUE(V1ConfigMapVolumeSource_->OptionalIsSet()); + EXPECT_EQ(0, V1ConfigMapVolumeSource_->IsOptional()); +} + +TEST_F(KubeClientModelTest, V1ContainerPortTest) +{ + std::shared_ptr V1ContainerPort_ = std::make_shared(); + V1ContainerPort_->UnsetName(); + V1ContainerPort_->UnsetProtocol(); + V1ContainerPort_->UnsetHostPort(); + V1ContainerPort_->UnsetContainerPort(); + V1ContainerPort_->UnsetHostIP(); + V1ContainerPort_->SetContainerPort(0); + V1ContainerPort_->SetHostIP("0"); + V1ContainerPort_->SetHostPort(0); + V1ContainerPort_->SetName("0"); + V1ContainerPort_->SetProtocol("0"); + auto json = V1ContainerPort_->ToJson(); + auto result = V1ContainerPort_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ContainerPort_->ContainerPortIsSet()); + EXPECT_EQ(0, V1ContainerPort_->GetContainerPort()); + EXPECT_TRUE(V1ContainerPort_->HostIPIsSet()); + EXPECT_EQ("0", V1ContainerPort_->GetHostIP()); + EXPECT_TRUE(V1ContainerPort_->HostPortIsSet()); + EXPECT_EQ(0, V1ContainerPort_->GetHostPort()); + EXPECT_TRUE(V1ContainerPort_->NameIsSet()); + EXPECT_EQ("0", V1ContainerPort_->GetName()); + EXPECT_TRUE(V1ContainerPort_->ProtocolIsSet()); + EXPECT_EQ("0", V1ContainerPort_->GetProtocol()); +} + +TEST_F(KubeClientModelTest, V1ContainerStatusTest) +{ + std::shared_ptr V1ContainerStatus_ = std::make_shared(); + V1ContainerStatus_->UnsetState(); + V1ContainerStatus_->UnsetStarted(); + V1ContainerStatus_->UnsetRestartCount(); + V1ContainerStatus_->UnsetReady(); + V1ContainerStatus_->UnsetName(); + V1ContainerStatus_->UnsetLastState(); + V1ContainerStatus_->UnsetImageID(); + V1ContainerStatus_->UnsetImage(); + V1ContainerStatus_->UnsetContainerID(); + V1ContainerStatus_->SetContainerID("0"); + V1ContainerStatus_->SetImage("0"); + V1ContainerStatus_->SetImageID("0"); + std::shared_ptr LastState = std::make_shared(); + V1ContainerStatus_->SetLastState(LastState); + V1ContainerStatus_->SetName("0"); + V1ContainerStatus_->SetReady(0); + V1ContainerStatus_->SetRestartCount(0); + V1ContainerStatus_->SetStarted(0); + std::shared_ptr State = std::make_shared();; + V1ContainerStatus_->SetState(State); + auto json = V1ContainerStatus_->ToJson(); + auto result = V1ContainerStatus_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ContainerStatus_->ContainerIDIsSet()); + EXPECT_EQ("0", V1ContainerStatus_->GetContainerID()); + EXPECT_TRUE(V1ContainerStatus_->ImageIsSet()); + EXPECT_EQ("0", V1ContainerStatus_->GetImage()); + EXPECT_TRUE(V1ContainerStatus_->ImageIDIsSet()); + EXPECT_EQ("0", V1ContainerStatus_->GetImageID()); + EXPECT_TRUE(V1ContainerStatus_->LastStateIsSet()); + EXPECT_TRUE(V1ContainerStatus_->NameIsSet()); + EXPECT_EQ("0", V1ContainerStatus_->GetName()); + EXPECT_TRUE(V1ContainerStatus_->ReadyIsSet()); + EXPECT_EQ(0, V1ContainerStatus_->IsReady()); + EXPECT_TRUE(V1ContainerStatus_->RestartCountIsSet()); + EXPECT_EQ(0, V1ContainerStatus_->GetRestartCount()); + EXPECT_TRUE(V1ContainerStatus_->StartedIsSet()); + EXPECT_EQ(0, V1ContainerStatus_->IsStarted()); + EXPECT_TRUE(V1ContainerStatus_->StateIsSet()); +} + +TEST_F(KubeClientModelTest, V1ContainerTest) +{ + std::shared_ptr V1Container_ = std::make_shared(); + V1Container_->UnsetWorkingDir(); + V1Container_->UnsetVolumeMounts(); + V1Container_->UnSetSecurityContext(); + V1Container_->UnsetResources(); + V1Container_->UnsetReadinessProbe(); + V1Container_->UnsetPorts(); + V1Container_->UnsetName(); + V1Container_->UnsetLivenessProbe(); + V1Container_->UnsetLifecycle(); + V1Container_->UnsetImage(); + V1Container_->UnsetEnv(); + V1Container_->UnsetEnvFrom(); + V1Container_->UnsetCommand(); + V1Container_->UnsetArgs(); + V1Container_->UnsetTerminationMessagePath(); + V1Container_->UnsetTerminationMessagePolicy(); + std::vector Args; + V1Container_->SetArgs(Args); + std::vector Command; + V1Container_->SetCommand(Command); + std::vector> Env; + V1Container_->SetEnv(Env); + std::vector> EnvFrom; + V1Container_->SetEnvFrom(EnvFrom); + V1Container_->SetImage("0"); + std::shared_ptr Lifecycle = std::make_shared(); + V1Container_->SetLifecycle(Lifecycle); + std::shared_ptr LivenessProbe; + V1Container_->SetLivenessProbe(LivenessProbe); + V1Container_->SetName("0"); + std::vector> Ports; + V1Container_->SetPorts(Ports); + std::shared_ptr ReadinessProbe; + V1Container_->SetReadinessProbe(ReadinessProbe); + std::shared_ptr Resources; + V1Container_->SetResources(Resources); + std::shared_ptr SecurityContext; + V1Container_->SetSecurityContext(SecurityContext); + std::shared_ptr StartupProbe; + std::vector> VolumeMounts; + V1Container_->SetVolumeMounts(VolumeMounts); + V1Container_->SetWorkingDir("0"); + V1Container_->SetTerminationMessagePath("/var/tmp/log"); + V1Container_->SetTerminationMessagePolicy("FILE"); + auto json = V1Container_->ToJson(); + auto result = V1Container_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Container_->ArgsIsSet()); + EXPECT_TRUE(V1Container_->CommandIsSet()); + EXPECT_TRUE(V1Container_->EnvIsSet()); + EXPECT_TRUE(V1Container_->EnvFromIsSet()); + EXPECT_TRUE(V1Container_->ImageIsSet()); + EXPECT_EQ("0", V1Container_->GetImage()); + EXPECT_TRUE(V1Container_->LifecycleIsSet()); + EXPECT_TRUE(V1Container_->LivenessProbeIsSet()); + EXPECT_TRUE(V1Container_->NameIsSet()); + EXPECT_EQ("0", V1Container_->GetName()); + EXPECT_TRUE(V1Container_->PortsIsSet()); + EXPECT_TRUE(V1Container_->ReadinessProbeIsSet()); + EXPECT_TRUE(V1Container_->ResourcesIsSet()); + EXPECT_TRUE(V1Container_->SecurityContextIsSet()); + EXPECT_TRUE(V1Container_->VolumeMountsIsSet()); + EXPECT_TRUE(V1Container_->WorkingDirIsSet()); + EXPECT_EQ("0", V1Container_->GetWorkingDir()); +} + +TEST_F(KubeClientModelTest, V1DeleteOptionsTest) +{ + std::shared_ptr V1DeleteOptions_ = std::make_shared(); + V1DeleteOptions_->UnsetPropagationPolicy(); + V1DeleteOptions_->UnsetOrphanDependents(); + V1DeleteOptions_->UnsetKind(); + V1DeleteOptions_->UnsetGracePeriodSeconds(); + V1DeleteOptions_->UnsetDryRun(); + V1DeleteOptions_->UnsetApiVersion(); + V1DeleteOptions_->SetApiVersion("0"); + std::vector DryRun; + V1DeleteOptions_->SetDryRun(DryRun); + V1DeleteOptions_->SetGracePeriodSeconds(0); + V1DeleteOptions_->SetKind("0"); + V1DeleteOptions_->SetOrphanDependents(0); + V1DeleteOptions_->SetPropagationPolicy("0"); + auto json = V1DeleteOptions_->ToJson(); + auto result = V1DeleteOptions_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1DeleteOptions_->ApiVersionIsSet()); + EXPECT_EQ("0", V1DeleteOptions_->GetApiVersion()); + EXPECT_TRUE(V1DeleteOptions_->DryRunIsSet()); + EXPECT_TRUE(V1DeleteOptions_->GracePeriodSecondsIsSet()); + EXPECT_EQ(0, V1DeleteOptions_->GetGracePeriodSeconds()); + EXPECT_TRUE(V1DeleteOptions_->KindIsSet()); + EXPECT_EQ("0", V1DeleteOptions_->GetKind()); + EXPECT_TRUE(V1DeleteOptions_->OrphanDependentsIsSet()); + EXPECT_EQ(0, V1DeleteOptions_->IsOrphanDependents()); + EXPECT_TRUE(V1DeleteOptions_->PropagationPolicyIsSet()); + EXPECT_EQ("0", V1DeleteOptions_->GetPropagationPolicy()); +} + +TEST_F(KubeClientModelTest, V1DeploymentListTest) +{ + std::shared_ptr V1DeploymentList_ = std::make_shared(); + V1DeploymentList_->UnsetApiVersion(); + V1DeploymentList_->UnsetItems(); + V1DeploymentList_->UnsetKind(); + V1DeploymentList_->UnsetMetadata(); + V1DeploymentList_->SetApiVersion("0"); + std::vector> Items; + V1DeploymentList_->SetItems(Items); + V1DeploymentList_->SetKind("0"); + std::shared_ptr Metadata = std::make_shared(); + V1DeploymentList_->SetMetadata(Metadata); + auto json = V1DeploymentList_->ToJson(); + auto result = V1DeploymentList_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1DeploymentList_->ApiVersionIsSet()); + EXPECT_EQ("0", V1DeploymentList_->GetApiVersion()); + EXPECT_TRUE(V1DeploymentList_->ItemsIsSet()); + EXPECT_TRUE(V1DeploymentList_->KindIsSet()); + EXPECT_EQ("0", V1DeploymentList_->GetKind()); + EXPECT_TRUE(V1DeploymentList_->MetadataIsSet()); +} + +TEST_F(KubeClientModelTest, V1DeploymentSpecTest) +{ + std::shared_ptr V1DeploymentSpec_ = std::make_shared(); + V1DeploymentSpec_->UnsetMinReadySeconds(); + V1DeploymentSpec_->UnsetPaused(); + V1DeploymentSpec_->UnsetReplicas(); + V1DeploymentSpec_->UnsetProgressDeadlineSeconds(); + V1DeploymentSpec_->UnSetSelector(); + V1DeploymentSpec_->UnsetRevisionHistoryLimit(); + V1DeploymentSpec_->UnsetMinReadySeconds(); + V1DeploymentSpec_->UnsetRTemplate(); + V1DeploymentSpec_->SetMinReadySeconds(0); + V1DeploymentSpec_->SetPaused(0); + V1DeploymentSpec_->SetProgressDeadlineSeconds(0); + V1DeploymentSpec_->SetReplicas(0); + V1DeploymentSpec_->SetRevisionHistoryLimit(0); + std::shared_ptr Selector; + V1DeploymentSpec_->SetSelector(Selector); + std::shared_ptr r_template; + V1DeploymentSpec_->SetRTemplate(r_template); + auto json = V1DeploymentSpec_->ToJson(); + auto result = V1DeploymentSpec_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1DeploymentSpec_->MinReadySecondsIsSet()); + EXPECT_EQ(0, V1DeploymentSpec_->GetMinReadySeconds()); + EXPECT_TRUE(V1DeploymentSpec_->PausedIsSet()); + EXPECT_EQ(0, V1DeploymentSpec_->IsPaused()); + EXPECT_TRUE(V1DeploymentSpec_->ProgressDeadlineSecondsIsSet()); + EXPECT_EQ(0, V1DeploymentSpec_->GetProgressDeadlineSeconds()); + EXPECT_TRUE(V1DeploymentSpec_->ReplicasIsSet()); + EXPECT_EQ(0, V1DeploymentSpec_->GetReplicas()); + EXPECT_TRUE(V1DeploymentSpec_->RevisionHistoryLimitIsSet()); + EXPECT_EQ(0, V1DeploymentSpec_->GetRevisionHistoryLimit()); + EXPECT_TRUE(V1DeploymentSpec_->SelectorIsSet()); + EXPECT_TRUE(V1DeploymentSpec_->RTemplateIsSet()); +} + +TEST_F(KubeClientModelTest, V1DeploymentTest) +{ + std::shared_ptr V1Deployment_ = std::make_shared(); + V1Deployment_->UnsetMetadata(); + V1Deployment_->UnsetKind(); + V1Deployment_->UnsetApiVersion(); + V1Deployment_->UnsetSpec(); + V1Deployment_->SetApiVersion("0"); + V1Deployment_->SetKind("0"); + std::shared_ptr Metadata; + V1Deployment_->SetMetadata(Metadata); + std::shared_ptr Spec; + V1Deployment_->SetSpec(Spec); + std::shared_ptr deploymentStatus = std::make_shared(); + deploymentStatus->SetAvailableReplicas(1); + deploymentStatus->SetReplicas(2); + V1Deployment_->SetStatus(deploymentStatus); + auto json = V1Deployment_->ToJson(); + auto result = V1Deployment_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Deployment_->ApiVersionIsSet()); + EXPECT_EQ("0", V1Deployment_->GetApiVersion()); + EXPECT_TRUE(V1Deployment_->KindIsSet()); + EXPECT_EQ("0", V1Deployment_->GetKind()); + EXPECT_TRUE(V1Deployment_->MetadataIsSet()); + EXPECT_TRUE(V1Deployment_->SpecIsSet()); + EXPECT_TRUE(V1Deployment_->SpecIsSet()); + EXPECT_EQ(1, V1Deployment_->GetStatus()->GetAvailableReplicas()); + EXPECT_EQ(2, V1Deployment_->GetStatus()->GetReplicas()); +} + +TEST_F(KubeClientModelTest, V1EmptyDirVolumeSourceTest) +{ + std::shared_ptr V1EmptyDirVolumeSource_ = std::make_shared(); + V1EmptyDirVolumeSource_->SetMedium("0"); + V1EmptyDirVolumeSource_->SetSizeLimit("0"); + auto json = V1EmptyDirVolumeSource_->ToJson(); + auto result = V1EmptyDirVolumeSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1EmptyDirVolumeSource_->MediumIsSet()); + EXPECT_EQ("0", V1EmptyDirVolumeSource_->GetMedium()); + EXPECT_TRUE(V1EmptyDirVolumeSource_->SizeLimitIsSet()); + EXPECT_EQ("0", V1EmptyDirVolumeSource_->GetSizeLimit()); +} + +TEST_F(KubeClientModelTest, V1EnvVarTest) +{ + std::shared_ptr V1EnvVar_ = std::make_shared(); + V1EnvVar_->SetName("0"); + V1EnvVar_->SetValue("0"); + std::shared_ptr ValueFrom; + V1EnvVar_->SetValueFrom(ValueFrom); + auto json = V1EnvVar_->ToJson(); + auto result = V1EnvVar_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1EnvVar_->NameIsSet()); + EXPECT_EQ("0", V1EnvVar_->GetName()); + EXPECT_TRUE(V1EnvVar_->ValueIsSet()); + EXPECT_EQ("0", V1EnvVar_->GetValue()); + EXPECT_TRUE(V1EnvVar_->ValueFromIsSet()); +} + +TEST_F(KubeClientModelTest, V1EnvVarSourceTest) +{ + std::shared_ptr V1EnvVarSource_ = std::make_shared(); + std::shared_ptr FieldRef; + V1EnvVarSource_->SetFieldRef(FieldRef); + std::shared_ptr ResourceFieldRef; + V1EnvVarSource_->SetResourceFieldRef(ResourceFieldRef); + std::shared_ptr SecretKeyRef; + V1EnvVarSource_->SetSecretKeyRef(SecretKeyRef); + auto json = V1EnvVarSource_->ToJson(); + auto result = V1EnvVarSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1EnvVarSource_->FieldRefIsSet()); + EXPECT_TRUE(V1EnvVarSource_->ResourceFieldRefIsSet()); + EXPECT_TRUE(V1EnvVarSource_->SecretKeyRefIsSet()); +} + +TEST_F(KubeClientModelTest, V1EnvFromSourceTest) +{ + std::shared_ptr V1EnvFromSource_ = std::make_shared(); + V1EnvFromSource_->UnsetConfigMapRef(); + V1EnvFromSource_->UnsetPrefix(); + V1EnvFromSource_->UnsetSecretRef(); + std::shared_ptr configMapEnvRef; + V1EnvFromSource_->SetConfigMapRef(configMapEnvRef); + std::shared_ptr secretEnvRef; + V1EnvFromSource_->SetSecretRef(secretEnvRef); + V1EnvFromSource_->SetPrefix("prefix"); + auto json = V1EnvFromSource_->ToJson(); + auto result = V1EnvFromSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1EnvFromSource_->SecretRefIsSet()); + EXPECT_TRUE(V1EnvFromSource_->ConfigMapRefIsSet()); + EXPECT_TRUE(V1EnvFromSource_->PrefixIsSet()); +} + +TEST_F(KubeClientModelTest, V1ConfigMapEnvSourceTest) +{ + std::shared_ptr V1ConfigMapEnvSource_ = std::make_shared(); + V1ConfigMapEnvSource_->UnsetName(); + V1ConfigMapEnvSource_->UnsetOptional(); + V1ConfigMapEnvSource_->SetName("name"); + V1ConfigMapEnvSource_->SetOptional(true); + auto json = V1ConfigMapEnvSource_->ToJson(); + auto result = V1ConfigMapEnvSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ConfigMapEnvSource_->NameIsSet()); + EXPECT_TRUE(V1ConfigMapEnvSource_->OptionalIsSet()); +} + +TEST_F(KubeClientModelTest, V1SecretEnvSourceTest) +{ + std::shared_ptr V1SecretEnvSource_ = std::make_shared(); + V1SecretEnvSource_->UnsetName(); + V1SecretEnvSource_->UnsetOptional(); + V1SecretEnvSource_->SetName("name"); + V1SecretEnvSource_->SetOptional(true); + auto json = V1SecretEnvSource_->ToJson(); + auto result = V1SecretEnvSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1SecretEnvSource_->NameIsSet()); + EXPECT_TRUE(V1SecretEnvSource_->OptionalIsSet()); +} + +TEST_F(KubeClientModelTest, V1ExecActionTest) +{ + std::shared_ptr V1ExecAction_ = std::make_shared(); + std::vector Command; + V1ExecAction_->SetCommand(Command); + auto json = V1ExecAction_->ToJson(); + auto result = V1ExecAction_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ExecAction_->CommandIsSet()); +} + +TEST_F(KubeClientModelTest, V1HostAliasTest) +{ + std::shared_ptr V1HostAlias_ = std::make_shared(); + std::vector Hostnames; + V1HostAlias_->SetHostnames(Hostnames); + V1HostAlias_->SetIp("0"); + auto json = V1HostAlias_->ToJson(); + auto result = V1HostAlias_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1HostAlias_->HostnamesIsSet()); + EXPECT_TRUE(V1HostAlias_->IpIsSet()); + EXPECT_EQ("0", V1HostAlias_->GetIp()); +} + +TEST_F(KubeClientModelTest, V1HostPathVolumeSourceTest) +{ + std::shared_ptr V1HostPathVolumeSource_ = std::make_shared(); + V1HostPathVolumeSource_->SetPath("0"); + V1HostPathVolumeSource_->SetType("0"); + auto json = V1HostPathVolumeSource_->ToJson(); + auto result = V1HostPathVolumeSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1HostPathVolumeSource_->PathIsSet()); + EXPECT_EQ("0", V1HostPathVolumeSource_->GetPath()); + EXPECT_TRUE(V1HostPathVolumeSource_->TypeIsSet()); + EXPECT_EQ("0", V1HostPathVolumeSource_->GetType()); +} + +TEST_F(KubeClientModelTest, V1KeyToPathTest) +{ + std::shared_ptr V1KeyToPath_ = std::make_shared(); + V1KeyToPath_->SetKey("0"); + V1KeyToPath_->SetMode(0); + V1KeyToPath_->SetPath("0"); + auto json = V1KeyToPath_->ToJson(); + auto result = V1KeyToPath_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1KeyToPath_->KeyIsSet()); + EXPECT_EQ("0", V1KeyToPath_->GetKey()); + EXPECT_TRUE(V1KeyToPath_->ModeIsSet()); + EXPECT_EQ(0, V1KeyToPath_->GetMode()); + EXPECT_TRUE(V1KeyToPath_->PathIsSet()); + EXPECT_EQ("0", V1KeyToPath_->GetPath()); +} + +TEST_F(KubeClientModelTest, V1LabelSelectorRequirementTest) +{ + std::shared_ptr V1LabelSelectorRequirement_ = + std::make_shared(); + V1LabelSelectorRequirement_->SetKey("0"); + V1LabelSelectorRequirement_->SetROperator("0"); + std::vector Values; + V1LabelSelectorRequirement_->SetValues(Values); + auto json = V1LabelSelectorRequirement_->ToJson(); + auto result = V1LabelSelectorRequirement_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1LabelSelectorRequirement_->KeyIsSet()); + EXPECT_EQ("0", V1LabelSelectorRequirement_->GetKey()); + EXPECT_TRUE(V1LabelSelectorRequirement_->ROperatorIsSet()); + EXPECT_EQ("0", V1LabelSelectorRequirement_->GetROperator()); + EXPECT_TRUE(V1LabelSelectorRequirement_->ValuesIsSet()); +} + +TEST_F(KubeClientModelTest, V1LabelSelectorTest) +{ + std::shared_ptr V1LabelSelector_ = std::make_shared(); + std::vector> MatchExpressions; + V1LabelSelector_->SetMatchExpressions(MatchExpressions); + std::map MatchLabels; + V1LabelSelector_->SetMatchLabels(MatchLabels); + auto json = V1LabelSelector_->ToJson(); + auto result = V1LabelSelector_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1LabelSelector_->MatchExpressionsIsSet()); + EXPECT_TRUE(V1LabelSelector_->MatchLabelsIsSet()); +} + +TEST_F(KubeClientModelTest, V1LifecycleHandlerTest) +{ + std::shared_ptr V1LifecycleHandler_ = std::make_shared(); + std::shared_ptr Exec; + V1LifecycleHandler_->SetExec(Exec); + std::shared_ptr TcpSocket; + auto json = V1LifecycleHandler_->ToJson(); + auto result = V1LifecycleHandler_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1LifecycleHandler_->ExecIsSet()); +} + +TEST_F(KubeClientModelTest, V1LifecycleTest) +{ + std::shared_ptr V1Lifecycle_ = std::make_shared(); + std::shared_ptr PostStart; + V1Lifecycle_->SetPostStart(PostStart); + std::shared_ptr PreStop; + V1Lifecycle_->SetPreStop(PreStop); + auto json = V1Lifecycle_->ToJson(); + auto result = V1Lifecycle_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Lifecycle_->PostStartIsSet()); + EXPECT_TRUE(V1Lifecycle_->PreStopIsSet()); +} + +TEST_F(KubeClientModelTest, V1ListMetaTest) +{ + std::shared_ptr V1ListMeta_ = std::make_shared(); + V1ListMeta_->SetRContinue("0"); + V1ListMeta_->SetRemainingItemCount(0); + V1ListMeta_->SetResourceVersion("0"); + V1ListMeta_->SetSelfLink("0"); + auto json = V1ListMeta_->ToJson(); + auto result = V1ListMeta_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ListMeta_->RContinueIsSet()); + EXPECT_EQ("0", V1ListMeta_->GetRContinue()); + EXPECT_TRUE(V1ListMeta_->RemainingItemCountIsSet()); + EXPECT_EQ(0, V1ListMeta_->GetRemainingItemCount()); + EXPECT_TRUE(V1ListMeta_->ResourceVersionIsSet()); + EXPECT_EQ("0", V1ListMeta_->GetResourceVersion()); + EXPECT_TRUE(V1ListMeta_->SelfLinkIsSet()); + EXPECT_EQ("0", V1ListMeta_->GetSelfLink()); +} + +TEST_F(KubeClientModelTest, V1LocalObjectReferenceTest) +{ + std::shared_ptr V1LocalObjectReference_ = std::make_shared(); + V1LocalObjectReference_->SetName("0"); + auto json = V1LocalObjectReference_->ToJson(); + auto result = V1LocalObjectReference_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1LocalObjectReference_->NameIsSet()); + EXPECT_EQ("0", V1LocalObjectReference_->GetName()); +} + +TEST_F(KubeClientModelTest, V1NodeAddressTest) +{ + std::shared_ptr V1NodeAddress_ = std::make_shared(); + V1NodeAddress_->SetAddress("0"); + V1NodeAddress_->SetType("0"); + auto json = V1NodeAddress_->ToJson(); + auto result = V1NodeAddress_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1NodeAddress_->AddressIsSet()); + EXPECT_EQ("0", V1NodeAddress_->GetAddress()); + EXPECT_TRUE(V1NodeAddress_->TypeIsSet()); + EXPECT_EQ("0", V1NodeAddress_->GetType()); +} + +TEST_F(KubeClientModelTest, V1NodeListTest) +{ + std::shared_ptr V1NodeList_ = std::make_shared(); + V1NodeList_->UnsetApiVersion(); + V1NodeList_->UnsetKind(); + V1NodeList_->UnsetMetadata(); + V1NodeList_->UnsetItems(); + V1NodeList_->SetApiVersion("0"); + std::vector> Items; + V1NodeList_->SetItems(Items); + V1NodeList_->SetKind("0"); + std::shared_ptr Metadata = std::make_shared(); + V1NodeList_->SetMetadata(Metadata); + auto json = V1NodeList_->ToJson(); + auto result = V1NodeList_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1NodeList_->ApiVersionIsSet()); + EXPECT_EQ("0", V1NodeList_->GetApiVersion()); + EXPECT_TRUE(V1NodeList_->ItemsIsSet()); + EXPECT_TRUE(V1NodeList_->KindIsSet()); + EXPECT_EQ("0", V1NodeList_->GetKind()); + EXPECT_TRUE(V1NodeList_->MetadataIsSet()); +} + +TEST_F(KubeClientModelTest, V1NodeSpecTest) +{ + std::shared_ptr V1NodeSpec_ = std::make_shared(); + V1NodeSpec_->UnsetExternalID(); + V1NodeSpec_->UnSetPodCIDR(); + V1NodeSpec_->UnSetPodCIDRs(); + V1NodeSpec_->UnsetTaints(); + V1NodeSpec_->UnsetProviderID(); + V1NodeSpec_->UnsetUnschedulable(); + V1NodeSpec_->SetExternalID("0"); + V1NodeSpec_->SetPodCIDR("0"); + std::vector PodCIDRs; + V1NodeSpec_->SetPodCIDRs(PodCIDRs); + V1NodeSpec_->SetProviderID("0"); + std::vector> Taints; + V1NodeSpec_->SetTaints(Taints); + V1NodeSpec_->SetUnschedulable(0); + auto json = V1NodeSpec_->ToJson(); + auto result = V1NodeSpec_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1NodeSpec_->ExternalIDIsSet()); + EXPECT_EQ("0", V1NodeSpec_->GetExternalID()); + EXPECT_TRUE(V1NodeSpec_->PodCIDRIsSet()); + EXPECT_EQ("0", V1NodeSpec_->GetPodCIDR()); + EXPECT_TRUE(V1NodeSpec_->PodCIDRsIsSet()); + EXPECT_TRUE(V1NodeSpec_->ProviderIDIsSet()); + EXPECT_EQ("0", V1NodeSpec_->GetProviderID()); + EXPECT_TRUE(V1NodeSpec_->TaintsIsSet()); + EXPECT_TRUE(V1NodeSpec_->UnschedulableIsSet()); + EXPECT_EQ(0, V1NodeSpec_->IsUnschedulable()); +} + +TEST_F(KubeClientModelTest, V1NodeStatusTest) +{ + std::shared_ptr V1NodeStatus_ = std::make_shared(); + std::vector> Addresses; + V1NodeStatus_->SetAddresses(Addresses); + auto json = V1NodeStatus_->ToJson(); + auto result = V1NodeStatus_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1NodeStatus_->AddressesIsSet()); +} + +TEST_F(KubeClientModelTest, V1NodeTest) +{ + std::shared_ptr V1Node_ = std::make_shared(); + V1Node_->UnsetKind(); + V1Node_->UnsetApiVersion(); + V1Node_->UnsetKind(); + V1Node_->UnsetMetadata(); + V1Node_->UnsetSpec(); + V1Node_->UnsetStatus(); + V1Node_->SetApiVersion("0"); + V1Node_->SetKind("0"); + std::shared_ptr Metadata; + V1Node_->SetMetadata(Metadata); + std::shared_ptr Spec; + V1Node_->SetSpec(Spec); + std::shared_ptr Status; + V1Node_->SetStatus(Status); + auto json = V1Node_->ToJson(); + auto result = V1Node_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Node_->ApiVersionIsSet()); + EXPECT_EQ("0", V1Node_->GetApiVersion()); + EXPECT_TRUE(V1Node_->KindIsSet()); + EXPECT_EQ("0", V1Node_->GetKind()); + EXPECT_TRUE(V1Node_->MetadataIsSet()); + EXPECT_TRUE(V1Node_->SpecIsSet()); + EXPECT_TRUE(V1Node_->StatusIsSet()); +} + +TEST_F(KubeClientModelTest, V1ObjectFieldSelectorTest) +{ + std::shared_ptr V1ObjectFieldSelector_ = std::make_shared(); + V1ObjectFieldSelector_->SetApiVersion("0"); + V1ObjectFieldSelector_->SetFieldPath("0"); + auto json = V1ObjectFieldSelector_->ToJson(); + auto result = V1ObjectFieldSelector_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ObjectFieldSelector_->ApiVersionIsSet()); + EXPECT_EQ("0", V1ObjectFieldSelector_->GetApiVersion()); + EXPECT_TRUE(V1ObjectFieldSelector_->FieldPathIsSet()); + EXPECT_EQ("0", V1ObjectFieldSelector_->GetFieldPath()); +} + +TEST_F(KubeClientModelTest, V1ObjectMetaTest) +{ + std::shared_ptr V1ObjectMeta_ = std::make_shared(); + V1ObjectMeta_->UnsetOwnerReferences(); + V1ObjectMeta_->UnsetRNamespace(); + V1ObjectMeta_->UnsetName(); + V1ObjectMeta_->UnsetLabels(); + V1ObjectMeta_->UnsetGenerateName(); + V1ObjectMeta_->UnsetFinalizers(); + V1ObjectMeta_->UnsetAnnotations(); + V1ObjectMeta_->UnsetDeletionTimestamp(); + std::map Annotations; + Annotations["test"] = "abc"; + V1ObjectMeta_->SetAnnotations(Annotations); + V1ObjectMeta_->SetCreationTimestamp(utility::Datetime()); + V1ObjectMeta_->SetDeletionTimestamp(utility::Datetime()); + std::vector Finalizers; + V1ObjectMeta_->SetFinalizers(Finalizers); + V1ObjectMeta_->SetGenerateName("0"); + V1ObjectMeta_->SetGeneration(0); + std::map Labels; + V1ObjectMeta_->SetLabels(Labels); + V1ObjectMeta_->SetName("0"); + V1ObjectMeta_->SetRNamespace("0"); + std::vector> OwnerReferences; + V1ObjectMeta_->SetOwnerReferences(OwnerReferences); + V1ObjectMeta_->SetResourceVersion("0"); + V1ObjectMeta_->SetSelfLink("0"); + V1ObjectMeta_->SetUid("0"); + auto json = V1ObjectMeta_->ToJson(); + auto result = V1ObjectMeta_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ObjectMeta_->AnnotationsIsSet()); + EXPECT_EQ("abc", V1ObjectMeta_->GetAnnotations()["test"]); + EXPECT_TRUE(V1ObjectMeta_->CreationTimestampIsSet()); + EXPECT_TRUE(V1ObjectMeta_->DeletionTimestampIsSet()); + EXPECT_TRUE(V1ObjectMeta_->FinalizersIsSet()); + EXPECT_EQ(static_cast(0), V1ObjectMeta_->GetFinalizers().size()); + EXPECT_TRUE(V1ObjectMeta_->GenerateNameIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetGenerateName()); + EXPECT_TRUE(V1ObjectMeta_->GenerationIsSet()); + EXPECT_EQ(0, V1ObjectMeta_->GetGeneration()); + EXPECT_TRUE(V1ObjectMeta_->LabelsIsSet()); + EXPECT_TRUE(V1ObjectMeta_->NameIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetName()); + EXPECT_TRUE(V1ObjectMeta_->RNamespaceIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetRNamespace()); + EXPECT_TRUE(V1ObjectMeta_->OwnerReferencesIsSet()); + EXPECT_TRUE(V1ObjectMeta_->ResourceVersionIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetResourceVersion()); + EXPECT_TRUE(V1ObjectMeta_->SelfLinkIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetSelfLink()); + EXPECT_TRUE(V1ObjectMeta_->UidIsSet()); + EXPECT_EQ("0", V1ObjectMeta_->GetUid()); +} + +TEST_F(KubeClientModelTest, V1OwnerReferenceTest) +{ + std::shared_ptr V1OwnerReference_ = std::make_shared(); + V1OwnerReference_->UnsetApiVersion(); + V1OwnerReference_->UnsetKind(); + V1OwnerReference_->UnsetName(); + V1OwnerReference_->UnsetUid(); + V1OwnerReference_->UnsetController(); + V1OwnerReference_->UnsetBlockOwnerDeletion(); + V1OwnerReference_->SetApiVersion("0"); + V1OwnerReference_->SetBlockOwnerDeletion(0); + V1OwnerReference_->SetController(0); + V1OwnerReference_->SetKind("0"); + V1OwnerReference_->SetName("0"); + V1OwnerReference_->SetUid("0"); + auto json = V1OwnerReference_->ToJson(); + auto result = V1OwnerReference_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1OwnerReference_->ApiVersionIsSet()); + EXPECT_EQ("0", V1OwnerReference_->GetApiVersion()); + EXPECT_TRUE(V1OwnerReference_->BlockOwnerDeletionIsSet()); + EXPECT_EQ(0, V1OwnerReference_->IsBlockOwnerDeletion()); + EXPECT_TRUE(V1OwnerReference_->ControllerIsSet()); + EXPECT_EQ(0, V1OwnerReference_->IsController()); + EXPECT_TRUE(V1OwnerReference_->KindIsSet()); + EXPECT_EQ("0", V1OwnerReference_->GetKind()); + EXPECT_TRUE(V1OwnerReference_->NameIsSet()); + EXPECT_EQ("0", V1OwnerReference_->GetName()); + EXPECT_TRUE(V1OwnerReference_->UidIsSet()); + EXPECT_EQ("0", V1OwnerReference_->GetUid()); +} + +TEST_F(KubeClientModelTest, V1PodAffinityTermTest) +{ + std::shared_ptr V1PodAffinityTerm_ = std::make_shared(); + V1PodAffinityTerm_->UnsetLabelSelector(); + V1PodAffinityTerm_->UnsetNamespaceSelector(); + V1PodAffinityTerm_->UnsetNamespaces(); + V1PodAffinityTerm_->UnsetTopologyKey(); + std::shared_ptr LabelSelector = std::make_shared(); + V1PodAffinityTerm_->SetLabelSelector(LabelSelector); + std::shared_ptr NamespaceSelector = std::make_shared(); + V1PodAffinityTerm_->SetNamespaceSelector(NamespaceSelector); + std::vector Namespaces; + V1PodAffinityTerm_->SetNamespaces(Namespaces); + V1PodAffinityTerm_->SetTopologyKey("0"); + auto json = V1PodAffinityTerm_->ToJson(); + auto result = V1PodAffinityTerm_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodAffinityTerm_->LabelSelectorIsSet()); + EXPECT_TRUE(V1PodAffinityTerm_->NamespaceSelectorIsSet()); + EXPECT_TRUE(V1PodAffinityTerm_->NamespacesIsSet()); + EXPECT_TRUE(V1PodAffinityTerm_->TopologyKeyIsSet()); + EXPECT_EQ("0", V1PodAffinityTerm_->GetTopologyKey()); +} + +TEST_F(KubeClientModelTest, V1PodAffinityTest) +{ + std::shared_ptr V1PodAffinity_ = std::make_shared(); + std::vector> PreferredDuringSchedulingIgnoredDuringExecution; + V1PodAffinity_->SetPreferredDuringSchedulingIgnoredDuringExecution(PreferredDuringSchedulingIgnoredDuringExecution); + std::vector> RequiredDuringSchedulingIgnoredDuringExecution; + V1PodAffinity_->SetRequiredDuringSchedulingIgnoredDuringExecution(RequiredDuringSchedulingIgnoredDuringExecution); + auto json = V1PodAffinity_->ToJson(); + auto result = V1PodAffinity_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodAffinity_->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_TRUE(V1PodAffinity_->RequiredDuringSchedulingIgnoredDuringExecutionIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodAntiAffinityTest) +{ + std::shared_ptr V1PodAntiAffinity_ = std::make_shared(); + std::vector> PreferredDuringSchedulingIgnoredDuringExecution; + V1PodAntiAffinity_->SetPreferredDuringSchedulingIgnoredDuringExecution( + PreferredDuringSchedulingIgnoredDuringExecution); + std::vector> RequiredDuringSchedulingIgnoredDuringExecution; + V1PodAntiAffinity_->SetRequiredDuringSchedulingIgnoredDuringExecution( + RequiredDuringSchedulingIgnoredDuringExecution); + auto json = V1PodAntiAffinity_->ToJson(); + auto result = V1PodAntiAffinity_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodAntiAffinity_->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_TRUE(V1PodAntiAffinity_->RequiredDuringSchedulingIgnoredDuringExecutionIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodListTest) +{ + std::shared_ptr V1PodList_ = std::make_shared(); + V1PodList_->SetApiVersion("0"); + std::vector> Items; + V1PodList_->SetItems(Items); + V1PodList_->SetKind("0"); + std::shared_ptr Metadata; + V1PodList_->SetMetadata(Metadata); + auto json = V1PodList_->ToJson(); + auto result = V1PodList_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodList_->ApiVersionIsSet()); + EXPECT_EQ("0", V1PodList_->GetApiVersion()); + EXPECT_TRUE(V1PodList_->ItemsIsSet()); + EXPECT_TRUE(V1PodList_->KindIsSet()); + EXPECT_EQ("0", V1PodList_->GetKind()); + EXPECT_TRUE(V1PodList_->MetadataIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodSecurityContextTest) +{ + std::shared_ptr V1PodSecurityContext_ = std::make_shared(); + V1PodSecurityContext_->SetFsGroup(0); + V1PodSecurityContext_->SetFsGroupChangePolicy("0"); + V1PodSecurityContext_->SetRunAsGroup(0); + V1PodSecurityContext_->SetRunAsNonRoot(0); + V1PodSecurityContext_->SetRunAsUser(0); + std::shared_ptr seccompProfile = std::make_shared(); + seccompProfile->SetType("RuntimeDefault"); + seccompProfile->SetLocalhostProfile("test"); + V1PodSecurityContext_->SetSeccompProfile(seccompProfile); + std::vector groupIDs{ 1000, 1002 }; + V1PodSecurityContext_->SetSupplementalGroups(groupIDs); + auto json = V1PodSecurityContext_->ToJson(); + auto result = V1PodSecurityContext_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodSecurityContext_->FsGroupIsSet()); + EXPECT_EQ(0, V1PodSecurityContext_->GetFsGroup()); + EXPECT_TRUE(V1PodSecurityContext_->FsGroupChangePolicyIsSet()); + EXPECT_EQ("0", V1PodSecurityContext_->GetFsGroupChangePolicy()); + EXPECT_TRUE(V1PodSecurityContext_->RunAsGroupIsSet()); + EXPECT_EQ(0, V1PodSecurityContext_->GetRunAsGroup()); + EXPECT_TRUE(V1PodSecurityContext_->RunAsNonRootIsSet()); + EXPECT_EQ(0, V1PodSecurityContext_->IsRunAsNonRoot()); + EXPECT_TRUE(V1PodSecurityContext_->RunAsUserIsSet()); + EXPECT_EQ(0, V1PodSecurityContext_->GetRunAsUser()); + EXPECT_EQ(2, V1PodSecurityContext_->GetSupplementalGroups().size()); +} + +TEST_F(KubeClientModelTest, V1PodSpecTest) +{ + std::shared_ptr V1PodSpec_ = std::make_shared(); + V1PodSpec_->UnsetAffinity(); + V1PodSpec_->UnsetAutomountServiceAccountToken(); + V1PodSpec_->UnsetContainers(); + V1PodSpec_->UnsetDnsPolicy(); + V1PodSpec_->UnsetHostAliases(); + V1PodSpec_->UnsetHostNetwork(); + V1PodSpec_->UnsetImagePullSecrets(); + V1PodSpec_->UnsetInitContainers(); + V1PodSpec_->UnsetNodeSelector(); + V1PodSpec_->UnsetPriorityClassName(); + V1PodSpec_->UnsetRestartPolicy(); + V1PodSpec_->UnsetSchedulerName(); + V1PodSpec_->UnSetSecurityContext(); + V1PodSpec_->UnSetServiceAccountName(); + V1PodSpec_->UnsetTerminationGracePeriodSeconds(); + V1PodSpec_->UnsetVolumes(); + V1PodSpec_->UnsetHostPID(); + V1PodSpec_->UnsetTopologySpreadConstraints(); + std::shared_ptr Affinity = std::make_shared(); + V1PodSpec_->SetAffinity(Affinity); + V1PodSpec_->SetAutomountServiceAccountToken(0); + std::vector> Containers; + V1PodSpec_->SetContainers(Containers); + V1PodSpec_->SetDnsPolicy("0"); + std::vector> HostAliases; + V1PodSpec_->SetHostAliases(HostAliases); + V1PodSpec_->SetHostNetwork(0); + std::vector> ImagePullSecrets; + V1PodSpec_->SetImagePullSecrets(ImagePullSecrets); + std::vector> InitContainers; + V1PodSpec_->SetInitContainers(InitContainers); + std::map NodeSelector; + NodeSelector["node"] = "abc"; + V1PodSpec_->SetNodeSelector(NodeSelector); + std::map Overhead; + V1PodSpec_->SetPriorityClassName("0"); + V1PodSpec_->SetRestartPolicy("0"); + V1PodSpec_->SetSchedulerName("0"); + std::shared_ptr SecurityContext; + V1PodSpec_->SetSecurityContext(SecurityContext); + V1PodSpec_->SetServiceAccountName("0"); + V1PodSpec_->SetTerminationGracePeriodSeconds(0); + std::vector> Volumes; + V1PodSpec_->SetVolumes(Volumes); + V1PodSpec_->SetHostPID(true); + auto V1TopologySpreadConstraint_ = std::make_shared(); + V1TopologySpreadConstraint_->SetMaxSkew(1); + V1TopologySpreadConstraint_->SetMinDomains(1); + V1PodSpec_->SetTopologySpreadConstraints({V1TopologySpreadConstraint_}); + auto json = V1PodSpec_->ToJson(); + auto result = V1PodSpec_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodSpec_->AffinityIsSet()); + EXPECT_TRUE(V1PodSpec_->AutomountServiceAccountTokenIsSet()); + EXPECT_EQ(0, V1PodSpec_->IsAutomountServiceAccountToken()); + EXPECT_TRUE(V1PodSpec_->ContainersIsSet()); + EXPECT_TRUE(V1PodSpec_->DnsPolicyIsSet()); + EXPECT_EQ("0", V1PodSpec_->GetDnsPolicy()); + EXPECT_TRUE(V1PodSpec_->HostAliasesIsSet()); + EXPECT_TRUE(V1PodSpec_->HostNetworkIsSet()); + EXPECT_EQ(0, V1PodSpec_->IsHostNetwork()); + EXPECT_TRUE(V1PodSpec_->ImagePullSecretsIsSet()); + EXPECT_TRUE(V1PodSpec_->InitContainersIsSet()); + EXPECT_TRUE(V1PodSpec_->NodeSelectorIsSet()); + EXPECT_TRUE(V1PodSpec_->PriorityClassNameIsSet()); + EXPECT_EQ("0", V1PodSpec_->GetPriorityClassName()); + EXPECT_TRUE(V1PodSpec_->RestartPolicyIsSet()); + EXPECT_EQ("0", V1PodSpec_->GetRestartPolicy()); + EXPECT_TRUE(V1PodSpec_->SchedulerNameIsSet()); + EXPECT_EQ("0", V1PodSpec_->GetSchedulerName()); + EXPECT_TRUE(V1PodSpec_->SecurityContextIsSet()); + EXPECT_TRUE(V1PodSpec_->ServiceAccountNameIsSet()); + EXPECT_EQ("0", V1PodSpec_->GetServiceAccountName()); + EXPECT_TRUE(V1PodSpec_->TerminationGracePeriodSecondsIsSet()); + EXPECT_EQ(0, V1PodSpec_->GetTerminationGracePeriodSeconds()); + EXPECT_TRUE(V1PodSpec_->VolumesIsSet()); + EXPECT_TRUE(V1PodSpec_->TopologySpreadConstraintsIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodStatusTest) +{ + std::shared_ptr V1PodStatus_ = std::make_shared(); + V1PodStatus_->UnsetContainerStatuses(); + V1PodStatus_->UnsetEphemeralContainerStatuses(); + V1PodStatus_->UnsetHostIP(); + V1PodStatus_->UnsetInitContainerStatuses(); + V1PodStatus_->UnsetMessage(); + V1PodStatus_->UnsetNominatedNodeName(); + V1PodStatus_->UnsetPhase(); + V1PodStatus_->UnSetPodIP(); + V1PodStatus_->UnsetQosClass(); + V1PodStatus_->UnsetReason(); + V1PodStatus_->UnsetStartTime(); + std::vector> ContainerStatuses; + V1PodStatus_->SetContainerStatuses(ContainerStatuses); + std::vector> EphemeralContainerStatuses; + V1PodStatus_->SetEphemeralContainerStatuses(EphemeralContainerStatuses); + V1PodStatus_->SetHostIP("0"); + std::vector> InitContainerStatuses; + V1PodStatus_->SetInitContainerStatuses(InitContainerStatuses); + V1PodStatus_->SetMessage("0"); + V1PodStatus_->SetNominatedNodeName("0"); + V1PodStatus_->SetPhase("0"); + V1PodStatus_->SetPodIP("0"); + V1PodStatus_->SetQosClass("0"); + V1PodStatus_->SetReason("0"); + V1PodStatus_->SetStartTime(utility::Datetime()); + auto json = V1PodStatus_->ToJson(); + auto result = V1PodStatus_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodStatus_->ContainerStatusesIsSet()); + EXPECT_TRUE(V1PodStatus_->EphemeralContainerStatusesIsSet()); + EXPECT_TRUE(V1PodStatus_->HostIPIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetHostIP()); + EXPECT_TRUE(V1PodStatus_->InitContainerStatusesIsSet()); + EXPECT_TRUE(V1PodStatus_->MessageIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetMessage()); + EXPECT_TRUE(V1PodStatus_->NominatedNodeNameIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetNominatedNodeName()); + EXPECT_TRUE(V1PodStatus_->PhaseIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetPhase()); + EXPECT_TRUE(V1PodStatus_->PodIPIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetPodIP()); + EXPECT_TRUE(V1PodStatus_->QosClassIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetQosClass()); + EXPECT_TRUE(V1PodStatus_->ReasonIsSet()); + EXPECT_EQ("0", V1PodStatus_->GetReason()); + EXPECT_TRUE(V1PodStatus_->StartTimeIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodTemplateSpecTest) +{ + std::shared_ptr V1PodTemplateSpec_ = std::make_shared(); + std::shared_ptr Metadata; + V1PodTemplateSpec_->SetMetadata(Metadata); + std::shared_ptr Spec; + V1PodTemplateSpec_->SetSpec(Spec); + auto json = V1PodTemplateSpec_->ToJson(); + auto result = V1PodTemplateSpec_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1PodTemplateSpec_->MetadataIsSet()); + EXPECT_TRUE(V1PodTemplateSpec_->SpecIsSet()); +} + +TEST_F(KubeClientModelTest, V1PodTest) +{ + std::shared_ptr V1Pod_ = std::make_shared(); + V1Pod_->SetApiVersion("0"); + V1Pod_->SetKind("0"); + std::shared_ptr Metadata; + V1Pod_->SetMetadata(Metadata); + std::shared_ptr Spec; + V1Pod_->SetSpec(Spec); + std::shared_ptr Status; + V1Pod_->SetStatus(Status); + auto json = V1Pod_->ToJson(); + auto result = V1Pod_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Pod_->ApiVersionIsSet()); + EXPECT_EQ("0", V1Pod_->GetApiVersion()); + EXPECT_TRUE(V1Pod_->KindIsSet()); + EXPECT_EQ("0", V1Pod_->GetKind()); + EXPECT_TRUE(V1Pod_->MetadataIsSet()); + EXPECT_TRUE(V1Pod_->SpecIsSet()); + EXPECT_TRUE(V1Pod_->StatusIsSet()); +} + +TEST_F(KubeClientModelTest, V1ProbeTest) +{ + std::shared_ptr V1Probe_ = std::make_shared(); + V1Probe_->UnsetFailureThreshold(); + V1Probe_->UnsetInitialDelaySeconds(); + V1Probe_->UnsetPeriodSeconds(); + V1Probe_->UnsetSuccessThreshold(); + V1Probe_->UnsetTcpSocket(); + V1Probe_->UnsetExec(); + V1Probe_->UnsetTerminationGracePeriodSeconds(); + V1Probe_->UnsetTimeoutSeconds(); + V1Probe_->SetFailureThreshold(0); + V1Probe_->SetInitialDelaySeconds(0); + V1Probe_->SetPeriodSeconds(0); + V1Probe_->SetSuccessThreshold(0); + std::shared_ptr TcpSocket; + V1Probe_->SetTcpSocket(TcpSocket); + std::shared_ptr execAction; + V1Probe_->SetExec(execAction); + V1Probe_->SetTerminationGracePeriodSeconds(0); + V1Probe_->SetTimeoutSeconds(0); + auto json = V1Probe_->ToJson(); + auto result = V1Probe_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Probe_->FailureThresholdIsSet()); + EXPECT_EQ(0, V1Probe_->GetFailureThreshold()); + EXPECT_TRUE(V1Probe_->InitialDelaySecondsIsSet()); + EXPECT_EQ(0, V1Probe_->GetInitialDelaySeconds()); + EXPECT_TRUE(V1Probe_->PeriodSecondsIsSet()); + EXPECT_EQ(0, V1Probe_->GetPeriodSeconds()); + EXPECT_TRUE(V1Probe_->SuccessThresholdIsSet()); + EXPECT_EQ(0, V1Probe_->GetSuccessThreshold()); + EXPECT_TRUE(V1Probe_->TcpSocketIsSet()); + EXPECT_TRUE(V1Probe_->ExecIsSet()); + EXPECT_TRUE(V1Probe_->TerminationGracePeriodSecondsIsSet()); + EXPECT_EQ(0, V1Probe_->GetTerminationGracePeriodSeconds()); + EXPECT_TRUE(V1Probe_->TimeoutSecondsIsSet()); + EXPECT_EQ(0, V1Probe_->GetTimeoutSeconds()); +} + +TEST_F(KubeClientModelTest, V1ResourceFieldSelectorTest) +{ + std::shared_ptr V1ResourceFieldSelector_ = std::make_shared(); + V1ResourceFieldSelector_->SetContainerName("0"); + V1ResourceFieldSelector_->SetDivisor("0"); + V1ResourceFieldSelector_->SetResource("0"); + auto json = V1ResourceFieldSelector_->ToJson(); + auto result = V1ResourceFieldSelector_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ResourceFieldSelector_->ContainerNameIsSet()); + EXPECT_EQ("0", V1ResourceFieldSelector_->GetContainerName()); + EXPECT_TRUE(V1ResourceFieldSelector_->DivisorIsSet()); + EXPECT_EQ("0", V1ResourceFieldSelector_->GetDivisor()); + EXPECT_TRUE(V1ResourceFieldSelector_->ResourceIsSet()); + EXPECT_EQ("0", V1ResourceFieldSelector_->GetResource()); +} + +TEST_F(KubeClientModelTest, V1ResourceRequirementsTest) +{ + std::shared_ptr V1ResourceRequirements_ = std::make_shared(); + std::map Limits; + V1ResourceRequirements_->SetLimits(Limits); + std::map Requests; + V1ResourceRequirements_->SetRequests(Requests); + auto json = V1ResourceRequirements_->ToJson(); + auto result = V1ResourceRequirements_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ResourceRequirements_->LimitsIsSet()); + EXPECT_TRUE(V1ResourceRequirements_->RequestsIsSet()); +} + +TEST_F(KubeClientModelTest, V1SecretVolumeSourceTest) +{ + std::shared_ptr V1SecretVolumeSource_ = std::make_shared(); + V1SecretVolumeSource_->UnsetItems(); + V1SecretVolumeSource_->UnSetSecretName(); + V1SecretVolumeSource_->UnsetDefaultMode(); + V1SecretVolumeSource_->UnsetOptional(); + V1SecretVolumeSource_->SetDefaultMode(0); + std::vector> Items; + V1SecretVolumeSource_->SetItems(Items); + V1SecretVolumeSource_->SetOptional(0); + V1SecretVolumeSource_->SetSecretName("0"); + auto json = V1SecretVolumeSource_->ToJson(); + auto result = V1SecretVolumeSource_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1SecretVolumeSource_->DefaultModeIsSet()); + EXPECT_EQ(0, V1SecretVolumeSource_->GetDefaultMode()); + EXPECT_TRUE(V1SecretVolumeSource_->ItemsIsSet()); + EXPECT_TRUE(V1SecretVolumeSource_->OptionalIsSet()); + EXPECT_EQ(0, V1SecretVolumeSource_->IsOptional()); + EXPECT_TRUE(V1SecretVolumeSource_->SecretNameIsSet()); + EXPECT_EQ("0", V1SecretVolumeSource_->GetSecretName()); +} + +TEST_F(KubeClientModelTest, V1SecurityContextTest) +{ + std::shared_ptr V1SecurityContext_ = std::make_shared(); + V1SecurityContext_->UnsetAllowPrivilegeEscalation(); + V1SecurityContext_->UnsetCapabilities(); + V1SecurityContext_->UnsetPrivileged(); + V1SecurityContext_->UnsetRunAsGroup(); + V1SecurityContext_->UnsetRunAsNonRoot(); + V1SecurityContext_->UnsetRunAsUser(); + V1SecurityContext_->UnSetSeccompProfile(); + V1SecurityContext_->SetAllowPrivilegeEscalation(0); + std::shared_ptr seccompProfile = std::make_shared(); + V1SecurityContext_->SetSeccompProfile(seccompProfile); + std::shared_ptr Capabilities; + V1SecurityContext_->SetCapabilities(Capabilities); + V1SecurityContext_->SetPrivileged(0); + V1SecurityContext_->SetRunAsGroup(0); + V1SecurityContext_->SetRunAsNonRoot(0); + V1SecurityContext_->SetRunAsUser(0); + auto json = V1SecurityContext_->ToJson(); + auto result = V1SecurityContext_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1SecurityContext_->AllowPrivilegeEscalationIsSet()); + EXPECT_EQ(0, V1SecurityContext_->IsAllowPrivilegeEscalation()); + EXPECT_TRUE(V1SecurityContext_->CapabilitiesIsSet()); + EXPECT_TRUE(V1SecurityContext_->PrivilegedIsSet()); + EXPECT_EQ(0, V1SecurityContext_->IsPrivileged()); + EXPECT_TRUE(V1SecurityContext_->RunAsGroupIsSet()); + EXPECT_EQ(0, V1SecurityContext_->GetRunAsGroup()); + EXPECT_TRUE(V1SecurityContext_->RunAsNonRootIsSet()); + EXPECT_EQ(0, V1SecurityContext_->IsRunAsNonRoot()); + EXPECT_TRUE(V1SecurityContext_->RunAsUserIsSet()); + EXPECT_EQ(0, V1SecurityContext_->GetRunAsUser()); +} + +TEST_F(KubeClientModelTest, V1TaintTest) +{ + std::shared_ptr V1Taint_ = std::make_shared(); + V1Taint_->UnSetEffect(); + V1Taint_->UnsetKey(); + V1Taint_->UnsetTimeAdded(); + V1Taint_->UnsetValue(); + V1Taint_->SetEffect("0"); + V1Taint_->SetKey("0"); + V1Taint_->SetTimeAdded(utility::Datetime()); + V1Taint_->SetValue("0"); + auto json = V1Taint_->ToJson(); + auto result = V1Taint_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Taint_->EffectIsSet()); + EXPECT_EQ("0", V1Taint_->GetEffect()); + EXPECT_TRUE(V1Taint_->KeyIsSet()); + EXPECT_EQ("0", V1Taint_->GetKey()); + EXPECT_TRUE(V1Taint_->TimeAddedIsSet()); + EXPECT_TRUE(V1Taint_->ValueIsSet()); + EXPECT_EQ("0", V1Taint_->GetValue()); +} + +TEST_F(KubeClientModelTest, V1TCPSocketActionTest) +{ + std::shared_ptr V1TCPSocketAction_ = std::make_shared(); + V1TCPSocketAction_->SetHost("0"); + V1TCPSocketAction_->SetPort(0); + auto json = V1TCPSocketAction_->ToJson(); + auto result = V1TCPSocketAction_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1TCPSocketAction_->HostIsSet()); + EXPECT_EQ("0", V1TCPSocketAction_->GetHost()); + EXPECT_TRUE(V1TCPSocketAction_->PortIsSet()); + EXPECT_EQ(0, V1TCPSocketAction_->GetPort()); +} + +TEST_F(KubeClientModelTest, V1VolumeMountTest) +{ + std::shared_ptr V1VolumeMount_ = std::make_shared(); + V1VolumeMount_->UnsetName(); + V1VolumeMount_->UnsetMountPath(); + V1VolumeMount_->UnsetMountPropagation(); + V1VolumeMount_->UnsetReadOnly(); + V1VolumeMount_->UnsetSubPath(); + V1VolumeMount_->UnsetSubPathExpr(); + V1VolumeMount_->SetMountPath("0"); + V1VolumeMount_->SetMountPropagation("0"); + V1VolumeMount_->SetName("0"); + V1VolumeMount_->SetReadOnly(0); + V1VolumeMount_->SetSubPath("0"); + V1VolumeMount_->SetSubPathExpr("0"); + auto json = V1VolumeMount_->ToJson(); + auto result = V1VolumeMount_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1VolumeMount_->MountPathIsSet()); + EXPECT_EQ("0", V1VolumeMount_->GetMountPath()); + EXPECT_TRUE(V1VolumeMount_->MountPropagationIsSet()); + EXPECT_EQ("0", V1VolumeMount_->GetMountPropagation()); + EXPECT_TRUE(V1VolumeMount_->NameIsSet()); + EXPECT_EQ("0", V1VolumeMount_->GetName()); + EXPECT_TRUE(V1VolumeMount_->ReadOnlyIsSet()); + EXPECT_EQ(0, V1VolumeMount_->IsReadOnly()); + EXPECT_TRUE(V1VolumeMount_->SubPathIsSet()); + EXPECT_EQ("0", V1VolumeMount_->GetSubPath()); + EXPECT_TRUE(V1VolumeMount_->SubPathExprIsSet()); + EXPECT_EQ("0", V1VolumeMount_->GetSubPathExpr()); +} + +TEST_F(KubeClientModelTest, V1VolumeTest) +{ + std::shared_ptr V1Volume_ = std::make_shared(); + V1Volume_->UnsetName(); + V1Volume_->UnsetConfigMap(); + V1Volume_->UnsetHostPath(); + V1Volume_->UnSetSecret(); + V1Volume_->UnsetEmptyDir(); + V1Volume_->UnsetProjected(); + V1Volume_->SetName("0"); + // configmap + std::shared_ptr ConfigMap = std::make_shared(); + ConfigMap->SetDefaultMode(640); + ConfigMap->SetName("config-map"); + ConfigMap->SetOptional(true); + std::shared_ptr v1KeyToPath = std::make_shared(); + v1KeyToPath->SetMode(640); + v1KeyToPath->SetPath("path"); + v1KeyToPath->SetKey("key1"); + std::vector> v1KeyToPaths{v1KeyToPath}; + ConfigMap->SetItems(v1KeyToPaths); + V1Volume_->SetConfigMap(ConfigMap); + // empty dir + std::shared_ptr EmptyDir = std::make_shared(); + EmptyDir->SetMedium("test"); + EmptyDir->SetSizeLimit("10Gi"); + V1Volume_->SetEmptyDir(EmptyDir); + // host path + std::shared_ptr HostPath = std::make_shared(); + HostPath->SetType("host"); + HostPath->SetPath("/home"); + V1Volume_->SetHostPath(HostPath); + // secret + std::shared_ptr Secret = std::make_shared(); + Secret->SetOptional(true); + Secret->SetSecretName("secret-1"); + Secret->SetDefaultMode(500); + Secret->SetItems(v1KeyToPaths); + V1Volume_->SetSecret(Secret); + // projected volume + std::shared_ptr projected = std::make_shared(); + std::vector> projectionVolumes; + std::shared_ptr objectFieldSelector = std::make_shared(); + objectFieldSelector->SetFieldPath("testPath"); + std::shared_ptr v1DownwardApiVolumeFile = std::make_shared(); + v1DownwardApiVolumeFile->SetFieldRef(objectFieldSelector); + v1DownwardApiVolumeFile->SetPath("test-path"); + v1DownwardApiVolumeFile->SetMode(1); + std::shared_ptr resourceFieldSelector = std::make_shared(); + resourceFieldSelector->SetContainerName("test"); + resourceFieldSelector->SetDivisor("divisor"); + resourceFieldSelector->SetResource("resource"); + v1DownwardApiVolumeFile->SetResourceFieldRef(resourceFieldSelector); + std::shared_ptr secretKeySelector = std::make_shared(); + secretKeySelector->SetKey("test"); + secretKeySelector->SetName("divisor"); + v1DownwardApiVolumeFile->SetSecretKeyRef(secretKeySelector); + std::vector> downwardAPIVolumeFiles; + downwardAPIVolumeFiles.emplace_back(v1DownwardApiVolumeFile); + std::shared_ptr downwardApiProjection = std::make_shared(); + downwardApiProjection->SetItems(downwardAPIVolumeFiles); + std::shared_ptr projectionVol = std::make_shared(); + projectionVol->SetDownwardAPI(downwardApiProjection); + projectionVolumes.emplace_back(projectionVol); + std::shared_ptr projectedVolumeSource = std::make_shared(); + projectedVolumeSource->SetSources(projectionVolumes); + V1Volume_->SetProjected(projectedVolumeSource); + // download api + std::shared_ptr v1DownwardApiVolumeSource = std::make_shared(); + v1DownwardApiVolumeFile->SetPath("down"); + v1DownwardApiVolumeFile->SetMode(500); + v1DownwardApiVolumeFile->SetFieldRef(objectFieldSelector); + v1DownwardApiVolumeFile->SetResourceFieldRef(resourceFieldSelector); + v1DownwardApiVolumeFile->SetSecretKeyRef(secretKeySelector); + V1Volume_->SetDownwardAPI(v1DownwardApiVolumeSource); + // pvc + std::shared_ptr pvcVolume = std::make_shared(); + pvcVolume->SetClaimName("pvc-1"); + pvcVolume->SetReadOnly(true); + V1Volume_->SetPersistentVolumeClaim(pvcVolume); + auto json = V1Volume_->ToJson(); + auto result = V1Volume_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Volume_->ConfigMapIsSet()); + EXPECT_TRUE(V1Volume_->EmptyDirIsSet()); + EXPECT_TRUE(V1Volume_->HostPathIsSet()); + EXPECT_TRUE(V1Volume_->NameIsSet()); + EXPECT_EQ("0", V1Volume_->GetName()); + EXPECT_TRUE(V1Volume_->SecretIsSet()); + EXPECT_TRUE(V1Volume_->ProjectedIsSet()); + EXPECT_TRUE(V1Volume_->DownwardAPIIsSet()); + EXPECT_TRUE(V1Volume_->PersistentVolumeClaimIsSet()); + EXPECT_EQ("pvc-1", V1Volume_->GetPersistentVolumeClaim()->GetClaimName()); +} + +TEST_F(KubeClientModelTest, V1WeightedPodAffinityTermTest) +{ + std::shared_ptr V1WeightedPodAffinityTerm_ = + std::make_shared(); + std::shared_ptr PodAffinityTerm; + V1WeightedPodAffinityTerm_->SetPodAffinityTerm(PodAffinityTerm); + V1WeightedPodAffinityTerm_->SetWeight(0); + auto json = V1WeightedPodAffinityTerm_->ToJson(); + auto result = V1WeightedPodAffinityTerm_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1WeightedPodAffinityTerm_->PodAffinityTermIsSet()); + EXPECT_TRUE(V1WeightedPodAffinityTerm_->WeightIsSet()); + EXPECT_EQ(0, V1WeightedPodAffinityTerm_->GetWeight()); +} + +TEST_F(KubeClientModelTest, V1ContainerStateTest) +{ + std::shared_ptr V1ContainerState_ = std::make_shared(); + V1ContainerState_->UnsetWaiting(); + V1ContainerState_->UnsetTerminated(); + V1ContainerState_->UnsetRunning(); + std::shared_ptr Running = std::make_shared(); + Running->SetStartedAt(utility::Datetime()); + auto json = Running->ToJson(); + auto result = Running->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(Running->StartedAtIsSet()); + + std::shared_ptr Terminated = std::make_shared(); + Terminated->SetContainerID("0"); + Terminated->SetExitCode(0); + Terminated->SetFinishedAt(utility::Datetime()); + Terminated->SetStartedAt(utility::Datetime()); + json = Terminated->ToJson(); + result = Terminated->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(Terminated->ContainerIDIsSet()); + EXPECT_EQ("0", Terminated->GetContainerID()); + EXPECT_TRUE(Terminated->ExitCodeIsSet()); + EXPECT_EQ(0, Terminated->GetExitCode()); + EXPECT_TRUE(Terminated->FinishedAtIsSet()); + EXPECT_TRUE(Terminated->StartedAtIsSet()); + + std::shared_ptr Waiting = std::make_shared(); + Waiting->SetMessage("0"); + Waiting->SetReason("0"); + json = Waiting->ToJson(); + result = Waiting->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(Waiting->MessageIsSet()); + EXPECT_EQ("0", Waiting->GetMessage()); + EXPECT_TRUE(Waiting->ReasonIsSet()); + EXPECT_EQ("0", Waiting->GetReason()); + + V1ContainerState_->SetRunning(Running); + V1ContainerState_->SetTerminated(Terminated); + V1ContainerState_->SetWaiting(Waiting); + json = V1ContainerState_->ToJson(); + result = V1ContainerState_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1ContainerState_->RunningIsSet()); + EXPECT_TRUE(V1ContainerState_->TerminatedIsSet()); + EXPECT_TRUE(V1ContainerState_->WaitingIsSet()); +} + +TEST_F(KubeClientModelTest, V1TolerationTest) +{ + auto V1Toleration_ = std::make_shared(); + V1Toleration_->SetEffect("0"); + V1Toleration_->SetKey("0"); + V1Toleration_->SetROperator("0"); + V1Toleration_->SetTolerationSeconds(0); + V1Toleration_->SetValue("0"); + auto json = V1Toleration_->ToJson(); + auto result = V1Toleration_->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(V1Toleration_->EffectIsSet()); + EXPECT_EQ("0", V1Toleration_->GetEffect()); + EXPECT_TRUE(V1Toleration_->KeyIsSet()); + EXPECT_EQ("0", V1Toleration_->GetKey()); + EXPECT_TRUE(V1Toleration_->ROperatorIsSet()); + EXPECT_EQ("0", V1Toleration_->GetROperator()); + EXPECT_TRUE(V1Toleration_->TolerationSecondsIsSet()); + EXPECT_EQ(0, V1Toleration_->GetTolerationSeconds()); + EXPECT_TRUE(V1Toleration_->ValueIsSet()); + EXPECT_EQ("0", V1Toleration_->GetValue()); +} + + +TEST_F(KubeClientModelTest, V1TopologySpreadConstraintTest) +{ + auto V1TopologySpreadConstraint_ = std::make_shared(); + V1TopologySpreadConstraint_->UnsetMaxSkew(); + V1TopologySpreadConstraint_->UnsetMatchLabelKeys(); + V1TopologySpreadConstraint_->UnsetTopologyKey(); + V1TopologySpreadConstraint_->UnsetWhenUnsatisfiable(); + V1TopologySpreadConstraint_->UnsetLabelSelector(); + V1TopologySpreadConstraint_->UnsetMatchLabelKeys(); + V1TopologySpreadConstraint_->UnsetNodeAffinityPolicy(); + V1TopologySpreadConstraint_->UnsetNodeTaintsPolicy(); + V1TopologySpreadConstraint_->SetMaxSkew(1); + V1TopologySpreadConstraint_->SetMinDomains(1); + V1TopologySpreadConstraint_->SetTopologyKey("zone"); + V1TopologySpreadConstraint_->SetWhenUnsatisfiable("DoNotSchedule"); + auto labelSelector = std::make_shared(); + V1TopologySpreadConstraint_->SetLabelSelector(labelSelector); + V1TopologySpreadConstraint_->SetMatchLabelKeys({"test"}); + V1TopologySpreadConstraint_->SetNodeAffinityPolicy("test"); + V1TopologySpreadConstraint_->SetNodeTaintsPolicy("test"); + auto json = V1TopologySpreadConstraint_->ToJson(); + auto topologySpreadConstraint = std::make_shared(); + auto result = topologySpreadConstraint->FromJson(json); + EXPECT_TRUE(result); + EXPECT_EQ(1, topologySpreadConstraint->GetMaxSkew()); + EXPECT_TRUE(topologySpreadConstraint->MaxSkewIsSet()); + EXPECT_EQ(1, topologySpreadConstraint->GetMinDomains()); + EXPECT_TRUE(topologySpreadConstraint->MinDomainsIsSet()); + EXPECT_EQ("zone", topologySpreadConstraint->GetTopologyKey()); + EXPECT_TRUE(topologySpreadConstraint->TopologyKeyIsSet()); + EXPECT_EQ("DoNotSchedule", topologySpreadConstraint->GetWhenUnsatisfiable()); + EXPECT_TRUE(topologySpreadConstraint->WhenUnsatisfiableIsSet()); + EXPECT_TRUE(topologySpreadConstraint->LabelSelectorIsSet()); + EXPECT_TRUE(topologySpreadConstraint->MatchLabelKeysIsSet()); + EXPECT_EQ("test", topologySpreadConstraint->GetNodeAffinityPolicy()); + EXPECT_TRUE(topologySpreadConstraint->NodeAffinityPolicyIsSet()); + EXPECT_EQ("test", topologySpreadConstraint->GetNodeTaintsPolicy()); + EXPECT_TRUE(topologySpreadConstraint->NodeTaintsPolicyIsSet()); +} + +TEST_F(KubeClientModelTest, V1LeaseTest) +{ + auto v1LeaseSpec = std::make_shared(); + v1LeaseSpec->UnsetAcquireTime(); + v1LeaseSpec->UnsetHolderIdentity(); + v1LeaseSpec->UnsetLeaseDurationSeconds(); + v1LeaseSpec->UnsetLeaseTransitions(); + v1LeaseSpec->UnsetRenewTime(); + v1LeaseSpec->SetAcquireTime(utility::Datetime()); + v1LeaseSpec->SetHolderIdentity("test"); + v1LeaseSpec->SetLeaseDurationSeconds(10); + v1LeaseSpec->SetRenewTime(utility::Datetime()); + v1LeaseSpec->SetLeaseTransitions(10); + auto json = v1LeaseSpec->ToJson(); + auto newLeaseSpec = std::make_shared(); + auto result = newLeaseSpec->FromJson(json); + EXPECT_TRUE(result); + auto v1Lease = std::make_shared(); + v1Lease->UnsetKind(); + v1Lease->UnsetApiVersion(); + v1Lease->UnsetMetadata(); + v1Lease->UnsetSpec(); + v1Lease->SetKind("V1Lease"); + v1Lease->SetApiVersion("v1"); + v1Lease->SetSpec(newLeaseSpec); + auto v1ObjectMeta = std::make_shared(); + v1Lease->SetMetadata(v1ObjectMeta); + json = v1Lease->ToJson(); + auto newV1Lease = std::make_shared(); + result = newV1Lease->FromJson(json); + EXPECT_TRUE(result); + EXPECT_TRUE(newV1Lease->KindIsSet()); + EXPECT_TRUE(newV1Lease->SpecIsSet()); + EXPECT_TRUE(newV1Lease->ApiVersionIsSet()); + EXPECT_TRUE(newV1Lease->MetadataIsSet()); + EXPECT_EQ("V1Lease", newV1Lease->GetKind()); + EXPECT_EQ("v1", newV1Lease->GetApiVersion()); + EXPECT_EQ("test", newV1Lease->GetSpec()->GetHolderIdentity()); + EXPECT_EQ("", newV1Lease->GetSpec()->GetRenewTime().ToString()); + EXPECT_EQ("", newV1Lease->GetSpec()->GetAcquireTime().ToString()); + EXPECT_EQ(10, newV1Lease->GetSpec()->GetLeaseDurationSeconds()); + EXPECT_EQ(10, newV1Lease->GetSpec()->GetLeaseTransitions()); +} +} // namespace functionsystem::kube_client::test diff --git a/functionsystem/tests/unit/common/leader/leader_test.cpp b/functionsystem/tests/unit/common/leader/leader_test.cpp index 663b4920ca21c6c36c1c46e6de8e77a31d9a3003..ff3ba445db85dff1229e37af09b7b7171d25cbbb 100644 --- a/functionsystem/tests/unit/common/leader/leader_test.cpp +++ b/functionsystem/tests/unit/common/leader/leader_test.cpp @@ -22,11 +22,14 @@ #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" #include "common/explorer/etcd_explorer_actor.h" +#include "common/explorer/k8s_explorer_actor.h" #include "common/explorer/explorer.h" #include "common/leader/etcd_leader_actor.h" -#include "logs/logging.h" -#include "metadata/metadata.h" +#include "common/leader/k8s_leader_actor.h" +#include "common/logs/logging.h" +#include "common/metadata/metadata.h" #include "common/utils/generate_message.h" +#include "mocks/mock_kube_client.h" #include "mocks/mock_meta_store_client.h" #include "mocks/mock_scheduler.h" #include "utils/future_test_helper.h" @@ -403,4 +406,193 @@ TEST_F(LeaderTest, RepeatElect) litebus::Terminate(leaderActor->GetAID()); litebus::Await(leaderActor->GetAID()); } + +/** + * Feature: K8s Election + * Description: elect without leader + * 1. create 3 candidate wait to elect + * 2. invoke elect + * Expectation: one become leader, other become candidate + */ +TEST_F(LeaderTest, K8sMultiElectWithoutLeader) +{ + auto mockClient1 = std::make_shared(); + explorer::ElectionInfo electionInfo1{.identity = "proposal001", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto candidateLeader1 = std::make_shared("function-master", electionInfo1, mockClient1, "default"); + explorer::ElectionInfo electionInfo2{.identity = "proposal002", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient2 = std::make_shared(); + auto candidateLeader2 = std::make_shared("function-master", electionInfo2, mockClient2, "default"); + explorer::ElectionInfo electionInfo3{.identity = "proposal003", .mode = K8S_ELECTION_MODE,.electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient3 = std::make_shared(); + auto candidateLeader3 = std::make_shared("function-master", electionInfo3, mockClient3, "default"); + litebus::Promise> errPromise; + errPromise.SetFailed(404); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillOnce(testing::Return(errPromise.GetFuture())).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal002"))); + EXPECT_CALL(*mockClient2, ReadNamespacedLease).WillOnce(testing::Return(errPromise.GetFuture())).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal002"))); + EXPECT_CALL(*mockClient3, ReadNamespacedLease).WillOnce(testing::Return(errPromise.GetFuture())).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal002"))); + EXPECT_CALL(*mockClient1, CreateNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient2, CreateNamespacedLease).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal002"))); + EXPECT_CALL(*mockClient3, CreateNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient2, ReplaceNamespacedLease).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal002"))); + litebus::Spawn(candidateLeader1); + litebus::Spawn(candidateLeader2); + litebus::Spawn(candidateLeader3); + litebus::Async(candidateLeader1->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader2->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader3->GetAID(), &K8sLeaderActor::Elect); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader1->GetObservedRecord() != nullptr; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader2->GetObservedRecord() != nullptr; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader3->GetObservedRecord() != nullptr; }); + EXPECT_EQ("proposal002", candidateLeader1->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal002", candidateLeader2->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal002", candidateLeader3->GetObservedRecord()->holderIdentity); + litebus::Terminate(candidateLeader1->GetAID()); + litebus::Await(candidateLeader1->GetAID()); + litebus::Terminate(candidateLeader2->GetAID()); + litebus::Await(candidateLeader2->GetAID()); + litebus::Terminate(candidateLeader3->GetAID()); + litebus::Await(candidateLeader3->GetAID()); +} + +/** + * Feature: K8s Election + * Description: elect with empty leader + * 1. create 3 candidate wait to elect + * 2. invoke elect + * Expectation: one become leader, other become candidate + */ +TEST_F(LeaderTest, K8sMultiElectWithEmptyLeader) +{ + auto mockClient1 = std::make_shared(); + explorer::ElectionInfo electionInfo1{.identity = "proposal001", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto candidateLeader1 = std::make_shared("function-master", electionInfo1, mockClient1, "default"); + explorer::ElectionInfo electionInfo2{.identity = "proposal002", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient2 = std::make_shared(); + auto candidateLeader2 = std::make_shared("function-master", electionInfo2, mockClient2, "default"); + explorer::ElectionInfo electionInfo3{.identity = "proposal003", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient3 = std::make_shared(); + auto candidateLeader3 = std::make_shared("function-master", electionInfo3, mockClient3, "default"); + litebus::Promise> errPromise; + errPromise.SetFailed(400); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease(""))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient2, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease(""))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient3, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease(""))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient1, ReplaceNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient2, ReplaceNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient3, ReplaceNamespacedLease).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + litebus::Spawn(candidateLeader1); + litebus::Spawn(candidateLeader2); + litebus::Spawn(candidateLeader3); + litebus::Async(candidateLeader1->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader2->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader3->GetAID(), &K8sLeaderActor::Elect); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader1->GetObservedRecord() != nullptr && candidateLeader1->GetObservedRecord()->holderIdentity != ""; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader2->GetObservedRecord() != nullptr && candidateLeader2->GetObservedRecord()->holderIdentity != ""; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader3->GetObservedRecord() != nullptr && candidateLeader3->GetObservedRecord()->holderIdentity != ""; }); + EXPECT_EQ("proposal003", candidateLeader1->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal003", candidateLeader2->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal003", candidateLeader3->GetObservedRecord()->holderIdentity); + litebus::Terminate(candidateLeader1->GetAID()); + litebus::Await(candidateLeader1->GetAID()); + litebus::Terminate(candidateLeader2->GetAID()); + litebus::Await(candidateLeader2->GetAID()); + litebus::Terminate(candidateLeader3->GetAID()); + litebus::Await(candidateLeader3->GetAID()); +} + +/** + * Feature: K8s Election + * Description: elect without leader is overdue + * 1. create 3 candidate wait to elect + * 2. invoke elect + * Expectation: one become leader, other become candidate + */ +TEST_F(LeaderTest, K8sMultiElectWithLeaderOverdue) +{ + auto mockClient1 = std::make_shared(); + explorer::ElectionInfo electionInfo1{.identity = "proposal001", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300, }; + auto candidateLeader1 = std::make_shared("function-master", electionInfo1, mockClient1, "default"); + explorer::ElectionInfo electionInfo2{.identity = "proposal002", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient2 = std::make_shared(); + auto candidateLeader2 = std::make_shared("function-master", electionInfo2, mockClient2, "default"); + explorer::ElectionInfo electionInfo3{.identity = "proposal003", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval= 300,}; + auto mockClient3 = std::make_shared(); + auto candidateLeader3 = std::make_shared("function-master", electionInfo3, mockClient3, "default"); + litebus::Promise> errPromise; + errPromise.SetFailed(400); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease("proposal004", "2000-01-01 00:00:00", "2000-01-01 00:00:00"))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient2, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease("proposal004", "2000-01-01 00:00:00", "2000-01-01 00:00:00"))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient3, ReadNamespacedLease).WillOnce(testing::Return(MockKubeClient::CreateLease("proposal004", "2000-01-01 00:00:00", "2000-01-01 00:00:00"))).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + EXPECT_CALL(*mockClient1, ReplaceNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient2, ReplaceNamespacedLease).WillRepeatedly(testing::Return(errPromise.GetFuture())); + EXPECT_CALL(*mockClient3, ReplaceNamespacedLease).WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal003"))); + litebus::Spawn(candidateLeader1); + litebus::Spawn(candidateLeader2); + litebus::Spawn(candidateLeader3); + litebus::Async(candidateLeader1->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader2->GetAID(), &K8sLeaderActor::Elect); + litebus::Async(candidateLeader3->GetAID(), &K8sLeaderActor::Elect); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader1->GetObservedRecord() != nullptr && candidateLeader1->GetObservedRecord()->holderIdentity != "proposal004"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader2->GetObservedRecord() != nullptr && candidateLeader2->GetObservedRecord()->holderIdentity != "proposal004"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader3->GetObservedRecord() != nullptr && candidateLeader3->GetObservedRecord()->holderIdentity != "proposal004"; }); + EXPECT_EQ("proposal003", candidateLeader1->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal003", candidateLeader2->GetObservedRecord()->holderIdentity); + EXPECT_EQ("proposal003", candidateLeader3->GetObservedRecord()->holderIdentity); + litebus::Terminate(candidateLeader1->GetAID()); + litebus::Await(candidateLeader1->GetAID()); + litebus::Terminate(candidateLeader2->GetAID()); + litebus::Await(candidateLeader2->GetAID()); + litebus::Terminate(candidateLeader3->GetAID()); + litebus::Await(candidateLeader3->GetAID()); +} + +/** + * Feature: K8s Election + * Description: elect when other leader is valid + * 1. create 1 candidate wait to elect + * 2. invoke elect + * Expectation: elect failed + */ +TEST_F(LeaderTest, K8sElectCandidateFail) +{ + auto mockClient1 = std::make_shared(); + explorer::ElectionInfo electionInfo1{.identity = "proposal001", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, }; + auto candidateLeader1 = std::make_shared("function-master", electionInfo1, mockClient1, "default"); + litebus::Promise> promise; + promise.SetValue(MockKubeClient::CreateLease("proposal003")); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillRepeatedly(testing::Return(promise.GetFuture())); + litebus::Spawn(candidateLeader1); + litebus::Async(candidateLeader1->GetAID(), &K8sLeaderActor::Elect); + ASSERT_AWAIT_TRUE([&]() -> bool { return candidateLeader1->GetObservedRecord() != nullptr; }); + EXPECT_EQ("proposal003", candidateLeader1->GetObservedRecord()->holderIdentity); + litebus::Terminate(candidateLeader1->GetAID()); + litebus::Await(candidateLeader1->GetAID()); +} + +/** + * Feature: K8s Election + * Description: elect renew lease + * 1. create 3 candidate wait to elect + * 2. invoke elect + * Expectation: one become leader, other become candidate + */ +TEST_F(LeaderTest, K8sElectRenewLease) +{ + auto mockClient1 = std::make_shared(); + explorer::ElectionInfo electionInfo1{.identity = "proposal001", .mode = K8S_ELECTION_MODE, .electKeepAliveInterval = 1, .electLeaseTTL = 300, .electRenewInterval = 1 }; + auto candidateLeader1 = std::make_shared("function-master", electionInfo1, mockClient1, "default"); + litebus::Promise> promise; + promise.SetValue(MockKubeClient::CreateLease("proposal001")); + EXPECT_CALL(*mockClient1, ReadNamespacedLease).WillRepeatedly(testing::Return(promise.GetFuture())); + litebus::Future> body1; + EXPECT_CALL(*mockClient1, ReplaceNamespacedLease) + .WillOnce(testing::DoAll(test::FutureArg<2>(&body1), testing::Return(MockKubeClient::CreateLease("proposal001")))) + .WillRepeatedly(testing::Return(MockKubeClient::CreateLease("proposal001"))); + litebus::Spawn(candidateLeader1); + litebus::Async(candidateLeader1->GetAID(), &K8sLeaderActor::Elect); + ASSERT_AWAIT_READY(body1); + litebus::Terminate(candidateLeader1->GetAID()); + litebus::Await(candidateLeader1->GetAID()); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/leader/txn_leader_test.cpp b/functionsystem/tests/unit/common/leader/txn_leader_test.cpp index 0b553ccbea2d6ee2dc2ca8a1f3d4d1c0ab8a0597..e077cc9ae403c698b6c42453d52caf162cb1954d 100644 --- a/functionsystem/tests/unit/common/leader/txn_leader_test.cpp +++ b/functionsystem/tests/unit/common/leader/txn_leader_test.cpp @@ -22,7 +22,7 @@ #include "common/etcd_service/etcd_service_driver.h" #include "common/explorer/explorer.h" #include "common/leader/txn_leader_actor.h" -#include "metadata/metadata.h" +#include "common/metadata/metadata.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -34,14 +34,14 @@ class TxnLeaderTest : public ::testing::Test { public: void SetUp() override { - EXPECT_AWAIT_READY(metaStoreClient_->Delete("/", { .prevKv = false, .prefix = true })); + EXPECT_AWAIT_READY(metaStoreClient_->Delete("/", { .prefix = true })); } void TearDown() override { } - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -51,7 +51,7 @@ public: metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost }, {}, {}); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { metaStoreClient_ = nullptr; @@ -129,8 +129,10 @@ TEST_F(TxnLeaderTest, TxnLeader_Fail_Test) actor->leaseID_ = 0; // mock grant success litebus::Async(aid, &TxnLeaderActor::KeepAlive, -1); - auto result = litebus::Async(aid, &TxnLeaderActor::Sync); + auto getResponse = metaStoreClient_->Get(DEFAULT_MASTER_ELECTION_KEY, {}); + ASSERT_AWAIT_READY(getResponse); + auto result = litebus::Async(aid, &TxnLeaderActor::Sync, getResponse.Get()); ASSERT_AWAIT_READY(result); EXPECT_TRUE(result.Get().status.IsOk()); diff --git a/functionsystem/tests/unit/common/meta_store_adapter/instance_operator_test.cpp b/functionsystem/tests/unit/common/meta_store_adapter/instance_operator_test.cpp index 526dfa92bdcec008645accdb39952f8713a341d8..06c205f8555a419b326b2f58fe98e75f12ded71f 100644 --- a/functionsystem/tests/unit/common/meta_store_adapter/instance_operator_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_adapter/instance_operator_test.cpp @@ -23,7 +23,7 @@ #include "common/etcd_service/etcd_service_driver.h" #include "meta_store_monitor/meta_store_monitor_factory.h" #include "meta_store_client/meta_store_struct.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "utils/future_test_helper.h" #include "utils/grpc_client_helper.h" #include "utils/port_helper.h" @@ -45,7 +45,7 @@ protected: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -53,7 +53,7 @@ protected: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); ASSERT_TRUE(client->Delete(INSTANCE_ROUTE_PATH_PREFIX, DeleteOption{ .prevKv = false, .prefix = true }) @@ -119,15 +119,19 @@ TEST_F(InstanceOperatorTest, CreateInstanceExist) auto fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); EXPECT_AWAIT_READY(fut); + fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); + EXPECT_AWAIT_READY(fut); + EXPECT_TRUE(fut.Get().status.IsOk()); + const std::string value2 = R"({"instanceID":"0ee7cafc-93b9-4be3-ae01-000000000075","requestID":"job-3d8f88d4-task-daf90ea7-f29e-4c9e-ada4-b11cea549201-694d1ff7031c-0","functionProxyID":"siaphis12332-22737"})"; instancePutInfo->value = value2; - routePutInfo->value = value2; fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); EXPECT_AWAIT_READY(fut); EXPECT_TRUE(fut.Get().status.IsError()); instancePutInfo->key = instanceKey; + routePutInfo->value = value2; fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); EXPECT_AWAIT_READY(fut); EXPECT_TRUE(fut.Get().status.IsError()); @@ -139,6 +143,30 @@ TEST_F(InstanceOperatorTest, CreateInstanceExist) EXPECT_TRUE(fut.Get().status.IsError()); } +TEST_F(InstanceOperatorTest, CreateRouteExist) +{ + InstanceOperator instanceOpt(metaStoreClient_); + + const std::string value1 = + R"({"instanceID":"0ee7cafc-93b9-4be3-ae01-000000000075","requestID":"job-3d8f88d4-task-daf90ea7-f29e-4c9e-ada4-b11cea549201-694d1ff7031c-0","functionProxyID":"siaphis12332-22736"})"; + std::shared_ptr instancePutInfo = std::make_shared(key, value1); + std::shared_ptr routePutInfo = std::make_shared(routeKey, value1); + auto fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); + EXPECT_AWAIT_READY(fut); + + const std::string value2 = + R"({"instanceID":"0ee7cafc-93b9-4be3-ae01-000000000075","requestID":"job-3d8f88d4-task-daf90ea7-f29e-4c9e-ada4-b11cea549201-694d1ff7031c-0","functionProxyID":"siaphis12332-22737"})"; + instancePutInfo->key = "fake_key"; + routePutInfo->value = value2; + fut = instanceOpt.Create(instancePutInfo, routePutInfo, false); + EXPECT_AWAIT_READY(fut); + EXPECT_TRUE(fut.Get().status.IsError()); + EXPECT_EQ(fut.Get().status.StatusCode(), StatusCode::INSTANCE_TRANSACTION_WRONG_VERSION); + + InstanceInfo instanceInfoSaved; + EXPECT_TRUE(TransToInstanceInfoFromJson(instanceInfoSaved, fut.Get().value)); +} + TEST_F(InstanceOperatorTest, CreateInstanceETCDUnavailable) { auto helper = GrpcClientHelper(10); diff --git a/functionsystem/tests/unit/common/meta_store_adapter/meta_store_monitor_test.cpp b/functionsystem/tests/unit/common/meta_store_adapter/meta_store_monitor_test.cpp index 075324da1e0cca4854346834e5549ebebea06675..b7c8744af20b734da20dfcea69acda8884b945e5 100644 --- a/functionsystem/tests/unit/common/meta_store_adapter/meta_store_monitor_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_adapter/meta_store_monitor_test.cpp @@ -24,7 +24,7 @@ namespace functionsystem::test { class MetaStoreMonitorTest : public ::testing::Test { protected: - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { int metaStoreServerPort = functionsystem::test::FindAvailablePort(); metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); diff --git a/functionsystem/tests/unit/common/meta_store_client/election_client_strategy_test.cpp b/functionsystem/tests/unit/common/meta_store_client/election_client_strategy_test.cpp index 5186e380bcef9c80ab2d83f4b1c2b7f9694a6357..c0a7a3a61d2f30e3d90978e7b22faffa695abfaf 100644 --- a/functionsystem/tests/unit/common/meta_store_client/election_client_strategy_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/election_client_strategy_test.cpp @@ -143,13 +143,13 @@ class ElectionClientStrategyTest : public ::testing::Test { public: ElectionClientStrategyTest() = default; ~ElectionClientStrategyTest() override = default; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { mockElectionService_ = std::make_shared(); litebus::Spawn(mockElectionService_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { litebus::Terminate(mockElectionService_->GetAID()); litebus::Await(mockElectionService_->GetAID()); diff --git a/functionsystem/tests/unit/common/meta_store_client/kv_client_strategy_test.cpp b/functionsystem/tests/unit/common/meta_store_client/kv_client_strategy_test.cpp index 7ad7d51f687817bbb6625cf1db27befc38c3b022..691cd1c5ef3407867ee0ee7be84cbddb77977df6 100644 --- a/functionsystem/tests/unit/common/meta_store_client/kv_client_strategy_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/kv_client_strategy_test.cpp @@ -57,7 +57,7 @@ class EtcdKvClientStrategyTest : public ::testing::Test { public: EtcdKvClientStrategyTest() = default; ~EtcdKvClientStrategyTest() override = default; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -65,7 +65,7 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -105,7 +105,7 @@ class MetaStoreKvClientStrategyTest : public ::testing::Test { public: MetaStoreKvClientStrategyTest() = default; ~MetaStoreKvClientStrategyTest() override = default; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -113,7 +113,7 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -272,12 +272,13 @@ void TransactionTxn(const std::shared_ptr &client) EXPECT_EQ(std::get(txnResponse->responses[0].response).prevKvs[0].key(), "llt/sn/workers/xxx"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.key(), "llt/sn/workers/yyy"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.value(), "1.0"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(2)); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(3)); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].key(), "llt/sn/workers/zzz"); EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); } -void TransactionWithReqTxn(const std::shared_ptr &client){ +void TransactionWithReqTxn(const std::shared_ptr &client) +{ auto transaction = std::make_unique(client->GetAID()); transaction->If(TxnCompare::OfValue("llt/sn/workers/xxx", CompareOperator::EQUAL, "1.0")); DeleteOption delOption = { true, false }; @@ -305,7 +306,7 @@ void TransactionWithReqTxn(const std::shared_ptr & EXPECT_EQ(std::get(txnResponse->responses[0].response).prevKvs[0].key(), "llt/sn/workers/xxx"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.key(), "llt/sn/workers/yyy"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.value(), "1.0"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(2)); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(3)); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].key(), "llt/sn/workers/zzz"); EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); } @@ -342,15 +343,17 @@ void TransactionTxnElse(const std::shared_ptr &cli EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKvs.size(), static_cast(1)); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKvs[0].key(), "llt/sn/workers/yyy"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKvs[0].value(), "1.0"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(2)); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(3)); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].key(), "llt/sn/workers/zzz"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "2.0"); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ } void WatchTest(const std::shared_ptr &client) { auto observer = [](const std::vector &events, bool) -> bool { return true; }; - auto syncer = []() -> litebus::Future { return SyncResult{ Status::OK(), 0 }; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; auto watcher = client->Watch("llt/sn/workers", option, observer, syncer, nullptr); ASSERT_AWAIT_READY(watcher); @@ -378,7 +381,9 @@ void GetAndWatchTest(const std::shared_ptr &client) auto putResponse = client->Put("llt/sn/workers/vvv", "2.0", putOption).Get(); // await EXPECT_EQ(putResponse->status, Status::OK()); } - auto syncer = []() -> litebus::Future { return SyncResult{ Status::OK(), 0 }; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; auto watcher = client->GetAndWatch("llt/sn/workers", option, observer, syncer, nullptr).Get(); // await ASSERT_AWAIT_READY(promise.GetFuture()); @@ -389,6 +394,144 @@ void GetAndWatchTest(const std::shared_ptr &client) }); // wait for cancel success } +void GetAndWatchWithHandlerTest(const std::shared_ptr &client) +{ + litebus::Promise promise; + ObserverFunction observer = [&](const std::vector &events, bool synced) -> bool { + EXPECT_EQ(synced, true); + EXPECT_EQ(events.size(), static_cast(4)); + EXPECT_EQ(events[0].eventType, EVENT_TYPE_PUT); + EXPECT_EQ(events[0].kv.key(), "llt/sn/workers/vvv"); + EXPECT_EQ(events[0].kv.value(), "2.0"); + promise.SetValue(true); + return true; + }; + { + PutOption putOption = { .leaseId = 0, .prevKv = false }; + auto putResponse = client->Put("llt/sn/workers/vvv", "1.0", putOption).Get(); // await + EXPECT_EQ(putResponse->status, Status::OK()); + } + { + PutOption putOption = { .leaseId = 0, .prevKv = false }; + auto putResponse = client->Put("llt/sn/workers/vvv", "2.0", putOption).Get(); // await + EXPECT_EQ(putResponse->status, Status::OK()); + } + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; + + bool handlerRes = false; + auto handler = [&](const std::shared_ptr &response) -> litebus::Future { + handlerRes = response->kvs.size() != 0; + return Status::OK(); + }; + + WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; + GetAndWatchHandlers handlers{ .observer = observer, .syncer = syncer, .responseHandler = handler }; + auto watcher = client->GetAndWatchWithHandler("llt/sn/workers", option, handlers, nullptr).Get(); // await + ASSERT_AWAIT_READY(promise.GetFuture()); + + EXPECT_TRUE(handlerRes); + watcher->Close(); + ASSERT_AWAIT_TRUE([&]() -> bool { + return (client->readyRecords_.find(watcher->GetWatchId()) == client->readyRecords_.end()); + }); // wait for cancel success +} + +void UnhealthyTest(const std::shared_ptr &client) +{ + client->OnHealthyStatus(Status(StatusCode::FAILED)); + { + PutOption op = { .leaseId = 0, .prevKv = false }; + auto response = client->Put("llt/sn/workers/xxx", "2.0", op).Get(); + EXPECT_TRUE(response->status.IsError()); + } + + { + DeleteOption delOp = { false, false }; + auto response = client->Delete("llt/sn/workers/xxx", delOp).Get(); + EXPECT_TRUE(response->status.IsError()); + } + + { + GetOption op = { false, false, false, 0, SortOrder::DESCEND, SortTarget::MODIFY }; + auto response = client->Get("llt/sn/workers/xxx", op).Get(); + EXPECT_TRUE(response->status.IsError()); + } + client->OnHealthyStatus(Status::OK()); +} + +void UnhealthyTxnTest(const std::shared_ptr &client) +{ + client->OnHealthyStatus(Status(StatusCode::FAILED)); + auto transaction = std::make_unique(client->GetAID()); + transaction->If(TxnCompare::OfValue("llt/sn/workers/xxx", CompareOperator::EQUAL, "1.0")); + DeleteOption delOption = { true, false }; + transaction->Then(TxnOperation::Create("llt/sn/workers/xxx", delOption)); + GetOption getOption = { true, false, false, 0, SortOrder::DESCEND, SortTarget::KEY }; + transaction->Else(TxnOperation::Create("llt/sn/workers/", getOption)); + std::shared_ptr txnResponse = transaction->Commit().Get(); + EXPECT_NE(txnResponse, nullptr); + EXPECT_TRUE(txnResponse->status.IsError()); + + client->OnHealthyStatus(Status::OK()); +} + +void UnhealthyTxnWithReqTest(const std::shared_ptr &client) +{ + client->OnHealthyStatus(Status(StatusCode::FAILED)); + auto transaction = std::make_unique(client->GetAID()); + transaction->If(TxnCompare::OfValue("llt/sn/workers/xxx", CompareOperator::EQUAL, "1.0")); + DeleteOption delOption = { true, false }; + transaction->Then(TxnOperation::Create("llt/sn/workers/xxx", delOption)); + transaction->Else(TxnOperation::Create("llt/sn/workers/", delOption)); + + ::etcdserverpb::TxnRequest request; + client->BuildTxnRequest(request, transaction->compares, transaction->thenOps, transaction->elseOps); + + auto response = client->CommitWithReq(request, true); + ASSERT_AWAIT_READY(response); + + EXPECT_TRUE(!response.Get()->has_header()); + client->OnHealthyStatus(Status::OK()); +} + +void OnUnhealthyTest(const std::shared_ptr &client) +{ + auto putFuture = client->Put("llt/sn/workers/xxx", "2.0", { .leaseId = 0, .prevKv = false }); + + DeleteOption delOp = { false, false }; + auto delFuture = client->Delete("llt/sn/workers/xxx", delOp); + + GetOption getOp = { false, false, false, 0, SortOrder::DESCEND, SortTarget::MODIFY }; + auto getFuture = client->Get("llt/sn/workers/xxx", getOp); + + auto transaction = std::make_unique(client->GetAID()); + transaction->If(TxnCompare::OfValue("llt/sn/workers/xxx", CompareOperator::EQUAL, "1.0")); + DeleteOption delOption = { true, false }; + transaction->Then(TxnOperation::Create("llt/sn/workers/xxx", delOption)); + transaction->Else(TxnOperation::Create("llt/sn/workers/", delOption)); + + ::etcdserverpb::TxnRequest request; + client->BuildTxnRequest(request, transaction->compares, transaction->thenOps, transaction->elseOps); + + auto response = client->CommitWithReq(request, true); + + auto txnResponse = transaction->Commit(); + + client->OnHealthyStatus(Status(StatusCode::FAILED)); + + ASSERT_AWAIT_READY(txnResponse); + + EXPECT_TRUE(!response.Get()->has_header()); + EXPECT_TRUE(txnResponse.Get()->status.IsError()); + EXPECT_TRUE(getFuture.Get()->status.IsError()); + EXPECT_TRUE(delFuture.Get()->status.IsError()); + EXPECT_TRUE(putFuture.Get()->status.IsError()); + + client->OnHealthyStatus(Status::OK()); +} + TEST_F(EtcdKvClientStrategyTest, EtcdCancelTest) { auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); @@ -402,15 +545,15 @@ TEST_F(EtcdKvClientStrategyTest, EtcdCancelTest) litebus::Promise promise; ObserverFunction observer = [&](const std::vector &events, bool synced) -> bool { return true; }; litebus::Promise promise1; - auto syncer1 = [&]() -> litebus::Future { + auto syncer1 = [&](const std::shared_ptr &) -> litebus::Future { promise1.SetValue(true); - return SyncResult{ Status::OK(), 1000 + 1 }; + return SyncResult{ Status::OK() }; }; litebus::Promise promise2; - auto syncer2 = [&]() -> litebus::Future { + auto syncer2 = [&](const std::shared_ptr &) -> litebus::Future { promise2.SetValue(false); - return SyncResult{ Status(StatusCode::FAILED, "mock sync failed"), 0 }; + return SyncResult{ Status(StatusCode::FAILED, "mock sync failed") }; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; @@ -436,19 +579,17 @@ TEST_F(EtcdKvClientStrategyTest, EtcdCancelTest) ASSERT_AWAIT_READY(promise2.GetFuture()); ASSERT_FALSE(promise2.GetFuture().Get()); - EXPECT_EQ(watcher1->GetWatchId(), -1); - EXPECT_EQ(watcher2->GetWatchId(), -1); + ASSERT_AWAIT_TRUE([&]() { return watcher1->GetWatchId() == 2; }); + ASSERT_AWAIT_TRUE([&]() { return watcher2->GetWatchId() == 3; }); sleep(1); // watch for reconnect successfully auto records = client->GetRecords(); - EXPECT_EQ(records[0]->option.revision, 1001); + EXPECT_EQ(records[0]->option.revision, 4); EXPECT_GT(records[1]->option.revision, 0); - // test cancel event revision is samll than watcher revision, go to reconnect branch rsp->set_watch_id(records[0]->watcher->GetWatchId()); cancelStatus = client->Cancel(rsp); EXPECT_EQ(cancelStatus, Status::OK()); - EXPECT_EQ(cancelStatus.GetMessage(), "[try to reconnect all watcher]"); litebus::Terminate(client->GetAID()); litebus::Await(client); @@ -533,11 +674,14 @@ TEST_F(EtcdKvClientStrategyTest, WatchTest) // NOLINT auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); auto kvClientActor = std::make_shared("KvClientActor_" + uuid.ToString(), metaStoreServerHost_, metaStoreTimeoutOpt); - bool ret = kvClientActor->ReconnectWatch(); + litebus::Spawn(kvClientActor); + bool ret = kvClientActor->ReconnectGrpcWatch().Get(); EXPECT_TRUE(ret); bool canceled = kvClientActor->TryErr(); EXPECT_TRUE(canceled); + litebus::Terminate(kvClientActor->GetAID()); + litebus::Await(kvClientActor->GetAID()); } TEST_F(EtcdKvClientStrategyTest, GetAndWatchTest) // NOLINT @@ -545,6 +689,41 @@ TEST_F(EtcdKvClientStrategyTest, GetAndWatchTest) // NOLINT GetAndWatchTest(client_); } +TEST_F(EtcdKvClientStrategyTest, UnhealthyTest) // NOLINT +{ + UnhealthyTest(client_); +} + +TEST_F(EtcdKvClientStrategyTest, UnhealthyTxnTest) // NOLINT +{ + + auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); + auto client = std::make_shared( + "KvClientActor_" + uuid.ToString(), metaStoreServerHost_, metaStoreTimeoutOpt, sslConfig); + litebus::Spawn(client); + UnhealthyTxnTest(client); + UnhealthyTxnWithReqTest(client); + litebus::Terminate(client->GetAID()); + litebus::Await(client); +} + +TEST_F(EtcdKvClientStrategyTest, UnhealthyTxnWithReqTest) // NOLINT +{ + + auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); + auto client = std::make_shared( + "KvClientActor_" + uuid.ToString(), metaStoreServerHost_, metaStoreTimeoutOpt, sslConfig); + litebus::Spawn(client); + UnhealthyTxnWithReqTest(client); + litebus::Terminate(client->GetAID()); + litebus::Await(client); +} + +TEST_F(EtcdKvClientStrategyTest, GetAndWatchWithHandlerTest) // NOLINT +{ + GetAndWatchWithHandlerTest(client_); +} + TEST_F(MetaStoreKvClientStrategyTest, PutTest) // NOLINT { PutTest(client_); @@ -573,6 +752,10 @@ TEST_F(MetaStoreKvClientStrategyTest, DeleteKeyValuePrevPrefix) // NOLINT TEST_F(MetaStoreKvClientStrategyTest, GetTest) // NOLINT { GetTest(client_); + + client_->OnHealthyStatus(Status(META_STORE_BACK_UP_ERR, "meta-store back up failed")); + GetTest(client_); + client_->OnHealthyStatus(Status::OK()); } TEST_F(MetaStoreKvClientStrategyTest, TransactionTxn) // NOLINT @@ -616,6 +799,7 @@ TEST_F(MetaStoreKvClientStrategyTest, TransactionTxnElse) // NOLINT litebus::Spawn(client); TransactionTxnElse(client); litebus::Async(client_->GetAID(), &meta_store::MetaStoreKvClientStrategy::OnAddressUpdated, ""); + ASSERT_AWAIT_TRUE([&]() { return client_->unknownWatchers_.empty(); }); litebus::Terminate(client->GetAID()); litebus::Await(client); } @@ -629,4 +813,50 @@ TEST_F(MetaStoreKvClientStrategyTest, GetAndWatchTest) // NOLINT { GetAndWatchTest(client_); } + +TEST_F(MetaStoreKvClientStrategyTest, UnhealthyTest) // NOLINT +{ + UnhealthyTest(client_); +} + +TEST_F(MetaStoreKvClientStrategyTest, UnhealthyTxnTest) // NOLINT +{ + auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); + auto client = std::make_shared("KvClientActor_" + uuid.ToString(), + metaStoreClinetAddr_, metaStoreTimeoutOpt); + litebus::Spawn(client); + UnhealthyTxnTest(client); + litebus::Terminate(client->GetAID()); + litebus::Await(client); +} + +TEST_F(MetaStoreKvClientStrategyTest, UnhealthyTxnWithReqTest) // NOLINT +{ + auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); + auto client = std::make_shared("KvClientActor_" + uuid.ToString(), + metaStoreClinetAddr_, metaStoreTimeoutOpt); + litebus::Spawn(client); + UnhealthyTxnWithReqTest(client); + litebus::Terminate(client->GetAID()); + litebus::Await(client); +} + +TEST_F(MetaStoreKvClientStrategyTest, OnUnhealthySendTest) // NOLINT +{ + auto uuid = litebus::uuid_generator::UUID::GetRandomUUID(); + auto client = std::make_shared("KvClientActor_" + uuid.ToString(), + "127.0.0.1:39999", metaStoreTimeoutOpt); + litebus::Spawn(client); + + OnUnhealthyTest(client); + + litebus::Terminate(client->GetAID()); + litebus::Await(client); +} + + +TEST_F(MetaStoreKvClientStrategyTest, GetAndWatchWithHandlerTest) // NOLINT +{ + GetAndWatchWithHandlerTest(client_); +} } // namespace functionsystem::meta_store::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/meta_store_client/kv_client_test.cpp b/functionsystem/tests/unit/common/meta_store_client/kv_client_test.cpp index fb98f5b9ffc50aacbe8f3bb1fd4042fea5b88bb4..b1f392c3357641ffb47ad5492cc8b89b057549d5 100644 --- a/functionsystem/tests/unit/common/meta_store_client/kv_client_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/kv_client_test.cpp @@ -28,7 +28,7 @@ using namespace functionsystem::test; class KvClientTest : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdKvService_ = std::make_shared(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -50,7 +50,7 @@ public: promise.GetFuture().Get(); // wait for init } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { if (etcdServer_ != nullptr) { etcdServer_->Shutdown(); diff --git a/functionsystem/tests/unit/common/meta_store_client/maintenance_client_strategy_test.cpp b/functionsystem/tests/unit/common/meta_store_client/maintenance_client_strategy_test.cpp index 3218e14ffab4ce693be18b2dd54bb721a5f75501..42cd760993b38f7d0ec80adc06223e004e2dc2d8 100644 --- a/functionsystem/tests/unit/common/meta_store_client/maintenance_client_strategy_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/maintenance_client_strategy_test.cpp @@ -18,7 +18,8 @@ #include "async/future.hpp" #include "meta_store_client/maintenance/meta_store_maintenance_client_strategy.h" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" +#include "mocks/mock_etcd_maintenance_service.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -65,19 +66,63 @@ public: ~MaintenanceClientStrategyTest() override = default; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { mockMaintenanceService_ = std::make_shared(); litebus::Spawn(mockMaintenanceService_); + + etcdMaintenanceService_ = std::make_shared(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + etcdAddress_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + + litebus::Promise promise; + std::thread thread = std::thread([&promise]() { + ::grpc::ServerBuilder builder; + builder.RegisterService(etcdMaintenanceService_.get()); + + builder.AddListeningPort(etcdAddress_, grpc::InsecureServerCredentials()); + etcdServer_ = builder.BuildAndStart(); // start server + + promise.SetValue(true); // init done + + etcdServer_->Wait(); // quit after shutdown + etcdMaintenanceService_ = nullptr; + }); + thread.detach(); + promise.GetFuture().Get(); // wait for init } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { litebus::Terminate(mockMaintenanceService_->GetAID()); litebus::Await(mockMaintenanceService_->GetAID()); mockMaintenanceService_ = nullptr; + + if (etcdServer_ != nullptr) { + etcdServer_->Shutdown(); + etcdServer_ = nullptr; + } + } + + static void TimeLoop(int32_t timeSpan) + { + auto start = std::chrono::high_resolution_clock::now(); + while (true) { + auto now = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(now - start); + if (duration.count() >= timeSpan) { + break; + } + std::this_thread::yield(); + } } +public: + inline static std::string etcdAddress_; + inline static std::shared_ptr etcdServer_ = nullptr; + inline static std::shared_ptr etcdMaintenanceService_ = nullptr; + inline static std::string localAddress_; + protected: void SetUp() override { @@ -136,9 +181,58 @@ TEST_F(MaintenanceClientStrategyTest, ReconnectTest) uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); EXPECT_EQ(reconnected.GetFuture().Get(), "127.0.0.1:" + std::to_string(port)); + ::etcdserverpb::StatusResponse ret; + EXPECT_CALL(*mockMaintenanceService_, HealthCheck).WillRepeatedly(Return(ret)); + litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::HeartBeatTimeout); + client_->explorer_ = std::make_shared(""); + + litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::OnAddressUpdated, ""); + litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::Exited, litebus::AID()); litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::TryReconnect); litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::ReconnectSuccess); litebus::Async(client_->GetAID(), &MetaStoreMaintenanceClientStrategy::OnAddressUpdated, ""); } + +TEST_F(MaintenanceClientStrategyTest, EtcdHealehCheckTest) +{ + auto etcdMaintenanceClient = std::make_shared( + "EtcdMaintenanceClient", etcdAddress_, std::make_shared(etcdAddress_), + MetaStoreTimeoutOption{}); + + EXPECT_CALL(*etcdMaintenanceService_, Status) // test persistence + .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::StatusRequest *, + ::etcdserverpb::StatusResponse *resp) -> ::grpc::Status { + TimeLoop(1); + *resp = ::etcdserverpb::StatusResponse{}; + resp->mutable_header()->set_revision(1); + return ::grpc::Status::OK; + })); + auto status = etcdMaintenanceClient->HealthCheck(); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().status.IsOk()); + + EXPECT_CALL(*etcdMaintenanceService_, Status) // test persistence + .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::StatusRequest *, + ::etcdserverpb::StatusResponse *resp) -> ::grpc::Status { + TimeLoop(1); + *resp = ::etcdserverpb::StatusResponse{}; + resp->mutable_header()->set_revision(1); + resp->add_errors("no leader"); + return ::grpc::Status::OK; + })); + auto failedStatus = etcdMaintenanceClient->HealthCheck(); + ASSERT_AWAIT_READY(failedStatus); + EXPECT_TRUE(failedStatus.Get().status.IsError()); + + EXPECT_CALL(*etcdMaintenanceService_, Status) // test persistence + .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::StatusRequest *, + ::etcdserverpb::StatusResponse *resp) -> ::grpc::Status { + TimeLoop(1); + return ::grpc::Status::CANCELLED; + })); + auto disconnect = etcdMaintenanceClient->HealthCheck(); + ASSERT_AWAIT_READY(disconnect); + EXPECT_TRUE(disconnect.Get().status.IsError()); +} } // namespace functionsystem::meta_store::test diff --git a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_back_off_retry_test.cpp b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_back_off_retry_test.cpp index 5cc844a0b206a8f9143751dcec9aa7dfa44accfb..06d5a240147953a24808b09ef03d93db75b342fb 100644 --- a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_back_off_retry_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_back_off_retry_test.cpp @@ -20,10 +20,10 @@ #include #include -#include "logs/logging.h" +#include "common/logs/logging.h" #include "meta_store_client/meta_store_client.h" #include "meta_store_client/meta_store_struct.h" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" #include "kv_service_actor.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -104,10 +104,10 @@ class MetaStoreClientBackOffRetryTest : public ::testing::Test { public: MetaStoreClientBackOffRetryTest() = default; ~MetaStoreClientBackOffRetryTest() override = default; - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } @@ -223,7 +223,9 @@ TEST_F(MetaStoreClientBackOffRetryTest, DropFirstSeveralAttemptsAndSuccess_PutGe return true; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client->Watch("llt/sn/workers", option, observer, syncer); ASSERT_AWAIT_READY(watcher); } @@ -306,7 +308,9 @@ TEST_F(MetaStoreClientBackOffRetryTest, DropFirstSeveralAttemptsAndSuccess_Txn) return true; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client->GetAndWatch("llt/sn/workers", option, observer, syncer); ASSERT_AWAIT_READY(watcher); @@ -338,9 +342,9 @@ TEST_F(MetaStoreClientBackOffRetryTest, DropFirstSeveralAttemptsAndSuccess_Txn) static_cast(1)); EXPECT_EQ(std::get(txnResponse->responses[0].response).prevKvs[0].key(), "llt/sn/workers/xxx"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.value(), ""); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(1)); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].key(), "llt/sn/workers/yyy"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "2.0"); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(1)); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].key(), "llt/sn/workers/xxx"); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); // Txn ÖÐµÄ Range ²éѯʼÖÕ²éµÄ»º´æ cache_ } ASSERT_AWAIT_READY(putPromise.GetFuture()); diff --git a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_mgr_test.cpp b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_mgr_test.cpp index 09b880196107890edeb44d2ecfc4e3667912c90b..75817940d5d7855b634353b2f32726e0172184b8 100644 --- a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_mgr_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_mgr_test.cpp @@ -38,11 +38,8 @@ class MetaStoreClientMgrTest : public ::testing::Test { TEST_F(MetaStoreClientMgrTest, EtcdMode) { - MetaStoreConfig metaStoreConfig{ .etcdAddress = metaStoreServerHost, - .metaStoreAddress = "", - .enableMetaStore = false, - .isMetaStorePassthrough = false, - .etcdTablePrefix = "/test" }; + MetaStoreConfig metaStoreConfig{ .etcdAddress = metaStoreServerHost, .metaStoreAddress = "", + .enableMetaStore= false, .etcdTablePrefix = "/test" }; auto metaStoreClientMgr = std::make_shared(metaStoreConfig); EXPECT_EQ(metaStoreClientMgr->Init(), Status::OK()); EXPECT_NE(metaStoreClientMgr->etcdKvClient_, nullptr); @@ -105,14 +102,9 @@ TEST_F(MetaStoreClientMgrTest, MetaStoreLocalModeNoEtcd) TEST_F(MetaStoreClientMgrTest, MetaStoreLocalModeWithEtcd) { // key(/yr/pool) needed to be stored in etcd - MetaStoreConfig metaStoreConfig{ .etcdAddress = metaStoreServerHost, - .metaStoreAddress = metaStoreServerHost, - .enableMetaStore = true, - .isMetaStorePassthrough = false, - .etcdTablePrefix = "/test", - .enableAutoSync = false, - .autoSyncInterval = 0, - .excludedKeys = { "yr/pool" } }; + MetaStoreConfig metaStoreConfig{ .etcdAddress = metaStoreServerHost, .metaStoreAddress = metaStoreServerHost, + .enableMetaStore = true, .isMetaStorePassthrough = false, + .etcdTablePrefix = "/test", .excludedKeys = { "yr/pool" } }; auto metaStoreClientMgr = std::make_shared(metaStoreConfig); EXPECT_EQ(metaStoreClientMgr->Init(), Status::OK()); EXPECT_NE(metaStoreClientMgr->etcdKvClient_, nullptr); @@ -127,8 +119,8 @@ TEST_F(MetaStoreClientMgrTest, MetaStoreLocalModeWithEtcd) EXPECT_NE(metaStoreClientMgr->GetMaintenanceClient(), nullptr); EXPECT_NE(metaStoreClientMgr->GetLeaseClient(), nullptr); EXPECT_NE(metaStoreClientMgr->GetElectionClient(), nullptr); - metaStoreClientMgr->UpdateMetaStoreAddress("127.1.1.0:3334"); - EXPECT_EQ(metaStoreClientMgr->metaStoreExplorer_->Explore().Get(), "127.1.1.0:3334"); + metaStoreClientMgr->UpdateMetaStoreAddress("http://127.1.1.0:3334"); + EXPECT_EQ(metaStoreClientMgr->metaStoreExplorer_->Explore().Get(), "http://127.1.1.0:3334"); } } diff --git a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_test.cpp b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_test.cpp index a40d964035655853d9869b57bea77f78ae9737e8..6b47afebbc1e932ec4c9fe4756b21c2860a1a13b 100644 --- a/functionsystem/tests/unit/common/meta_store_client/meta_store_client_test.cpp +++ b/functionsystem/tests/unit/common/meta_store_client/meta_store_client_test.cpp @@ -25,7 +25,7 @@ #include "meta_store_client/key_value/etcd_kv_client_strategy.h" #include "meta_store_client/lease/etcd_lease_client_strategy.h" #include "meta_store_client/meta_store_struct.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "mocks/mock_etcd_election_service.h" #include "mocks/mock_etcd_lease_service.h" #include "mocks/mock_etcd_watch_service.h" @@ -55,7 +55,7 @@ protected: inline static std::string watchHost_; inline static std::string electionHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -85,12 +85,12 @@ protected: wp->GetFuture().Get(); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); if (electionServer_ != nullptr) { - electionServer_->Shutdown(); + electionServer_->Shutdown(std::chrono::system_clock::now()); electionServer_ = nullptr; } @@ -98,7 +98,7 @@ protected: leaseService_ = nullptr; if (watchServer_ != nullptr) { - watchServer_->Shutdown(); + watchServer_->Shutdown(std::chrono::system_clock::now()); watchServer_ = nullptr; } watchService_ = nullptr; @@ -120,9 +120,10 @@ protected: void SetUp() override { PutOption op = { .leaseId = 0, .prevKv = false }; - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", - .enableMetaStore= false, .isMetaStorePassthrough = false, - .etcdTablePrefix = "/test" }); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }); client.Init(); client.UpdateMetaStoreAddress(metaStoreServerHost_); client.Put("llt/sn/workers/xxx", "1.0", op).Get(); @@ -135,7 +136,10 @@ protected: { DeleteOption op = { .prevKv = false, .prefix = true }; - MetaStoreClient metastoreClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }); + MetaStoreClient metastoreClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }); metastoreClient.Init(); metastoreClient.Delete("llt/sn/workers/", op).Get(); // delete all llt } @@ -184,7 +188,7 @@ protected: TEST_F(MetaStoreClientTest, OperateEtcdFailed) // NOLINT { auto helper = GrpcClientHelper(10); - MetaStoreClient errorClient(MetaStoreConfig{ .etcdAddress = "127.0.0.1:33333", .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient errorClient(MetaStoreConfig{ .etcdAddress = "127.0.0.1:33333", .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); errorClient.Init(); PutOption op = { .leaseId = 0, .prevKv = false }; auto response = errorClient.Put("llt/sn/error/put", "2.0", op).Get(); @@ -202,7 +206,7 @@ TEST_F(MetaStoreClientTest, OperateEtcdFailed) // NOLINT TEST_F(MetaStoreClientTest, PutKeyValue) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); PutOption op = { .leaseId = 0, .prevKv = false }; auto response = client.Put("llt/sn/workers/xxx", "2.0", op).Get(); @@ -217,7 +221,7 @@ TEST_F(MetaStoreClientTest, PutKeyValue) // NOLINT TEST_F(MetaStoreClientTest, DeleteKeyValue) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); DeleteOption op = { false, false }; auto response = client.Delete("llt/sn/workers/xxx", op).Get(); @@ -228,7 +232,7 @@ TEST_F(MetaStoreClientTest, DeleteKeyValue) // NOLINT TEST_F(MetaStoreClientTest, DeleteKeyValuePrevKv) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); DeleteOption op = { .prevKv = true, .prefix = false }; auto response = client.Delete("llt/sn/workers/xxx", op).Get(); @@ -241,7 +245,7 @@ TEST_F(MetaStoreClientTest, DeleteKeyValuePrevKv) // NOLINT TEST_F(MetaStoreClientTest, DeleteKeyValuePrefix) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); DeleteOption op = { false, true }; auto response = client.Delete("llt/sn/workers/", op).Get(); @@ -252,7 +256,7 @@ TEST_F(MetaStoreClientTest, DeleteKeyValuePrefix) // NOLINT TEST_F(MetaStoreClientTest, DeleteKeyValuePrevPrefix) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); DeleteOption op = { true, true }; auto response = client.Delete("llt/sn/workers/", op).Get(); @@ -265,7 +269,7 @@ TEST_F(MetaStoreClientTest, DeleteKeyValuePrevPrefix) // NOLINT TEST_F(MetaStoreClientTest, GetKeyValue) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); GetOption op = { false, false, false, 0, SortOrder::DESCEND, SortTarget::MODIFY }; auto response = client.Get("llt/sn/workers/xxx", op).Get(); @@ -299,7 +303,7 @@ TEST_F(MetaStoreClientTest, GetKeyValue) // NOLINT TEST_F(MetaStoreClientTest, TransactionTxn) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); auto transaction = client.BeginTransaction(); @@ -329,14 +333,14 @@ TEST_F(MetaStoreClientTest, TransactionTxn) // NOLINT EXPECT_EQ(TrimKeyPrefix(std::get(txnResponse->responses[1].response).prevKv.key(), "/test"), "llt/sn/workers/yyy"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKv.value(), "1.0"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(2)); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(3)); // TxnÖеIJéѯʼÖÕ²éµÄ»º´æ cache_ EXPECT_EQ(TrimKeyPrefix(std::get(txnResponse->responses[2].response).kvs[0].key(), "/test"), "llt/sn/workers/zzz"); EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); } TEST_F(MetaStoreClientTest, TransactionTxnTest) { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); auto transaction = client.BeginTransaction(); transaction->If(TxnCompare::OfVersion("llt/sn/workers/xxx", CompareOperator::EQUAL, 1)); @@ -349,7 +353,7 @@ TEST_F(MetaStoreClientTest, TransactionTxnTest) TEST_F(MetaStoreClientTest, TransactionTxnElse) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); auto transaction = client.BeginTransaction(); @@ -380,14 +384,14 @@ TEST_F(MetaStoreClientTest, TransactionTxnElse) // NOLINT EXPECT_EQ(TrimKeyPrefix(std::get(txnResponse->responses[1].response).prevKvs[0].key(), "/test"), "llt/sn/workers/yyy"); EXPECT_EQ(std::get(txnResponse->responses[1].response).prevKvs[0].value(), "1.0"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(2)); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs.size(), static_cast(3)); // TxnÖеIJéѯʼÖÕ²éµÄ»º´æ cache_ EXPECT_EQ(TrimKeyPrefix(std::get(txnResponse->responses[2].response).kvs[0].key(), "/test"), "llt/sn/workers/zzz"); - EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "2.0"); + EXPECT_EQ(std::get(txnResponse->responses[2].response).kvs[0].value(), "1.0"); // TxnÖеIJéѯʼÖÕ²éµÄ»º´æ cache_ } TEST_F(MetaStoreClientTest, GrantLease) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); bool put = false, deleted = false; auto observer = [&](const std::vector &events, bool) -> bool { @@ -409,7 +413,9 @@ TEST_F(MetaStoreClientTest, GrantLease) // NOLINT return true; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client.Watch("llt/sn/workers", option, observer, syncer).Get(); // await ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); @@ -435,7 +441,11 @@ TEST_F(MetaStoreClientTest, GrantLease) // NOLINT TEST_F(MetaStoreClientTest, RevokeLease) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); bool put = false, deleted = false; auto observer = [&](const std::vector &events, bool) -> bool { @@ -457,7 +467,9 @@ TEST_F(MetaStoreClientTest, RevokeLease) // NOLINT return true; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client.Watch("llt/sn/workers", option, observer, syncer).Get(); // await ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); @@ -486,7 +498,11 @@ TEST_F(MetaStoreClientTest, RevokeLease) // NOLINT TEST_F(MetaStoreClientTest, KeepAliveLease) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); bool put = false, deleted = false; auto observer = [&](const std::vector &events, bool) -> bool { @@ -508,7 +524,9 @@ TEST_F(MetaStoreClientTest, KeepAliveLease) // NOLINT return true; }; WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client.Watch("llt/sn/workers", option, observer, syncer).Get(); // await ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); @@ -526,14 +544,19 @@ TEST_F(MetaStoreClientTest, KeepAliveLease) // NOLINT ASSERT_AWAIT_TRUE([&deleted]() -> bool { return deleted; }); auto helper = GrpcClientHelper(10); - MetaStoreClient invalid_client(MetaStoreConfig{ .etcdAddress = "127.0.0.1:123", .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }); + MetaStoreClient invalid_client(MetaStoreConfig{ + .etcdAddress = "127.0.0.1:123", + .metaStoreAddress = "", + .enableMetaStore= false, + .etcdTablePrefix = "/test" + }); invalid_client.Init(); EXPECT_EQ(invalid_client.KeepAliveOnce(123).Get().status.StatusCode(), StatusCode::FAILED); } TEST_F(MetaStoreClientTest, CreateWatcher) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); int count = 0; auto observer = @@ -564,7 +587,9 @@ TEST_F(MetaStoreClientTest, CreateWatcher) // NOLINT auto revision = client.Get("llt/sn/workers", { .prefix = true }).Get()->header.revision; WatchOption option = { .prefix = true, .prevKv = true, .revision = revision + 1 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client.Watch("llt/sn/workers", option, observer, syncer).Get(); // receive put and delete event ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); @@ -585,7 +610,10 @@ TEST_F(MetaStoreClientTest, CreateWatcher) // NOLINT TEST_F(MetaStoreClientTest, WatchCanceledByServerTest) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = watchHost_ , .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client( + MetaStoreConfig{ + .etcdAddress = watchHost_, .metaStoreAddress = "", .enableMetaStore = false, .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); auto watchActor = std::make_shared(); litebus::Spawn(watchActor); @@ -595,9 +623,14 @@ TEST_F(MetaStoreClientTest, WatchCanceledByServerTest) // NOLINT EXPECT_CALL(*watchActor, Create).WillOnce(testing::DoAll(FutureArg<0>(&createRequest1), testing::Return())); WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = - client.Watch("/test", option, [](const std::vector &events, bool) -> bool { return true; }, syncer).Get(); + client + .Watch( + "/test", option, [](const std::vector &events, bool) -> bool { return true; }, syncer) + .Get(); ASSERT_AWAIT_READY(createRequest1); EXPECT_EQ(TrimKeyPrefix(createRequest1.Get().key(), "/test"), "/test"); @@ -631,7 +664,7 @@ TEST_F(MetaStoreClientTest, CloseWatcher) // NOLINT const std::string key = "/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function" "/0-yrcpp-yr-refcount/version/$latest/defaultaz/cf8e2758-dab0-4775-adff-a746df288052"; - MetaStoreClient watchClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient watchClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); watchClient.Init(); WatchEvent event; WatchOption watchOption = { .prefix = true, .prevKv = false, .revision = 0 }; @@ -639,11 +672,13 @@ TEST_F(MetaStoreClientTest, CloseWatcher) // NOLINT event = events.front(); return true; }; - auto syncer = []() -> litebus::Future { return SyncResult{Status::OK(), 0}; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = watchClient.Watch("/sn/instance/business/yrk/tenant/", watchOption, observer, syncer).Get(); ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); - MetaStoreClient handleClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient handleClient(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ , .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); handleClient.Init(); PutOption putOption = { .leaseId = 0, .prevKv = false }; handleClient.Put(key, "1.0", putOption).Get(); @@ -698,12 +733,11 @@ TEST_F(MetaStoreClientTest, DoRevoke) litebus::Terminate(leaseClient->GetAID()); litebus::Await(leaseClient->GetAID()); - ASSERT_TRUE(true); } TEST_F(MetaStoreClientTest, CampaignTest) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); StartElectionGrantLease(); @@ -766,7 +800,7 @@ TEST_F(MetaStoreClientTest, CampaignTest) // NOLINT TEST_F(MetaStoreClientTest, LeaderTest) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); StartElectionGrantLease(); @@ -802,7 +836,7 @@ TEST_F(MetaStoreClientTest, LeaderTest) // NOLINT TEST_F(MetaStoreClientTest, ResignTest) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); client.Init(); StartElectionGrantLease(); @@ -840,7 +874,11 @@ TEST_F(MetaStoreClientTest, ResignTest) // NOLINT TEST_F(MetaStoreClientTest, ObserveTest) // NOLINT { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = electionHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); StartElectionGrantLease(); @@ -896,7 +934,11 @@ TEST_F(MetaStoreClientTest, ObserveTest) // NOLINT TEST_F(MetaStoreClientTest, FallbreakTest) { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); client.OnHealthyStatus(Status(GRPC_UNKNOWN, "healthy checkfailed")); @@ -926,11 +968,17 @@ TEST_F(MetaStoreClientTest, FallbreakTest) auto resignRsp = client.Resign(LeaderKey()); EXPECT_TRUE(resignRsp.Get().status.IsError()); + + client.OnHealthyStatus(Status::OK()); } TEST_F(MetaStoreClientTest, IsConnectedTest) { - MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, .metaStoreAddress = "", .enableMetaStore= false, .isMetaStorePassthrough = false, .etcdTablePrefix = "/test" }, sslConfig, metaStoreTimeoutOpt_); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_, + .metaStoreAddress = "", + .enableMetaStore = false, + .etcdTablePrefix = "/test" }, + sslConfig, metaStoreTimeoutOpt_); client.Init(); auto isConnected = client.IsConnected(); ASSERT_AWAIT_READY(isConnected); diff --git a/functionsystem/tests/unit/common/metadata/metadata_test.cpp b/functionsystem/tests/unit/common/metadata/metadata_test.cpp index 74db2ccc5f8cfae34c716c9adf5428a46feca586..4e1cd7bae5ebac8e9e2ec33206b39750e043d017 100644 --- a/functionsystem/tests/unit/common/metadata/metadata_test.cpp +++ b/functionsystem/tests/unit/common/metadata/metadata_test.cpp @@ -19,7 +19,7 @@ #include #include -#include "metadata/metadata.h" +#include "common/metadata/metadata.h" namespace functionsystem::test { using namespace ::testing; @@ -710,12 +710,42 @@ TEST_F(LoaderTest, GetInstanceMetaFromJson) "instanceMetaData": { "maxInstance": 20, "minInstance": 2, - "concurrentNum": 1000 + "concurrentNum": 1000, + "diskLimit": 0, + "scalePolicy": "concurrency" } })"; FunctionMeta ins_function = GetFuncMetaFromJson(ins_json); EXPECT_EQ(ins_function.instanceMetaData.maxInstance, 20); EXPECT_EQ(ins_function.instanceMetaData.minInstance, 2); EXPECT_EQ(ins_function.instanceMetaData.concurrentNum, 1000); + EXPECT_EQ(ins_function.instanceMetaData.scalePolicy, "concurrency"); +} + +TEST_F(LoaderTest, GetExtendedMetaDataFromJson) +{ + const std::string ins_json = R"({ + "extendedMetaData": { + "initializer": { + "initializer_handler": "", + "initializer_timeout": 180 + }, + "user_agency": { + "accessKey": "123", + "secretKey": "", + "token": "", + "securityAk": "", + "securitySk": "", + "securityToken": "" + }, + "runtime_graceful_shutdown": { + "maxShutdownTimeout": 10 + } + } + })"; + FunctionMeta ins_function = GetFuncMetaFromJson(ins_json); + EXPECT_EQ(ins_function.extendedMetaData.initializer.timeout, 180); + EXPECT_EQ(ins_function.extendedMetaData.userAgency.accessKey, "123"); + EXPECT_EQ(ins_function.extendedMetaData.customGracefulShutdown.maxShutdownTimeout, 10); } } diff --git a/functionsystem/tests/unit/common/metrics/metrics_adapter_test.cpp b/functionsystem/tests/unit/common/metrics/metrics_adapter_test.cpp index 72a4721e8a319f96c80773702c7bd6c12f548198..33fb215672ab2d52a42e6d34e1a6fb776ac2426a 100644 --- a/functionsystem/tests/unit/common/metrics/metrics_adapter_test.cpp +++ b/functionsystem/tests/unit/common/metrics/metrics_adapter_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "metrics/metrics_adapter.h" +#include "common/metrics/metrics_adapter.h" #include @@ -22,24 +22,127 @@ #include #include "common/resource_view/view_utils.h" +#include "meta_store_client/meta_store_struct.h" #include "metrics/api/provider.h" +#define private public +#include "runtime_manager/metrics/metrics_actor.h" namespace functionsystem::test { namespace metrics_ = functionsystem::metrics; using namespace functionsystem::metrics; namespace MetricsApi = observability::api::metrics; + +const std::string PromExporterJsonStr = R"( +{ + "enabledMetrics": ["yr_instance_running_duration", "yr_app_instance_billing_invoke_latency", "yr_pod_resource", "yr_instance_memory_usage", "yr_node_cpu_usage", "yr_node_npu", "yr_instance_npu"], + "backends": [ + { + "immediatelyExport": { + "name": "Scenario", + "enable": true, + "exporters": [ + { + "prometheusPushExporter": { + } + } + ] + } + } + ] +} + )"; + +const std::string FileExporterJsonStr = R"( +{ + "backends": [ + { + "immediatelyExport": { + "name": "file", + "enable": true, + "exporters": [ + { + "fileExporter": { + "enable": true, + "fileDir": "/tmp/", + "rolling": { + "enable": true, + "maxFiles": 3, + "maxSize": 10000 + }, + "contentType": "STANDARD" + } + } + ] + } + } + ] +} + )"; + +const std::string AlarmExporterJsonStr = R"( +{ + "enabledMetrics": ["yr_k8s_alarm", "yr_proxy_alarm", "fake_metrics", "yr_election_alarm", "yr_etcd_alarm", "yr_metastore_alarm"], + "backends": [ + { + "immediatelyExport": { + "name": "Alarm", + "enable": true, + "custom": { + "labels": { + "site": "", + "tenant_id": "", + "application_id": "", + "service_id": "" + } + }, + "exporters": [ + { + "fileExporter": { + "enable": true, + "enabledInstruments": ["yr_etcd_alarm", "yr_election_alarm"], + "fileDir": "/tmp/", + "rolling": { + "enable": true, + "maxFiles": 3, + "maxSize": 10 + }, + "contentType": "LABELS" + } + } + ] + } + } + ] +} + )"; + +class TestMetricsActor : public runtime_manager::MetricsActor { +public: + TestMetricsActor() : runtime_manager::MetricsActor("test_metrics_actor") + { + } + + + resources::ResourceUnit BuildResourceUnit(const std::vector> &metricses) + { + return runtime_manager::MetricsActor::BuildResourceUnit(metricses); + } +}; + class MetricsAdapterTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } void SetUp() { + metricsActor_ = std::make_shared(); + litebus::Spawn(metricsActor_); } void TearDown() @@ -50,7 +153,11 @@ protected: MetricsAdapter::GetInstance().GetMetricsContext().EraseExtraBillingInstance(); MetricsAdapter::GetInstance().GetMetricsContext().SetAttr("component_name", ""); MetricsAdapter::GetInstance().GetMetricsContext().ErasePodResource(); + litebus::Terminate(metricsActor_->GetAID()); + litebus::Await(metricsActor_->GetAID()); } + + std::shared_ptr metricsActor_; }; TEST_F(MetricsAdapterTest, NrGaugeInstrument) @@ -179,130 +286,105 @@ TEST_F(MetricsAdapterTest, InvalidBackEndKey) EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullMeterProvider); } -TEST_F(MetricsAdapterTest, EtcdUnhealthy) +TEST_F(MetricsAdapterTest, EtcdFiring) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_etcd_alarm"], - "backends": [ - { - "immediatelyExport": { - "name": "Alarm", - "enable": true, - "custom": { - "labels": { - "site": "", - "tenant_id": "", - "application_id": "", - "service_id": "" - } - }, - "exporters": [ - { - "fileExporter": { - "enable": true, - "fileDir": "/tmp/", - "rolling": { - "enable": true, - "maxFiles": 3, - "maxSize": 10 - }, - "contentType": "LABELS" - } - } - ] - } - } - ] -} - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(AlarmExporterJsonStr), + GetMetricsFilesName, {}); MetricsAdapter::GetInstance().GetMetricsContext().SetAttr("component_name", "function_master"); - MetricsAdapter::GetInstance().EtcdUnhealthyFiring(AlarmLevel::CRITICAL, "firing"); + + // resolve etcd firing when no alarm exists + MetricsAdapter::GetInstance().StorageBackendUnhealthyResolved(AlarmLevel::CRITICAL, ETCD_BACKEND); auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::ETCD_ALARM) == alarmMap.end()); + auto alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongEtcdConnection00001") == alarmInfoMap.end()); + + // unhealthy etcd firing + MetricsAdapter::GetInstance().StorageBackendUnhealthyFiring(AlarmLevel::CRITICAL, "firing", ETCD_BACKEND); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); EXPECT_TRUE(alarmMap.find(metrics::ETCD_ALARM) != alarmMap.end()); - MetricsAdapter::GetInstance().EtcdUnhealthyResolved(AlarmLevel::CRITICAL); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongEtcdConnection00001") != alarmInfoMap.end()); + auto etcdAlarmInfo = alarmInfoMap.find("YuanrongEtcdConnection00001")->second; + EXPECT_EQ(etcdAlarmInfo.endsAt, 0); + + // resolve etcd firing + MetricsAdapter::GetInstance().StorageBackendUnhealthyResolved(AlarmLevel::CRITICAL, ETCD_BACKEND); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongEtcdConnection00001") == alarmInfoMap.end()); MetricsAdapter::GetInstance().CleanMetrics(); EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullptr); } -TEST_F(MetricsAdapterTest, ElectionAlarmFiring) -{ - const std::string jsonStr = R"( +TEST_F(MetricsAdapterTest, MetastoreFiring) { - "enabledMetrics": ["yr_election_alarm"], - "backends": [ - { - "immediatelyExport": { - "name": "Alarm", - "enable": true, - "custom": { - "labels": { - "site": "", - "tenant_id": "", - "application_id": "", - "service_id": "" - } - }, - "exporters": [ - { - "fileExporter": { - "enable": true, - "enabledInstruments": ["yr_etcd_alarm", "yr_election_alarm"], - "fileDir": "/tmp/", - "rolling": { - "enable": true, - "maxFiles": 3, - "maxSize": 10 - }, - "contentType": "LABELS" - } - } - ] - } - } - ] + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(AlarmExporterJsonStr), + GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().GetMetricsContext().SetAttr("component_name", "function_proxy"); + + // resolve metastore firing when no alarm exists + auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::METASTORE_ALARM) == alarmMap.end()); + MetricsAdapter::GetInstance().StorageBackendUnhealthyResolved(AlarmLevel::CRITICAL, METASTORE_BACKEND); + auto alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMetastoreConnection00001") == alarmInfoMap.end()); + + // unhealthy metastore firing + MetricsAdapter::GetInstance().StorageBackendUnhealthyFiring(AlarmLevel::CRITICAL, "firing", METASTORE_BACKEND); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::METASTORE_ALARM) != alarmMap.end()); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMetastoreConnection00001") != alarmInfoMap.end()); + auto metastoreAlarmInfo = alarmInfoMap.find("YuanrongMetastoreConnection00001")->second; + EXPECT_EQ(metastoreAlarmInfo.endsAt, 0); + + // resolve metastore firing + MetricsAdapter::GetInstance().StorageBackendUnhealthyResolved(AlarmLevel::CRITICAL, METASTORE_BACKEND); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMetastoreConnection00001") == alarmInfoMap.end()); + + MetricsAdapter::GetInstance().CleanMetrics(); + EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullptr); } - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + +TEST_F(MetricsAdapterTest, ElectionAlarmFiring) +{ + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(AlarmExporterJsonStr), + GetMetricsFilesName, {}); MetricsAdapter::GetInstance().GetMetricsContext().SetAttr("component_name", "function_master"); - MetricsAdapter::GetInstance().ElectionFiring("No leader elected for /yr/leader/function-master"); + + // resolve election firing when no alarm exists + MetricsAdapter::GetInstance().ElectionFiringResolved( + "leader 10.91.2.10 is elected successfully for /yr/leader/function-master"); auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::ELECTION_ALARM) == alarmMap.end()); + auto alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMasterElection00001") == alarmInfoMap.end()); + + // election firing + MetricsAdapter::GetInstance().ElectionFiring("No leader elected for /yr/leader/function-master"); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); EXPECT_TRUE(alarmMap.find(metrics::ELECTION_ALARM) != alarmMap.end()); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMasterElection00001") != alarmInfoMap.end()); + auto etcdAlarmInfo = alarmInfoMap.find("YuanrongMasterElection00001")->second; + EXPECT_EQ(etcdAlarmInfo.endsAt, 0); + + // resolve election firing + MetricsAdapter::GetInstance().ElectionFiringResolved( + "leader 10.91.2.10 is elected successfully for /yr/leader/function-master"); + alarmInfoMap = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetAlarmInfoMap(); + EXPECT_TRUE(alarmInfoMap.find("YuanrongMasterElection00001") == alarmInfoMap.end()); + MetricsAdapter::GetInstance().CleanMetrics(); EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullptr); } TEST_F(MetricsAdapterTest, DoubleGauge) { - const std::string jsonStr = R"( -{ - "backends": [ - { - "immediatelyExport": { - "name": "Alarm", - "enable": true, - "exporters": [ - { - "fileExporter": { - "enable": true, - "fileDir": "/tmp/", - "rolling": { - "enable": true, - "maxFiles": 3, - "maxSize": 10000 - }, - "contentType": "STANDARD" - } - } - ] - } - } - ] -} - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(FileExporterJsonStr), + GetMetricsFilesName, {}); MeterData data; data.value = 1.0f; data.labels["label_key"] = "label_value"; @@ -314,33 +396,8 @@ TEST_F(MetricsAdapterTest, DoubleGauge) TEST_F(MetricsAdapterTest, ReportClusterSourceState) { - const std::string jsonStr = R"( -{ - "backends": [ - { - "immediatelyExport": { - "name": "Alarm", - "enable": true, - "exporters": [ - { - "fileExporter": { - "enable": true, - "fileDir": "/tmp/", - "rolling": { - "enable": true, - "maxFiles": 3, - "maxSize": 10000 - }, - "contentType": "STANDARD" - } - } - ] - } - } - ] -} - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(FileExporterJsonStr), + GetMetricsFilesName, {}); std::shared_ptr unit = std::make_shared(); resources::Resource res; res.mutable_scalar()->set_value(1.0); @@ -356,31 +413,8 @@ TEST_F(MetricsAdapterTest, ReportClusterSourceState) TEST_F(MetricsAdapterTest, ReportBillingInvokeLatency) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_app_instance_billing_invoke_latency"], - "backends": [ - { - "immediatelyExport": { - "name": "Scenario", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } - } - ] -} - )"; - - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); - + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); std::string requestID = "test_request_id"; std::string functionName = "function_name_001"; std::string instanceID = "instance_id"; @@ -406,40 +440,17 @@ TEST_F(MetricsAdapterTest, ReportBillingInvokeLatency) MetricsAdapter::GetInstance().ReportBillingInvokeLatency(requestID, 0, 100000, 100005); + MetricsAdapter::GetInstance().GetMetricsContext().EraseBillingFunctionOptionItem(instanceID); MetricsAdapter::GetInstance().CleanMetrics(); EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullptr); } TEST_F(MetricsAdapterTest, ReportBillingInvokeLatencyNonEnabled) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_instance_running_duration"], - "backends": [ - { - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } - } - ] -} - )"; - - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(AlarmExporterJsonStr), + GetMetricsFilesName, {}); auto enabledInstruments = MetricsAdapter::GetInstance().GetMetricsContext().GetEnabledInstruments(); - EXPECT_TRUE(enabledInstruments.find(YRInstrument::YR_APP_INSTANCE_BILLING_INVOKE_LATENCY) == enabledInstruments.end()); - EXPECT_TRUE(enabledInstruments.find(YRInstrument::YR_INSTANCE_RUNNING_DURATION) != enabledInstruments.end()); std::string requestID = "test_request_id"; std::string functionName = "function_name_001"; @@ -453,36 +464,153 @@ TEST_F(MetricsAdapterTest, ReportBillingInvokeLatencyNonEnabled) EXPECT_EQ(MetricsApi::Provider::GetMeterProvider(), nullptr); } -TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) +TEST_F(MetricsAdapterTest, RegisterPhysicalCPUUtilization) { - const std::string jsonStr = R"( + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); + std::string agentID = "agent001"; + std::string nodeID = "node1ID"; + // register observable instrument + MetricsAdapter::GetInstance().RegisterPhysicalNodeCPUUtilization(); + auto observableInstrumentMap = MetricsAdapter::GetInstance().GetObservableInstrumentMap(); + EXPECT_TRUE(observableInstrumentMap.find("yr_node_cpu_usage") != observableInstrumentMap.end()); + + // init cpu metrics info + auto given = std::vector>{ + {runtime_manager::Metrics{26.3512, 0.0, {}, {}, runtime_manager::metrics_type::CPU}} + }; + auto resourceUnit = metricsActor_->BuildUpdateMetricsRequest(given); + messages::UpdateResourcesRequest req; + EXPECT_TRUE(req.ParseFromString(resourceUnit) == 1); + MetricsAdapter::GetInstance().GetMetricsContext().SetPhysicalMetrics(agentID, nodeID, req.resourceunit()); + + auto obRes = std::make_shared>(); + MetricsAdapter::GetInstance().CollectGeneralMetricsNodeData(obRes, "CPU", true, {}); + auto observedVal = obRes->Value(); + EXPECT_EQ(observedVal.size(), static_cast(1)); + EXPECT_NEAR(observedVal[0].second, 26.3512, 0.0009); + auto labels = observedVal[0].first; + std::string testAgentID = ""; + std::string testNodeID = ""; + for (auto it : labels) { + if (it.first == "agent_id") { + testAgentID = it.second; + } else { + testNodeID = it.second; + } + } + EXPECT_EQ(testAgentID, "agent001"); + EXPECT_EQ(testNodeID, "node1ID"); +} + +TEST_F(MetricsAdapterTest, RegisterInstNPUUsage) { - "enabledMetrics": ["yr_instance_running_duration"], - "backends": [ - { - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); + std::string agentID = "agent001"; + std::string nodeID = "node1ID"; + metricsActor_->nodeID = nodeID; + // register observable instrument + MetricsAdapter::GetInstance().RegisterInstNPUUsage(); + auto observableInstrumentMap = MetricsAdapter::GetInstance().GetObservableInstrumentMap(); + EXPECT_TRUE(observableInstrumentMap.find("yr_instance_npu") != observableInstrumentMap.end()); + + runtime_manager::DevClusterMetrics npuMetrics{ "a1b2c3d4", + 1, + { { "product_model", "Ascend310" } }, + { + { "ids", { 0 } }, + { "used_ids", { 0 } }, + { "usedMemory", {} }, + { "memory", {} }, + { "HBM", { 100} }, + { "usedHBM", { 50 } } } }; + auto given = std::vector>{ { runtime_manager::Metrics{ + 1.0, + 0.0, + {"d15b0853-d008-4000-8000-0000a80934ef"}, + {}, + runtime_manager::metrics_type::NPU, + runtime_manager::collector_type::INSTANCE, + { npuMetrics } } } }; + auto resourceUnitStr = metricsActor_->BuildUpdateMetricsRequest(given); + messages::UpdateResourcesRequest req; + EXPECT_TRUE(req.ParseFromString(resourceUnitStr) == 1); + + auto resourceUnit = req.mutable_resourceunit(); + resources::Value::Counter cnter; + cnter.mutable_items()->insert({ nodeID, 1 }); + resourceUnit->mutable_nodelabels()->insert({ "NODE_ID", cnter }); + + MetricsAdapter::GetInstance().GetMetricsContext().InitInstanceMetrics(req); + auto obRes = std::make_shared>(); + MetricsAdapter::GetInstance().CollectInstanceMetrics(obRes, "NPU"); + auto observedVal = obRes->Value(); + EXPECT_EQ(observedVal.size(), static_cast(1)); + auto labels = observedVal[0].first; + std::string expectedDetail = R"({"0":{"capacity":100.0,"type":"Ascend310","used":50.0}})"; + for (auto it : labels) { + if (it.first == "details") { + EXPECT_EQ(it.second, expectedDetail); + break; } - ] + } } - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); +TEST_F(MetricsAdapterTest, RegisterPhysicalNodeNpu) +{ + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), GetMetricsFilesName, + {}); + std::string agentID = "agent001"; + std::string nodeID = "node1ID"; + metricsActor_->nodeID = nodeID; + YRLOG_INFO("nodeID in metricsActor is {}", metricsActor_->nodeID); + MetricsAdapter::GetInstance().RegisterPhysicalNodeNpu(); + auto observableInstrumentMap = MetricsAdapter::GetInstance().GetObservableInstrumentMap(); + EXPECT_TRUE(observableInstrumentMap.find("yr_node_npu") != observableInstrumentMap.end()); + + runtime_manager::DevClusterMetrics npuMetrics{ "a1b2c3d4", + 1, + { { "product_model", "Ascend310" } }, + { { "HBM", { 100 } }, + { "ids", { 0 } }, + { "usedHBM", { 50 } } } }; + auto given = std::vector>{ { runtime_manager::Metrics{ + 1.0, + 1.0, + {}, + {}, + runtime_manager::metrics_type::NPU, + runtime_manager::collector_type::NODE, + { npuMetrics } } } }; + + auto resourceUnit = metricsActor_->BuildUpdateMetricsRequest(given); + messages::UpdateResourcesRequest req; + EXPECT_TRUE(req.ParseFromString(resourceUnit) == 1); + MetricsAdapter::GetInstance().GetMetricsContext().SetPhysicalMetrics(agentID, nodeID, req.resourceunit()); + + auto obRes = std::make_shared>(); + MetricsAdapter::GetInstance().CollectPhysicalNodeNPUData(obRes); + auto observedVal = obRes->Value(); + EXPECT_EQ(observedVal.size(), static_cast(1)); + auto labels = observedVal[0].first; + std::string expectedDetail = R"({"0":{"capacity":100.0,"type":"Ascend310","used":50.0}})"; + for (auto it : labels) { + if (it.first == "details") { + EXPECT_EQ(it.second, expectedDetail); + break; + } + } +} +TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) +{ + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); std::string requestID = "test_request_id"; std::string functionName = "function_name_001"; std::string instanceID = "instance_id"; + std::string agentId = "agent001"; std::map createOptions = { {"app_name", "testApp"}, {"endpoint", "127.0.0.1"} }; // register observable instrument @@ -490,9 +618,8 @@ TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) auto observableInstrumentMap = MetricsAdapter::GetInstance().GetObservableInstrumentMap(); EXPECT_TRUE(observableInstrumentMap.find("yr_instance_running_duration") != observableInstrumentMap.end()); - // init instance info - MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, createOptions); + MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, agentId, createOptions); auto startTimeMillis = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); auto billingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstanceMap(); auto billingInstance = billingInstanceMap.find(instanceID); @@ -502,7 +629,7 @@ TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) // init extra instance info auto endTimeMillis = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - MetricsAdapter::GetInstance().GetMetricsContext().InitExtraBillingInstance(instanceID, createOptions); + MetricsAdapter::GetInstance().GetMetricsContext().InitExtraBillingInstance(instanceID, agentId, createOptions); auto extraBillingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetExtraBillingInstanceMap(); auto extraBillingInstance = extraBillingInstanceMap.find(instanceID); EXPECT_TRUE(extraBillingInstance != extraBillingInstanceMap.end()); @@ -510,9 +637,6 @@ TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) EXPECT_TRUE(extraBillingInstance->second.lastReportTimeMillis == 0); EXPECT_TRUE(extraBillingInstance->second.startTimeMillis == 0); - NodeLabelsType nodeLabelsMap = { { "label1", { { "label_key1_1", 1 }, { "label_key1_2", 2 } } }, - { "label2", { { "label_key2_1", 11 }, { "label_key2_2", 22 } } } }; - MetricsAdapter::GetInstance().GetMetricsContext().SetBillingNodeLabels(instanceID, nodeLabelsMap); std::string cpuType = "Intel(R) Xeon(R) Gold 6161 CPU @ 2.20GHz"; MetricsAdapter::GetInstance().GetMetricsContext().SetBillingCpuType(instanceID, cpuType); @@ -527,29 +651,48 @@ TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) EXPECT_TRUE(observedVal[1].second >= static_cast(endTimeMillis)); auto labels = observedVal[0].first; std::string cpuTypeSet = ""; + std::string poolLabel = ""; for (auto it : labels) { if (it.first == "cpu_type") { cpuTypeSet = it.second; + } else if (it.first == "pool_label") { + poolLabel = it.second; } } EXPECT_EQ(cpuTypeSet, cpuType); + EXPECT_EQ(poolLabel, "[]"); extraBillingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetExtraBillingInstanceMap(); extraBillingInstance = extraBillingInstanceMap.find(instanceID); EXPECT_TRUE(extraBillingInstance == extraBillingInstanceMap.end()); + auto unit = view_utils::Get1DResourceUnit(agentId); + ::resources::Value::Counter cnter; + cnter.mutable_items()->insert({ "lable1", 1 }); + cnter.mutable_items()->insert({ "lable2", 2 }); + unit.mutable_nodelabels()->insert({ "nodelabel1", cnter }); + metrics::MetricsAdapter::GetInstance().GetMetricsContext().SetPodResource(agentId, unit); + // collect second time MetricsAdapter::GetInstance().GetMetricsContext().SetBillingInstanceReportTime(instanceID, reportTimeMillis - 10); MetricsAdapter::GetInstance().CollectBillingInstanceRunningDuration(obRes); observedVal = obRes->Value(); EXPECT_EQ(observedVal.size(), static_cast(1)); EXPECT_TRUE(observedVal[0].second >= 10); + labels = observedVal[0].first; + poolLabel = ""; + for (auto it : labels) { + if (it.first == "pool_label") { + poolLabel = it.second; + } + } + EXPECT_EQ(poolLabel, "[\"nodelabel1:lable1\",\"nodelabel1:lable2\"]"); //set end time auto billingInstanceInfo = MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstance(instanceID); MetricsAdapter::GetInstance().GetMetricsContext().SetBillingInstanceEndTime(instanceID, billingInstanceInfo.lastReportTimeMillis + 10); MetricsAdapter::GetInstance().CollectBillingInstanceRunningDuration(obRes); observedVal = obRes->Value(); - EXPECT_EQ(observedVal.size(), 1); + EXPECT_EQ(observedVal.size(), static_cast(1)); EXPECT_TRUE(observedVal[0].second >= 10); // instance is cleared, no observable res @@ -560,32 +703,66 @@ TEST_F(MetricsAdapterTest, RegisterBillingInstanceRunningDuration) EXPECT_TRUE(billingFunctionOptionMap.find(instanceID) == billingFunctionOptionMap.end()); } -TEST_F(MetricsAdapterTest, InvalidBillingInstanceRunningDuration) +TEST_F(MetricsAdapterTest, RegisterInstanceMetrics) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_instance_running_duration"], - "backends": [ - { - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); + messages::UpdateResourcesRequest req; + + auto resourceUnit = req.mutable_resourceunit(); + std::string agentId = "randomFuncAgentName_function-agent-pool08012-869f9697f8-7h8bj"; + resourceUnit->set_id(agentId); + + auto instances = resourceUnit->mutable_instances(); + + resources::InstanceInfo instanceInfo1; + auto instanceId = "d15b0853-d008-4000-8000-0000a80934ef"; + instanceInfo1.set_instanceid(instanceId); + + auto actualUse = instanceInfo1.mutable_actualuse()->mutable_resources(); + resources::Resource setMemory; + setMemory.set_name("Memory"); + auto memoryScalar = setMemory.mutable_scalar(); + double setMemoryValue = 111.99609375; + memoryScalar->set_value(setMemoryValue); + actualUse->insert({ "Memory", setMemory }); + + instances->insert({ instanceId, instanceInfo1 }); + + resources::Value::Counter cnter; + auto nodeId = "dggphispra26945"; + cnter.mutable_items()->insert({ nodeId, 1 }); + resourceUnit->mutable_nodelabels()->insert({ "NODE_ID", cnter }); + + MetricsAdapter::GetInstance().GetMetricsContext().InitInstanceMetrics(req); + auto instanceMetricsMap = MetricsAdapter::GetInstance().GetMetricsContext().GetInstanceMetricsMap(); + auto instanceInfoPair = instanceMetricsMap.find(instanceId); + EXPECT_TRUE(instanceInfoPair != instanceMetricsMap.end()); + EXPECT_EQ(instanceInfoPair->second.instanceId, instanceId); + EXPECT_EQ(instanceInfoPair->second.agentId, agentId); + EXPECT_TRUE(instanceInfoPair->second.nodeId == nodeId); + auto actualMemory = instanceInfoPair->second.resources.resources().at("Memory").scalar().value(); + EXPECT_TRUE(actualMemory == setMemoryValue); + + MetricsAdapter::GetInstance().RegisterInstMemoryUsage(); + auto observableInstrumentMap = MetricsAdapter::GetInstance().GetObservableInstrumentMap(); + EXPECT_TRUE(observableInstrumentMap.find("yr_instance_memory_usage") != observableInstrumentMap.end()); + + auto obRes = std::make_shared>(); + MetricsAdapter::GetInstance().CollectInstanceMetrics(obRes, "Memory"); + for (auto value : obRes->Value()) { + auto iter = value.first.front(); + if (iter.first == "instance_id") { + EXPECT_EQ(iter.second, instanceId); + break; } - ] + } } - )"; - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); +TEST_F(MetricsAdapterTest, InvalidBillingInstanceRunningDuration) +{ + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); std::string instanceID = "instance_id"; std::map createOptions = { {"app_name", "testApp"}, {"endpoint", "127.0.0.1"} }; @@ -595,7 +772,7 @@ TEST_F(MetricsAdapterTest, InvalidBillingInstanceRunningDuration) EXPECT_TRUE(observableInstrumentMap.find("yr_instance_running_duration") != observableInstrumentMap.end()); // init instance info - MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, createOptions); + MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, "", createOptions); auto startTimeMillis = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); auto billingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstanceMap(); auto billingInstance = billingInstanceMap.find(instanceID); @@ -612,39 +789,18 @@ TEST_F(MetricsAdapterTest, InvalidBillingInstanceRunningDuration) EXPECT_EQ(observedVal.size(), static_cast(1)); EXPECT_TRUE(observedVal[0].second >= static_cast(reportTimeMillis - startTimeMillis)); - // collect second time + // collect second time when endTime <= lastReportTime MetricsAdapter::GetInstance().GetMetricsContext().SetBillingInstanceReportTime(instanceID, reportTimeMillis + 10); MetricsAdapter::GetInstance().CollectBillingInstanceRunningDuration(obRes); observedVal = obRes->Value(); EXPECT_EQ(observedVal.size(), 0); + EXPECT_EQ(MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstanceMap().size(), 0); } TEST_F(MetricsAdapterTest, SystemInstanceRunningDuartion) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_instance_running_duration"], - "backends": [ - { - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } - } - ] -} - )"; - - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); std::string instanceID = "instance_id"; std::map createOptions = { {"app_name", "testApp"}, {"endpoint", "127.0.0.1"}}; @@ -655,12 +811,12 @@ TEST_F(MetricsAdapterTest, SystemInstanceRunningDuartion) EXPECT_TRUE(observableInstrumentMap.find("yr_instance_running_duration") != observableInstrumentMap.end()); // init instance info - MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, createOptions, true); + MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceID, "", createOptions, true); auto billingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstanceMap(); EXPECT_TRUE(billingInstanceMap.find(instanceID) == billingInstanceMap.end()); // init instance info - MetricsAdapter::GetInstance().GetMetricsContext().InitExtraBillingInstance(instanceID, createOptions, true); + MetricsAdapter::GetInstance().GetMetricsContext().InitExtraBillingInstance(instanceID, "", createOptions, true); auto extraBillingInstanceMap = MetricsAdapter::GetInstance().GetMetricsContext().GetExtraBillingInstanceMap(); EXPECT_TRUE(extraBillingInstanceMap.find(instanceID) == extraBillingInstanceMap.end()); @@ -673,31 +829,8 @@ TEST_F(MetricsAdapterTest, SystemInstanceRunningDuartion) TEST_F(MetricsAdapterTest, ReportBillingInvokeLantencySystemFunction) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_app_instance_billing_invoke_latency"], - "backends": [ - { - "immediatelyExport": { - "name": "Scenario", - "enable": true, - "exporters": [ - { - "prometheusPushExporter": { - "enable": true, - "ip": "prometheus-pushgateway.default.svc.cluster.local", - "port": 9091 - } - } - ] - } - } - ] -} - )"; - - MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); - + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), + GetMetricsFilesName, {}); std::string requestID = "test_request_id"; std::string functionName = "0-system-faasmanager"; std::string instanceID = "instance_id"; @@ -710,6 +843,100 @@ TEST_F(MetricsAdapterTest, ReportBillingInvokeLantencySystemFunction) EXPECT_EQ(billingInvokeOption.functionName, functionName); } +TEST_F(MetricsAdapterTest, ReportInstanceStatus) +{ + std::string PrometheusExporterJson = R"( +{ + "backends": [ + { + "batchExport": { + "name": "Prometheus", + "enable": true, + "exporters": [ + { + "prometheusPushExporter": { + "enable": true, + "batchSize": 20, + "batchIntervalSec": 10, + "failureQueueMaxSize": 100, + "failureDataDir": "/home/sn/metrics/failure", + "failureDataFileMaxCapacity": 10, + "enabledInstruments": [ + "yr_app_instance_status" + ], + "initConfig": { + "ip": "prometheus-pushgateway.default.svc.cluster.local", + "port": 9091, + "heartbeatUrl": "/healthy" + } + } + } + ] + } + } + ] +} +)"; + + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PrometheusExporterJson), + GetMetricsFilesName, {}); + + std::string instanceID = "instance_id"; + resource_view::InstanceInfo instanceInfo; + instanceInfo.set_function("function_1"); + instanceInfo.set_functionagentid("agentid"); + instanceInfo.set_parentid("parentid"); + EXPECT_NO_THROW(MetricsAdapter::GetInstance().ReportInstanceStatus(instanceID, instanceInfo)); +} + +TEST_F(MetricsAdapterTest, ReportInstanceExitLatency) +{ + std::string PrometheusExporterJson = R"( +{ + "backends": [ + { + "batchExport": { + "name": "Prometheus", + "enable": true, + "exporters": [ + { + "prometheusPushExporter": { + "enable": true, + "batchSize": 20, + "batchIntervalSec": 10, + "failureQueueMaxSize": 100, + "failureDataDir": "/home/sn/metrics/failure", + "failureDataFileMaxCapacity": 10, + "enabledInstruments": [ + "yr_instance_exit_latency" + ], + "initConfig": { + "ip": "prometheus-pushgateway.default.svc.cluster.local", + "port": 9091, + "heartbeatUrl": "/healthy" + } + } + } + ] + } + } + ] +} +)"; + + MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PrometheusExporterJson), + GetMetricsFilesName, {}); + + std::string instanceID = "instance_id"; + resource_view::InstanceInfo instanceInfo; + instanceInfo.set_function("function_1"); + instanceInfo.set_functionagentid("agentid"); + instanceInfo.set_parentid("parentid"); + + Status status; + EXPECT_NO_THROW(MetricsAdapter::GetInstance().ReportInstanceExitLatency(status, 1755846561000, instanceInfo)); +} + TEST_F(MetricsAdapterTest, convertNormalNodeLabels) { NodeLabelsType nodeLabelsMap = { { "label1", { { "label_value1_1", 1 }, { "label_value1_2", 2 } } }, @@ -736,30 +963,8 @@ TEST_F(MetricsAdapterTest, invalidYRMetrics) TEST_F(MetricsAdapterTest, SendK8sAlarm) { - const std::string jsonStr = R"( -{ - "enabledMetrics": ["yr_k8s_alarm"], - "backends": [ - { - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [ - { - "aomAlarmExporter": { - "enable": true, - "ip": "127.0.0.1:8080/", - "port": 9091 - } - } - ] - } - } - ] -} - )"; - - metrics::MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(jsonStr), GetMetricsFilesName, {}); + metrics::MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(AlarmExporterJsonStr), + GetMetricsFilesName, {}); metrics::MetricsAdapter::GetInstance().SendK8sAlarm("cluster1"); auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); EXPECT_TRUE(alarmMap.find(metrics::K8S_ALARM) == alarmMap.end()); @@ -909,24 +1114,8 @@ TEST_F(MetricsAdapterTest, PodResourceContextTest) TEST_F(MetricsAdapterTest, CollectPodResourceMetricsTest) { - const std::string podMetricsJson = R"( -{ - "enabledMetrics": ["yr_pod_resource"], - "backends": [{ - "immediatelyExport": { - "name": "LakeHouse", - "enable": true, - "exporters": [{ - "aomAlarmExporter": { - "enable": true, - "ip": "127.0.0.1:8080/", - "port": 9091 - } - }] - } - }] -})"; - metrics::MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(podMetricsJson), + MetricsAdapter::GetInstance().SetContextAttr("component_name", "function_master"); + metrics::MetricsAdapter::GetInstance().InitMetricsFromJson(nlohmann::json::parse(PromExporterJsonStr), GetMetricsFilesName, {}); auto unit = view_utils::Get1DResourceUnit("pod1"); @@ -949,8 +1138,8 @@ TEST_F(MetricsAdapterTest, CollectPodResourceMetricsTest) auto obRes = std::make_shared>(); MetricsAdapter::GetInstance().CollectPodResource(obRes); auto iter = obRes->Value().at(0).first.front(); - EXPECT_EQ(iter.first, "allocatable_cpu"); - EXPECT_EQ(iter.second, "100.100000"); + EXPECT_EQ(iter.first, "agent_id"); + EXPECT_EQ(iter.second, "pod1"); for (auto value : obRes->Value()) { for (const auto &key : value.first) { diff --git a/functionsystem/tests/unit/common/metrics/metrics_metrics_test.cpp b/functionsystem/tests/unit/common/metrics/metrics_metrics_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..48c2af88021cca71c2c90d11d008a23cdf0c75b3 --- /dev/null +++ b/functionsystem/tests/unit/common/metrics/metrics_metrics_test.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/metrics/metrics_adapter.h" +#include "common/metrics/metrics_context.h" + +namespace functionsystem::test { +using namespace functionsystem::metrics; +class MetricsMetricsTest : public ::testing::Test {}; + +TEST_F(MetricsMetricsTest, IsSystemFunc) +{ + EXPECT_FALSE( + MetricsAdapter::GetInstance().GetMetricsContext().IsSystemFunc("function-agent-pool08012-58f488c974-ldgwr")); + + EXPECT_TRUE(MetricsAdapter::GetInstance().GetMetricsContext().IsSystemFunc( + "function-agent-3570bc2a2afa-500m-500mi-faasmanager-55470025008f")); + + EXPECT_TRUE(MetricsAdapter::GetInstance().GetMetricsContext().IsSystemFunc( + "function-agent-18a9078842ae-500m-500mi-faasscheduler-1f0000002f")); + + EXPECT_TRUE(MetricsAdapter::GetInstance().GetMetricsContext().IsSystemFunc( + "function-agent-97ffe67a0963-500m-2048mi-faasfrontend-b655564e00")); + + EXPECT_TRUE(MetricsAdapter::GetInstance().GetMetricsContext().IsSystemFunc( + "function-agent-0-500m-500mi-faascontroller-4d62528400000000001f")); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/network/network_isolation_test.cpp b/functionsystem/tests/unit/common/network/network_isolation_test.cpp index c6264ede64154f9060e099b5441d4475aab5a209..e0d89b8b7b1c8c10bb99c388952b1cb290f3dd79 100644 --- a/functionsystem/tests/unit/common/network/network_isolation_test.cpp +++ b/functionsystem/tests/unit/common/network/network_isolation_test.cpp @@ -30,11 +30,11 @@ class NetworkIsolationTest : public ::testing::Test { class IpsetIpv4NetworkIsolationTest : public NetworkIsolationTest { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } @@ -54,10 +54,53 @@ protected: std::shared_ptr commandRunner_; }; -TEST_F(IpsetIpv4NetworkIsolationTest, IpsetExists) -{ +TEST_F(IpsetIpv4NetworkIsolationTest, IpsetExists) { + CommandExecResult result; + result.output = "Name: test_ipset\nMembers:\n192.168.1.1"; + result.error = ""; + EXPECT_CALL(*commandRunner_, ExecuteCommandWrapper(_)).Times(1).WillRepeatedly(Return(result)); + bool exists = isolation_->IsIpsetExist(); - EXPECT_FALSE(exists); + EXPECT_TRUE(exists); +} + +TEST_F(IpsetIpv4NetworkIsolationTest, GetAllRulesSuccess) { + CommandExecResult result; + result.output = "Members:\n192.168.1.1\n192.168.1.2"; + EXPECT_CALL(*commandRunner_, ExecuteCommandWrapper(_)).Times(1).WillRepeatedly(Return(result)); + + auto rules = isolation_->GetAllRules(); + EXPECT_EQ(rules.size(), (size_t)2); + EXPECT_THAT(rules, ::testing::ElementsAre("192.168.1.1", "192.168.1.2")); +} + +TEST_F(IpsetIpv4NetworkIsolationTest, AddRuleSuccess) { + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)).Times(1).WillRepeatedly(Return(true)); + + int result = isolation_->AddRule("192.168.1.1"); + EXPECT_EQ(result, 0); +} + +TEST_F(IpsetIpv4NetworkIsolationTest, AddRulesSuccess) { + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)).Times(1).WillRepeatedly(Return(true)); + + std::vector rules = {"192.168.1.1", "192.168.1.2"}; + int result = isolation_->AddRules(rules); + EXPECT_EQ(result, 0); +} + +TEST_F(IpsetIpv4NetworkIsolationTest, RemoveRuleSuccess) { + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)).Times(1).WillRepeatedly(Return(true)); + + int result = isolation_->RemoveRule("192.168.1.1"); + EXPECT_EQ(result, 0); +} + +TEST_F(IpsetIpv4NetworkIsolationTest, RemoveAllRulesSuccess) { + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)).Times(1).WillRepeatedly(Return(true)); + + int result = isolation_->RemoveAllRules(); + EXPECT_EQ(result, 0); } } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/register/register_helper_test.cpp b/functionsystem/tests/unit/common/register/register_helper_test.cpp index 05b978daa3fdff8a89a08c8765efdb02526aa11c..506146eb173cb7a1129ce8b2347a8f63f86ffe41 100644 --- a/functionsystem/tests/unit/common/register/register_helper_test.cpp +++ b/functionsystem/tests/unit/common/register/register_helper_test.cpp @@ -18,11 +18,11 @@ #include -#include "heartbeat/heartbeat_observer.h" -#include "heartbeat/ping_pong_driver.h" -#include "logs/logging.h" -#include "proto/pb/message_pb.h" -#include "status/status.h" +#include "common/heartbeat/heartbeat_client.h" +#include "common/heartbeat/heartbeat_observer.h" +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" +#include "common/status/status.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -50,15 +50,16 @@ public: YRLOG_INFO("register fail"); return; } - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); - registerHelper_->SetHeartbeatObserveDriver(DOWNSTREAM_ACTOR_NAME, "127.0.0.1:" + std::to_string(port), 1000, - [heartbeatTimeout(heartbeatTimeout_)](const litebus::AID &aid) { - YRLOG_INFO("upstream heartbeat timeout, aid: {}", - aid.HashString()); - if (heartbeatTimeout != nullptr) { - heartbeatTimeout->SetValue(true); - } - }); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + registerHelper_->SetHeartbeatObserveDriver(DOWNSTREAM_ACTOR_NAME, + "127.0.0.1:" + std::to_string(port), + 1000, + [heartbeatTimeout(heartbeatTimeout_)](const litebus::AID &aid, HeartbeatConnection) { + YRLOG_INFO("upstream heartbeat timeout, aid: {}", aid.HashString()); + if (heartbeatTimeout != nullptr) { + heartbeatTimeout->SetValue(true); + } + }, UPSTREAM_ACTOR_NAME); messages::Registered registeredMsg; registeredMsg.set_code(static_cast(StatusCode::SUCCESS)); registeredMsg.set_message("register successfully"); @@ -73,7 +74,7 @@ public: litebus::Future HeartbeatTimeoutFuture() { - return heartbeatTimeout_->GetFuture(); + return heartbeatTimeout_->GetFuture(); }; private: @@ -101,15 +102,18 @@ public: messages::Registered registeredMsg; registeredMsg.ParseFromString(msg); YRLOG_INFO("registered code: {}, message: {}", registeredMsg.code(), registeredMsg.message()); - registerHelper_->SetPingPongDriver( - 1000, [heartbeatTimeout(heartbeatTimeout_)](const litebus::AID &aid, HeartbeatConnection type) { + uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); + std::string actorAddress = "127.0.0.1:" + std::to_string(port); + registerHelper_->SetPingPongDriver(DOWNSTREAM_ACTOR_NAME, actorAddress, + 1000, [heartbeatTimeout(heartbeatTimeout_)](const litebus::AID &aid) { if (heartbeatTimeout != nullptr) { heartbeatTimeout->SetValue(true); } - }); + }, UPSTREAM_ACTOR_NAME); } - void RegisterTimeoutHandler(){ + void RegisterTimeoutHandler() + { registerTimeout_->SetValue(true); } @@ -120,8 +124,8 @@ public: uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); std::string actorAddress = "127.0.0.1:" + std::to_string(port); registerMsg.set_address(actorAddress); - registerHelper_->StartRegister(UPSTREAM_ACTOR_NAME, actorAddress, registerMsg.SerializeAsString(), - maxRegistersTimes); + registerHelper_->StartRegister( + UPSTREAM_ACTOR_NAME, actorAddress, registerMsg.SerializeAsString(), maxRegistersTimes); } bool IsRegistered() diff --git a/functionsystem/tests/unit/common/resource_lock/resource_lock_test.cpp b/functionsystem/tests/unit/common/resource_lock/resource_lock_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..91713adab1ff6a9ea2e88ad1ca9a550a3653803c --- /dev/null +++ b/functionsystem/tests/unit/common/resource_lock/resource_lock_test.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "common/resource_lock/lease_lock.h" +#include "mocks/mock_kube_client.h" +#include "utils/future_test_helper.h" + +using namespace functionsystem::resource_lock; +using V1Lease = functionsystem::kube_client::model::V1Lease; + +namespace functionsystem::test { + +class ResourceLockTest : public ::testing::Test { +protected: + [[maybe_unused]] static void SetUpTestSuite() { + } + + [[maybe_unused]] static void TearDownTestSuite() { + } + + void SetUp() + { + } + + void TearDown() + { + } +}; + +TEST_F(ResourceLockTest, LeaseLockTest) +{ + auto mockClient = std::make_shared(); + std::shared_ptr lock = std::make_shared("a", mockClient, "default", "function-master"); + EXPECT_EQ("a", lock->Identity()); + // get lease + litebus::Promise> promise; + promise.SetFailed(404); + EXPECT_CALL(*mockClient, ReadNamespacedLease).WillOnce(testing::Return(promise.GetFuture())); + auto res = lock->Get(); + res.Get(); + EXPECT_TRUE(res.IsError()); + litebus::Promise> promise1; + nlohmann::json podJson = nlohmann::json::parse(DEFAULT_LEASE_JSON_STR); + std::shared_ptr lease = std::make_shared(); + lease->FromJson(podJson); + promise1.SetValue(lease); + EXPECT_CALL(*mockClient, ReadNamespacedLease).WillOnce(testing::Return(promise1.GetFuture())); + res = lock->Get(); + res.Get(); + EXPECT_EQ("10.10.10.10:22770", res.Get()->GetSpec()->GetHolderIdentity()); + // create lease + std::shared_ptr record = std::make_shared(); + record->holderIdentity = "test"; + record->leaseTransitions = 1; + record->acquireTime = std::chrono::system_clock::now(); + record->renewTime = std::chrono::system_clock::now(); + record->leaseDurationSeconds = 2; + litebus::Promise> createPromise; + createPromise.SetFailed(404); + EXPECT_CALL(*mockClient, CreateNamespacedLease).WillOnce(testing::Return(createPromise.GetFuture())); + auto resStatus = lock->Create(record); + resStatus.Get(); + EXPECT_TRUE(resStatus.IsError()); + litebus::Future> body; + EXPECT_CALL(*mockClient, CreateNamespacedLease) + .WillOnce(testing::DoAll(test::FutureArg<1>(&body), testing::Return(lease))); + resStatus = lock->Create(record); + resStatus.Get(); + ASSERT_AWAIT_READY(body); + EXPECT_EQ("test", body.Get()->GetSpec()->GetHolderIdentity()); + // update + litebus::Promise> updatePromise; + updatePromise.SetFailed(404); + EXPECT_CALL(*mockClient, ReplaceNamespacedLease).WillOnce(testing::Return(createPromise.GetFuture())); + lock->SetLease(lease); + resStatus = lock->Update(record); + resStatus.Get(); + EXPECT_TRUE(resStatus.IsError()); + litebus::Future> body1; + EXPECT_CALL(*mockClient, ReplaceNamespacedLease) + .WillOnce(testing::DoAll(test::FutureArg<2>(&body1), testing::Return(lease))); + resStatus = lock->Update(record); + resStatus.Get(); + ASSERT_AWAIT_READY(body1); + EXPECT_EQ("test", body.Get()->GetSpec()->GetHolderIdentity()); +} + +} diff --git a/functionsystem/tests/unit/common/resource_view/resource_poller_test.cpp b/functionsystem/tests/unit/common/resource_view/resource_poller_test.cpp index 0533840cfa838b59eae3c712717bd0cac0b471ad..4b6f8507e7f0019b9576adba73795977e4cc597f 100644 --- a/functionsystem/tests/unit/common/resource_view/resource_poller_test.cpp +++ b/functionsystem/tests/unit/common/resource_view/resource_poller_test.cpp @@ -22,7 +22,7 @@ #include #include "async/asyncafter.hpp" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "litebus.hpp" #include "utils/future_test_helper.h" diff --git a/functionsystem/tests/unit/common/resource_view/resource_tool_test.cpp b/functionsystem/tests/unit/common/resource_view/resource_tool_test.cpp index 2061f10875eb433044983ae90f856ddac14943e8..ea7b0e3d1a75e893254196f76136f32c14d05e15 100644 --- a/functionsystem/tests/unit/common/resource_view/resource_tool_test.cpp +++ b/functionsystem/tests/unit/common/resource_view/resource_tool_test.cpp @@ -31,11 +31,11 @@ namespace functionsystem::test { class ResourceToolTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } @@ -666,4 +666,24 @@ TEST_F(ResourceToolTest, HasInnerAffinityTest) EXPECT_TRUE(HasInnerAffinity(instance)); } +TEST_F(ResourceToolTest, ValidateInstanceDiskResource) +{ + resource_view::InstanceInfo instance; + EXPECT_FALSE(HasDiskResource(instance)); + + resource_view::Resources rs = view_utils::GetDiskResources(); + (*instance.mutable_resources()) = rs; + EXPECT_TRUE(HasDiskResource(instance)); +} + +TEST_F(ResourceToolTest, ValidateResourceunitDiskResource) +{ + resource_view::ResourceUnit unit; + EXPECT_FALSE(HasDiskResource(unit)); + + resource_view::Resources rs = view_utils::GetDiskResources(); + (*unit.mutable_allocatable()) = rs; + EXPECT_TRUE(HasDiskResource(unit)); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/resource_view/resource_view_mgr_test.cpp b/functionsystem/tests/unit/common/resource_view/resource_view_mgr_test.cpp index 147ca719481ac4f23d8f918f02a151a5ed66a2b1..372a29ffb95638af64a27078183658dcd55a8ce3 100644 --- a/functionsystem/tests/unit/common/resource_view/resource_view_mgr_test.cpp +++ b/functionsystem/tests/unit/common/resource_view/resource_view_mgr_test.cpp @@ -19,6 +19,7 @@ #include #include "mocks/mock_resource_view.h" #include "utils/port_helper.h" +#include "utils/future_test_helper.h" namespace functionsystem::test { const std::string LITEBUS_URL("127.0.0.1:8080"); // NOLINT @@ -68,17 +69,18 @@ TEST_F(ResourceViewMgrTest, GetResources) mgr_->primary_ = mockPrimary; mgr_->virtual_ = mockVirtual; - EXPECT_CALL(*mockPrimary, GetFullResourceView).WillOnce(Return(std::make_shared())); - EXPECT_CALL(*mockVirtual, GetFullResourceView).WillOnce(Return(std::make_shared())); + EXPECT_CALL(*mockPrimary, GetFullResourceView).WillOnce(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*mockVirtual, GetFullResourceView).WillOnce(Return(AsyncReturn(std::make_shared()))); auto future = mgr_->GetResources(); auto resources = future.Get(); EXPECT_EQ(resources.size(), (size_t)2); EXPECT_EQ(resources.find(ResourceType::PRIMARY) != resources.end(), true); EXPECT_EQ(resources.find(ResourceType::VIRTUAL) != resources.end(), true); - - EXPECT_CALL(*mockPrimary, GetFullResourceView).WillOnce(Return(litebus::Status(StatusCode::FAILED))); - EXPECT_CALL(*mockVirtual, GetFullResourceView).WillOnce(Return(std::make_shared())); + litebus::Future> failure; + failure.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockPrimary, GetFullResourceView).WillOnce(Return(AsyncReturn(failure))); + EXPECT_CALL(*mockVirtual, GetFullResourceView).WillOnce(Return(AsyncReturn(std::make_shared()))); future = mgr_->GetResources(); resources = future.Get(); EXPECT_EQ(resources.size(), (size_t)0); @@ -92,17 +94,18 @@ TEST_F(ResourceViewMgrTest, GetChanges) mgr_->primary_ = mockPrimary; mgr_->virtual_ = mockVirtual; - EXPECT_CALL(*mockPrimary, GetResourceViewChanges).WillOnce(Return(std::make_shared())); - EXPECT_CALL(*mockVirtual, GetResourceViewChanges).WillOnce(Return(std::make_shared())); + EXPECT_CALL(*mockPrimary, GetResourceViewChanges).WillOnce(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*mockVirtual, GetResourceViewChanges).WillOnce(Return(AsyncReturn(std::make_shared()))); auto future = mgr_->GetChanges(); auto resources = future.Get(); EXPECT_EQ(resources.size(), (size_t)2); EXPECT_EQ(resources.find(ResourceType::PRIMARY) != resources.end(), true); EXPECT_EQ(resources.find(ResourceType::VIRTUAL) != resources.end(), true); - - EXPECT_CALL(*mockPrimary, GetResourceViewChanges).WillOnce(Return(litebus::Status(StatusCode::FAILED))); - EXPECT_CALL(*mockVirtual, GetResourceViewChanges).WillOnce(Return(std::make_shared())); + litebus::Future> failure; + failure.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockPrimary, GetResourceViewChanges).WillOnce(Return(AsyncReturn(failure))); + EXPECT_CALL(*mockVirtual, GetResourceViewChanges).WillOnce(Return(AsyncReturn(std::make_shared()))); future = mgr_->GetChanges(); resources = future.Get(); EXPECT_EQ(resources.size(), (size_t)0); diff --git a/functionsystem/tests/unit/common/resource_view/resource_view_test.cpp b/functionsystem/tests/unit/common/resource_view/resource_view_test.cpp index 171f67d079cce75df81f46255ec9bcf11ae10eea..facd31e7462d0b78a68acd62198e752b6fabfaa2 100644 --- a/functionsystem/tests/unit/common/resource_view/resource_view_test.cpp +++ b/functionsystem/tests/unit/common/resource_view/resource_view_test.cpp @@ -20,8 +20,8 @@ #include -#include "constants.h" -#include "metrics/metrics_adapter.h" +#include "common/constants/constants.h" +#include "common/metrics/metrics_adapter.h" #include "common/resource_view/resource_poller.h" #include "common/resource_view/resource_tool.h" #include "common/types/instance_state.h" @@ -29,6 +29,8 @@ #include "view_utils.h" #include "utils/port_helper.h" +#define private public + namespace functionsystem::test { using namespace functionsystem::resource_view; @@ -52,11 +54,11 @@ static const std::string NEED_RECOVER_VIEW = "needRecoverView"; class ResourceViewTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } @@ -1600,13 +1602,19 @@ TEST_F(ResourceViewTest, MergeResourceViewChanges) EXPECT_EQ(change8.modification().instancechanges(0).instance().instanceid(), inst1.instanceid()); // Situation 9: modify resourceunit(delete instance2) + modify resourceunit(add instance2) - // --> no changes + // --> modify resourceunit(delete instance2 + add instance2) ResourceUnitChanges result9; masterRevisionInPullRequest = currentRevision3; currentLocalRevision = currentRevision5; resourceView.MergeLocalResourceViewChanges(masterRevisionInPullRequest, currentLocalRevision, result9); + EXPECT_EQ(result9.changes_size(), 1); + const ResourceUnitChange &change9 = result9.changes(0); EXPECT_EQ(result9.endrevision(), currentLocalRevision); - EXPECT_EQ(result9.changes_size(), 0); + EXPECT_EQ(change9.Changed_case(), ResourceUnitChange::kModification); + EXPECT_EQ(change9.modification().instancechanges(0).changetype(), InstanceChange::DELETE); + EXPECT_EQ(change9.modification().instancechanges(0).instance().instanceid(), inst2.instanceid()); + EXPECT_EQ(change9.modification().instancechanges(1).changetype(), InstanceChange::ADD); + EXPECT_EQ(change9.modification().instancechanges(1).instance().instanceid(), inst2.instanceid()); } TEST_F(ResourceViewTest, MergeInstanceChange) @@ -1678,13 +1686,19 @@ TEST_F(ResourceViewTest, MergeInstanceChange) ASSERT_AWAIT_READY(ret); // Situation 1: delete instance2 + add instance2 - // --> no changes + // --> delete instance2 + add instance2 auto result = ResourceUnitChanges{}; masterRevisionInPullRequest = currentRevision3; currentLocalRevision = currentRevision5; resourceView.MergeLocalResourceViewChanges(masterRevisionInPullRequest, currentLocalRevision, result); EXPECT_EQ(result.endrevision(), currentLocalRevision); - EXPECT_EQ(result.changes_size(), 0); + EXPECT_EQ(result.changes_size(), 1); + auto change = result.changes(0); + EXPECT_EQ(change.Changed_case(), ResourceUnitChange::kModification); + EXPECT_EQ(change.modification().instancechanges(0).changetype(), InstanceChange::DELETE); + EXPECT_EQ(change.modification().instancechanges(0).instance().instanceid(), inst2.instanceid()); + EXPECT_EQ(change.modification().instancechanges(1).changetype(), InstanceChange::ADD); + EXPECT_EQ(change.modification().instancechanges(1).instance().instanceid(), inst2.instanceid()); // Situation 2: add instance2 + delete instance2 // --> no changes @@ -1702,7 +1716,7 @@ TEST_F(ResourceViewTest, MergeInstanceChange) currentLocalRevision = currentRevision6; resourceView.MergeLocalResourceViewChanges(masterRevisionInPullRequest, currentLocalRevision, result); EXPECT_EQ(result.changes_size(), 1); - auto change = result.changes(0); + change = result.changes(0); EXPECT_EQ(result.endrevision(), currentLocalRevision); EXPECT_EQ(change.Changed_case(), ResourceUnitChange::kModification); EXPECT_EQ(change.modification().instancechanges(0).changetype(), InstanceChange::DELETE); @@ -1722,12 +1736,18 @@ TEST_F(ResourceViewTest, MergeInstanceChange) EXPECT_EQ(change.modification().instancechanges(0).instance().instanceid(), inst2.instanceid()); // Situation 5: delete instance2 + add instance2 + delete instance2 + add instance2 - // --> no changes + // --> delete instance2 + add instance2 result = ResourceUnitChanges{}; masterRevisionInPullRequest = currentRevision3; currentLocalRevision = currentRevision7; resourceView.MergeLocalResourceViewChanges(masterRevisionInPullRequest, currentLocalRevision, result); - EXPECT_EQ(result.changes_size(), 0); + EXPECT_EQ(result.changes_size(), 1); + change = result.changes(0); + EXPECT_EQ(change.Changed_case(), ResourceUnitChange::kModification); + EXPECT_EQ(change.modification().instancechanges(0).changetype(), InstanceChange::DELETE); + EXPECT_EQ(change.modification().instancechanges(0).instance().instanceid(), inst2.instanceid()); + EXPECT_EQ(change.modification().instancechanges(1).changetype(), InstanceChange::ADD); + EXPECT_EQ(change.modification().instancechanges(1).instance().instanceid(), inst2.instanceid()); // Situation 6: add instance2 + delete instance2 + add instance2 + delete instance2 // --> no changes @@ -1902,6 +1922,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) litebus::GetActor(childNode + "-ResourceViewActor")->GetAID().Url()); ASSERT_FALSE(add.Get().IsOk()); + auto childView = child->GetFullResourceView().Get(); + auto curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 1.add resourceunit auto currentRevision1 = 1; auto unit = Get1DResourceUnit(); @@ -1927,6 +1952,15 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) EXPECT_EQ(bucket.total().monopolynum(), 1); EXPECT_EQ(bucket.allocatable().at(agentId).monopolynum(), 1); + auto emptySnap = parent->GetUnitSnapshotInfo(""); + EXPECT_EQ(emptySnap.Get().version(), 0); + EXPECT_TRUE(emptySnap.Get().localviewinittime().empty()); + + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 2.modify resourceunit -- add instance1(shared) rView = parent->GetResourceView(); ASSERT_AWAIT_READY(rView); @@ -1957,6 +1991,10 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) bucket = rView.Get()->fragment().at(agentId).bucketindexs().at(unitProportion).buckets().at(unitMem); EXPECT_EQ(bucket.total().monopolynum(), 0); EXPECT_EQ(bucket.allocatable().at(agentId).monopolynum(), 0); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); // 3.modify resourceunit -- add instance2(shared) rView = parent->GetResourceView(); @@ -1988,6 +2026,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) EXPECT_EQ(bucket.total().monopolynum(), 0); EXPECT_EQ(bucket.mutable_allocatable()->at(agentId).monopolynum(), 0); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 4.del instance1(shared) std::vector ids1; ids1.push_back(inst1.instanceid()); @@ -2007,6 +2050,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) EXPECT_EQ(bucket.total().monopolynum(), 0); EXPECT_EQ(bucket.allocatable().at(agentId).monopolynum(), 0); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 5.del instance2(shared) std::vector ids2; ids2.push_back(inst2.instanceid()); @@ -2027,6 +2075,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) EXPECT_EQ(bucket.total().monopolynum(), 1); EXPECT_EQ(bucket.allocatable().at(agentId).monopolynum(), 1); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 6.update status auto currentRevision6 = 6; begin = litebus::TimeWatch::Now(); @@ -2040,6 +2093,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) static_cast(resource_view::UnitStatus::EVICTING)); ASSERT_EQ(parent->GetLocalInfoInDomain(childNode).localRevisionInDomain, currentRevision6); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 7.modify resourceunit -- add instance3(monopoly) rView = parent->GetResourceView(); ASSERT_AWAIT_READY(rView); @@ -2062,6 +2120,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) EXPECT_EQ(bucket.total().monopolynum(), 0); EXPECT_EQ(bucket.mutable_allocatable()->at(agentId).monopolynum(), 0); + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); + // 8.del resourceunit begin = litebus::TimeWatch::Now(); ASSERT_TRUE(child->DeleteResourceUnit(agentId).Get().IsOk()); @@ -2073,6 +2136,11 @@ TEST_F(ResourceViewTest, PullResourceUnitTest) ASSERT_EQ(rView.Get()->fragment_size(), 0); bucket = rView.Get()->mutable_bucketindexs()->at(unitProportion).mutable_buckets()->at(unitMem); EXPECT_EQ(bucket.total().monopolynum(), 0); + + childView = child->GetFullResourceView().Get(); + curSnap = parent->GetUnitSnapshotInfo(childNode).Get(); + EXPECT_EQ(curSnap.version(), childView->revision()); + EXPECT_EQ(curSnap.localviewinittime(), childView->viewinittime()); } TEST_F(ResourceViewTest, PullResourceUnitWithSwitchDomainUrlTest) @@ -3070,14 +3138,14 @@ TEST_F(ResourceViewTest, idleToRecyclePodDeleteInstance) auto idleToRecycleLabels = GenerateIdleToRecycleLabel(); std::set disabledAgent; std::atomic count = 0; - viewPtr->RegisterUnitDisableFunc([&disabledAgent, &count](const std::string &agentID){ + viewPtr->RegisterUnitDisableFunc([&disabledAgent, &count](const std::string &agentID) { disabledAgent.emplace(agentID); count++; }); // pod has been set timer and in recycle, addInstance, pod will not be recycled; auto unit3 = Get1DResourceUnit(); - unit3.mutable_nodelabels()->insert({IDLE_TO_RECYCLE, idleToRecycleLabels[2]}); + unit3.mutable_nodelabels()->insert({ IDLE_TO_RECYCLE, idleToRecycleLabels[2] }); ASSERT_AWAIT_READY(resourceView.AddResourceUnit(unit3)); EXPECT_EQ(resourceView.GetReuseTimers().count(unit3.id()), 1); @@ -3127,7 +3195,7 @@ TEST_F(ResourceViewTest, idleToRecyclePodDeleteInstance) ret = resourceView.DeleteInstances(ids3); ASSERT_AWAIT_READY(ret); EXPECT_TRUE(ret.Get().IsOk()); - EXPECT_AWAIT_TRUE([&] () -> bool { return resourceView.GetReuseTimers().count(unit3.id()) == 0; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return resourceView.GetReuseTimers().count(unit3.id()) == 0; }); cache = resourceView.GetAgentCacheMap(); EXPECT_TRUE(cache.find(unit3.id()) == cache.end()); EXPECT_EQ(count, 1); @@ -3153,7 +3221,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) viewPtr->SetEnableTenantAffinity(true); std::set disabledAgent; std::atomic count = 0; - viewPtr->RegisterUnitDisableFunc([&disabledAgent, &count](const std::string &agentID){ + viewPtr->RegisterUnitDisableFunc([&disabledAgent, &count](const std::string &agentID) { disabledAgent.emplace(agentID); count++; }); @@ -3161,7 +3229,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) { // 1.pod not set idlePod, enable tenantAffinity and delete actually instance -> set timer and recycle auto unit1 = Get1DResourceUnit(); - unit1.mutable_nodelabels()->insert({IDLE_TO_RECYCLE, idleToRecycleLabels[1]}); + unit1.mutable_nodelabels()->insert({ IDLE_TO_RECYCLE, idleToRecycleLabels[1] }); ASSERT_AWAIT_READY(resourceView.AddResourceUnit(unit1)); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); @@ -3184,7 +3252,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) ASSERT_AWAIT_READY(ret); EXPECT_TRUE(ret.Get().IsOk()); - std::vector ids1{inst1.instanceid()}; + std::vector ids1{ inst1.instanceid() }; ret = resourceView.DeleteInstances(ids1); ASSERT_AWAIT_READY(ret); EXPECT_TRUE(ret.Get().IsOk()); @@ -3200,7 +3268,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) ret = resourceView.DeleteInstances(ids3); ASSERT_AWAIT_READY(ret); EXPECT_TRUE(ret.Get().IsOk()); - EXPECT_AWAIT_TRUE([&] () -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); cache = resourceView.GetAgentCacheMap(); EXPECT_TRUE(cache.find(unit1.id()) == cache.end()); @@ -3212,10 +3280,10 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) // pod not set idlePod, enable tenantAffinity and delete actually instance, // set timer and then add instance and then delete virtual instance -> set timer and recycle auto unit1 = Get1DResourceUnit(); - unit1.mutable_nodelabels()->insert({IDLE_TO_RECYCLE, idleToRecycleLabels[1]}); + unit1.mutable_nodelabels()->insert({ IDLE_TO_RECYCLE, idleToRecycleLabels[1] }); ASSERT_AWAIT_READY(resourceView.AddResourceUnit(unit1)); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); - count = 0; // reset + count = 0; // reset // mock instance is actually use, and delete auto inst2 = Get1DInstance(); @@ -3230,7 +3298,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) auto ret = resourceView.AddInstances(instances); ASSERT_AWAIT_READY(ret); - std::vector ids1{inst1.instanceid()}; + std::vector ids1{ inst1.instanceid() }; ret = resourceView.DeleteInstances(ids1); ASSERT_AWAIT_READY(ret); @@ -3247,10 +3315,10 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) ret = resourceView.AddInstances(instances2); ASSERT_AWAIT_READY(ret); - std::vector ids2{inst2.instanceid()}; + std::vector ids2{ inst2.instanceid() }; ASSERT_AWAIT_READY(resourceView.DeleteInstances(ids2, true)); - EXPECT_AWAIT_TRUE([&] () -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); cache = resourceView.GetAgentCacheMap(); EXPECT_TRUE(cache.find(unit1.id()) == cache.end()); EXPECT_EQ(count, 1); @@ -3258,12 +3326,13 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) } { - // pod not set idlePod, enable tenantAffinity and add unit and then delete virtual instance -> not set timer and not recycle + // pod not set idlePod, enable tenantAffinity and add unit and then delete virtual instance -> not set timer and + // not recycle auto unit1 = Get1DResourceUnit(); - unit1.mutable_nodelabels()->insert({IDLE_TO_RECYCLE, idleToRecycleLabels[1]}); + unit1.mutable_nodelabels()->insert({ IDLE_TO_RECYCLE, idleToRecycleLabels[1] }); ASSERT_AWAIT_READY(resourceView.AddResourceUnit(unit1)); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); - count = 0; // reset + count = 0; // reset // mock instance is virtual use, and delete auto inst2 = Get1DInstance(); @@ -3277,7 +3346,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) instances.emplace(inst1.instanceid(), resource_view::InstanceAllocatedInfo{ inst1, nullptr }); auto ret = resourceView.AddInstances(instances); ASSERT_AWAIT_READY(ret); - std::vector ids1{inst1.instanceid()}; + std::vector ids1{ inst1.instanceid() }; ret = resourceView.DeleteInstances(ids1, true); ASSERT_AWAIT_READY(ret); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); @@ -3286,7 +3355,7 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) instances.emplace(inst2.instanceid(), resource_view::InstanceAllocatedInfo{ inst2, nullptr }); ret = resourceView.AddInstances(instances2); ASSERT_AWAIT_READY(ret); - std::vector ids2{inst2.instanceid()}; + std::vector ids2{ inst2.instanceid() }; ret = resourceView.DeleteInstances(ids2, true); ASSERT_AWAIT_READY(ret); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); @@ -3296,10 +3365,10 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) // pod not set idlePod, enable tenantAffinity and add unit and then delete instance and add virtual instance, // timer is executed, and then virtual instance is deleted -> pod need to be recycled auto unit1 = Get1DResourceUnit(); - unit1.mutable_nodelabels()->insert({IDLE_TO_RECYCLE, idleToRecycleLabels[1]}); + unit1.mutable_nodelabels()->insert({ IDLE_TO_RECYCLE, idleToRecycleLabels[1] }); ASSERT_AWAIT_READY(resourceView.AddResourceUnit(unit1)); EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 0); - count = 0; // reset + count = 0; // reset auto inst2 = Get1DInstance(); auto inst1 = Get1DInstance(); inst1.set_unitid(unit1.id()); @@ -3312,10 +3381,10 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) instances.emplace(inst1.instanceid(), resource_view::InstanceAllocatedInfo{ inst1, nullptr }); auto ret = resourceView.AddInstances(instances); ASSERT_AWAIT_READY(ret); - std::vector ids1{inst1.instanceid()}; + std::vector ids1{ inst1.instanceid() }; ret = resourceView.DeleteInstances(ids1, false); ASSERT_AWAIT_READY(ret); - EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 1); // timer is set + EXPECT_EQ(resourceView.GetReuseTimers().count(unit1.id()), 1); // timer is set EXPECT_EQ(resourceView.GetAgentUsedMap().count(unit1.id()), 1); // mock virtual instance add @@ -3331,10 +3400,10 @@ TEST_F(ResourceViewTest, tenantAffinityRecyclePodDeleteInstance) EXPECT_TRUE(cache.find(unit1.id()) != cache.end()); // delete virtual instance , and pod is need to be recycled - std::vector ids2{inst2.instanceid()}; + std::vector ids2{ inst2.instanceid() }; ret = resourceView.DeleteInstances(ids2, true); ASSERT_AWAIT_READY(ret); - EXPECT_AWAIT_TRUE([&] () -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return resourceView.GetReuseTimers().count(unit1.id()) == 0; }); cache = resourceView.GetAgentCacheMap(); EXPECT_TRUE(cache.find(unit1.id()) == cache.end()); EXPECT_EQ(count, 1); @@ -3371,7 +3440,7 @@ TEST_F(ResourceViewTest, SetBillingPodResourceInstance) child->ToReady(); auto add = parent->AddResourceUnitWithUrl(*child->GetFullResourceView().Get(), - litebus::GetActor(childNode +"-ResourceViewActor")->GetAID().Url()); + litebus::GetActor(childNode + "-ResourceViewActor")->GetAID().Url()); ASSERT_TRUE(add.Get().IsOk()); auto map = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetPodResourceMap(); EXPECT_EQ(map.size(), 0); @@ -3392,4 +3461,103 @@ TEST_F(ResourceViewTest, SetBillingPodResourceInstance) EXPECT_NE(map.find(unit.id()), map.end()); } +TEST_F(ResourceViewTest, ValidateSchedulerLevel) +{ + // 1.scheduler_level = Non-root domain + auto parent = resource_view::ResourceView::CreateResourceView(DOMAIN_RESOUCE_VIEW_ID, PARENT_PARAM); + EXPECT_EQ(parent->GetResourceInfo().Get().schedulerLevel, SCHEDULER_LEVEL::NON_ROOT_DOMAIN); + + // 2.scheduler_level = root domain + parent->UpdateIsHeader(true); + EXPECT_EQ(parent->GetResourceInfo().Get().schedulerLevel, SCHEDULER_LEVEL::ROOT_DOMAIN); + + // 3.scheduler_level = local + auto child = resource_view::ResourceView::CreateResourceView(LOCAL_RESOUCE_VIEW_ID, CHILD_PARAM); + EXPECT_EQ(child->GetResourceInfo().Get().schedulerLevel, SCHEDULER_LEVEL::LOCAL); +} + +TEST_F(ResourceViewTest, TryDelResourceUnitChange) +{ + + std::string childNode = LOCAL_RESOUCE_VIEW_ID; + auto child = resource_view::ResourceView::CreateResourceView(childNode, CHILD_PARAM); + + // 1.add resourceunit + auto unit = Get1DResourceUnit(); + auto agentId = unit.id(); + GenerateMinimumUnitBucketInfo(unit); + ASSERT_TRUE(child->AddResourceUnit(unit).Get().IsOk()); + + // 2.add resourceunit + auto unit2 = Get1DResourceUnit(); + GenerateMinimumUnitBucketInfo(unit2); + auto agentId2 = unit2.id(); + ASSERT_TRUE(child->AddResourceUnit(unit2).Get().IsOk()); + + // 3.modify resourceunit -- add instance1(shared) + auto inst1 = Get1DInstance(); + (*inst1.mutable_schedulerchain()->Add()) = agentId; + inst1.set_unitid(agentId); + std::map instances1; + instances1.emplace(inst1.instanceid(), resource_view::InstanceAllocatedInfo{ inst1, nullptr }); + ASSERT_TRUE(child->AddInstances(instances1).Get().IsOk()); + + // 4.modify resourceunit -- add instance2(shared) + auto inst2 = Get1DInstance(); + (*inst2.mutable_schedulerchain()->Add()) = agentId; + inst2.set_unitid(agentId); + std::map instances2; + instances2.emplace(inst2.instanceid(), resource_view::InstanceAllocatedInfo{ inst2, nullptr }); + ASSERT_TRUE(child->AddInstances(instances2).Get().IsOk()); + + auto change = child->GetResourceViewChanges(); + auto childView = child->GetFullResourceView().Get(); + EXPECT_EQ(change.Get()->startrevision(), 0); + EXPECT_EQ(change.Get()->endrevision(), childView->revision()); + + // 5.del instance1(shared) + std::vector ids1; + ids1.push_back(inst1.instanceid()); + ASSERT_TRUE(child->DeleteInstances(ids1).Get().IsOk()); + + // 6.del instance2(shared) + std::vector ids2; + ids2.push_back(inst2.instanceid()); + ASSERT_TRUE(child->DeleteInstances(ids2).Get().IsOk()); + + // Test for not del because condition is not satisfied + child->TryDelResourceUnitChange(2, "AAA"); + childView = child->GetFullResourceView().Get(); + EXPECT_TRUE(child->implActor_->versionChanges_.find(1) != child->implActor_->versionChanges_.end()); + + child->TryDelResourceUnitChange(100, childView->viewinittime()); + childView = child->GetFullResourceView().Get(); + EXPECT_TRUE(child->implActor_->versionChanges_.find(1) != child->implActor_->versionChanges_.end()); + + child->TryDelResourceUnitChange(2, childView->viewinittime()); + childView = child->GetFullResourceView().Get(); // need get lastest version + EXPECT_TRUE(child->implActor_->versionChanges_.find(1) == child->implActor_->versionChanges_.end()); + + // 7.update status + ASSERT_TRUE(child->UpdateUnitStatus(agentId, resource_view::UnitStatus::EVICTING).Get().IsOk()); + + // 8.modify resourceunit -- add instance3(monopoly) + auto inst3 = Get1DInstance(); + (*inst3.mutable_schedulerchain()->Add()) = agentId2; + inst3.mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + inst3.set_unitid(agentId); + std::map instances3; + instances3.emplace(inst3.instanceid(), resource_view::InstanceAllocatedInfo{ inst3, nullptr }); + ASSERT_TRUE(child->AddInstances(instances3).Get().IsOk()); + + // 9.del resourceunit + ASSERT_TRUE(child->DeleteResourceUnit(agentId).Get().IsOk()); + child->TryDelResourceUnitChange(4, childView->viewinittime()); + + childView = child->GetFullResourceView().Get(); // need get lastest version + EXPECT_TRUE(child->implActor_->versionChanges_.find(3) == child->implActor_->versionChanges_.end()); + EXPECT_TRUE(child->implActor_->versionChanges_.find(4) != child->implActor_->versionChanges_.end()); + +} + } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/common/resource_view/view_utils.h b/functionsystem/tests/unit/common/resource_view/view_utils.h index adef5df7918eaedaa22c06ec33b6271461154e40..4aa003a88afc477e12bb203580d9dac4c810eafd 100644 --- a/functionsystem/tests/unit/common/resource_view/view_utils.h +++ b/functionsystem/tests/unit/common/resource_view/view_utils.h @@ -21,7 +21,7 @@ #include "async/uuid_generator.hpp" #include "common/resource_view/resource_tool.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" namespace functionsystem::test::view_utils { @@ -248,6 +248,28 @@ inline resource_view::ResourceUnit Get1DResourceUnitWithSpecificNpuNumber(const return unit; } +inline resource_view::ResourceUnit Get1DResourceUnitWithSpecificNpuNumber(const std::vector &hbm, + const std::vector &latency, + const std::vector &stream, + const std::string cardType = DEFAULT_NPU_TYPE) +{ + resource_view::ResourceUnit unit; + auto id = "Test_ResID_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + unit.set_id(id); + (*unit.mutable_capacity()) = GetCpuMemNpuResources(cardType); + auto key = (*unit.mutable_capacity()->mutable_resources())[cardType].vectors().values(). + at(resource_view::HETEROGENEOUS_MEM_KEY).vectors().begin()->first; + (*unit.mutable_allocatable()) = GetCpuMemNpuResourcesWithSpecificNpuNumber(hbm, + latency, + stream, + cardType, key); + (*unit.mutable_actualuse()) = GetCpuMemNpuResourcesWithSpecificNpuNumber({ 0,0,0,0,0,0,0,0 }, + { 0,0,0,0,0,0,0,0 }, + {100,100,100,100,100,100,100,100}, + cardType, key); + return unit; +} + inline resource_view::ResourceUnit Get1DResourceUnitWithNpu(const std::string cardType = DEFAULT_NPU_TYPE) { resource_view::ResourceUnit unit; @@ -307,6 +329,55 @@ inline resource_view::Resources GetCpuMemResources() return rs; } +inline resource_view::Resource GetDiskResource( + const std::vector vectorValue = { 100, 100, 100 }, + std::string nodeid = litebus::uuid_generator::UUID::GetRandomUUID().ToString()) +{ + resource_view::Resource r; + r.set_name(resource_view::DISK_RESOURCE_NAME); + r.set_type(resource_view::ValueType::Value_Type_VECTORS); + + auto categories = r.mutable_vectors()->mutable_values(); + + auto &vectors1 = (*categories)[resource_view::DISK_RESOURCE_NAME]; + auto &vector1 = (*vectors1.mutable_vectors())[nodeid]; + for (int i = 0; i < static_cast(vectorValue.size()) ;i++) { + auto value = vectorValue[i]; + vector1.mutable_values()->Add(value); + resource_view::ResourceExtension extension; + extension.mutable_disk()->set_size(value); + extension.mutable_disk()->set_name("disk"+ std::to_string(i)); + extension.mutable_disk()->set_mountpoints("/tmp/abc" + std::to_string(i) + "/"); + + (*r.mutable_extensions()->Add()) = extension; + } + return r; +} + +inline resource_view::Resources GetDiskResources( + const std::vector vectorValue = { 100, 100, 100 }, + std::string nodeid = litebus::uuid_generator::UUID::GetRandomUUID().ToString()) +{ + resource_view::Resources rs; + (*rs.mutable_resources())[resource_view::DISK_RESOURCE_NAME] = GetDiskResource(vectorValue, nodeid); + return rs; +} + + +inline resource_view::ResourceUnit Get1DResourceUnitWithDisk( + const std::vector vectorValue = { 100, 100, 100 }, + std::string nodeid = litebus::uuid_generator::UUID::GetRandomUUID().ToString()) +{ + resource_view::ResourceUnit unit; + auto id = "Test_ResID_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + unit.set_id(id); + (*unit.mutable_capacity()) = GetDiskResources(vectorValue, nodeid); + (*unit.mutable_allocatable()) = GetDiskResources(vectorValue, nodeid); + std::vector actualuse(vectorValue.size(), 0); + (*unit.mutable_actualuse()) = GetDiskResources(actualuse, nodeid); + return unit; +} + inline resource_view::Resources GetCpuMemWithOtherEmptyResources() { resource_view::Resources rs; @@ -339,6 +410,14 @@ inline resource_view::InstanceInfo Get1DInstance() return GetInstanceWithResourceAndPriority(0, INST_SCALA_VALUE, INST_SCALA_VALUE); } +inline resource_view::InstanceInfo Get1DInstanceWithDiskResource(int diskSize = 100) +{ + resource_view::InstanceInfo ins = view_utils::Get1DInstance(); + (*ins.mutable_resources()->mutable_resources())[resource_view::DISK_RESOURCE_NAME] + .mutable_scalar()->set_value(diskSize); + return ins; +} + inline resource_view::InstanceInfo Get1DInstanceWithNpuResource(int hbm, int latency, int stream, std::string cardType = DEFAULT_NPU_TYPE) { diff --git a/functionsystem/tests/unit/common/rpc/client/grpc_client_test.cpp b/functionsystem/tests/unit/common/rpc/client/grpc_client_test.cpp index 71662d141e205f1ea884c168bd39068199f57cbf..cdceefb285bf6eda112eeb1f5d9cfa43b3eb2c18 100644 --- a/functionsystem/tests/unit/common/rpc/client/grpc_client_test.cpp +++ b/functionsystem/tests/unit/common/rpc/client/grpc_client_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "rpc/client/grpc_client.h" +#include "common/rpc/client/grpc_client.h" #include #include @@ -23,10 +23,10 @@ #include #include "async/future.hpp" -#include "common_flags/common_flags.h" -#include "logs/logging.h" -#include "files.h" -#include "hex/hex.h" +#include "common/common_flags/common_flags.h" +#include "common/logs/logging.h" +#include "common/utils/files.h" +#include "common/hex/hex.h" #include "etcd/api/etcdserverpb/rpc.grpc.pb.h" #include "etcd/server/etcdserver/api/v3election/v3electionpb/v3election.grpc.pb.h" #include "utils/port_helper.h" @@ -35,7 +35,7 @@ using namespace etcdserverpb; namespace functionsystem::test { -std::string GRPC_TEST_SERVER_ADDR = std::string("127.0.0.1:50001"); +std::string serverAddress_ = std::string("127.0.0.1:50001"); constexpr uint64_t GRPC_TEST_RSP_HEADER_CLUSTER_ID = 123456; constexpr int GRPC_TEST_WATCH_ID = 1234567; @@ -96,22 +96,7 @@ public: } }; -void RunServer(std::shared_ptr> p, grpc::Server **s) -{ - std::string serverAddr(GRPC_TEST_SERVER_ADDR); - TestEtcdKvService kvService; - TestEtcdWatchService watchService; - - grpc::ServerBuilder builder; - builder.AddListeningPort(serverAddr, grpc::InsecureServerCredentials()); - builder.RegisterService(&kvService); - builder.RegisterService(&watchService); - std::unique_ptr server(builder.BuildAndStart()); - YRLOG_DEBUG("server listening on {}", serverAddr); - *s = server.get(); - p->SetValue(true); - server->Wait(); -} + static int MakeTmpDir(const std::string &path = "tmp") { @@ -123,8 +108,11 @@ static int MakeTmpDir(const std::string &path = "tmp") grpc::Server *g_server = nullptr; class GrpcClientTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { + serverAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + YRLOG_INFO("Start GRPC Server on net port: {}", serverAddress_); + auto p = std::make_shared>(); YRLOG_DEBUG("start test grpc server {}", "test"); auto t = std::thread(RunServer, p, &g_server); @@ -132,7 +120,7 @@ protected: p->GetFuture().Get(); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { if (g_server != nullptr) { g_server->Shutdown(); @@ -140,13 +128,32 @@ protected: } } - void SetUp() + void SetUp() override + { + } + + void TearDown() override { } - void TearDown() + static void RunServer(std::shared_ptr> p, grpc::Server **s) { + std::string serverAddr(serverAddress_); + TestEtcdKvService kvService; + TestEtcdWatchService watchService; + + grpc::ServerBuilder builder; + builder.AddListeningPort(serverAddr, grpc::InsecureServerCredentials()); + builder.RegisterService(&kvService); + builder.RegisterService(&watchService); + std::unique_ptr server(builder.BuildAndStart()); + YRLOG_DEBUG("server listening on {}", serverAddr); + *s = server.get(); + p->SetValue(true); + server->Wait(); } + + inline static std::string serverAddress_; private: }; @@ -194,7 +201,7 @@ TEST_F(GrpcClientTest, GrpcEtcdKvPutSuccess) req.set_key("test_key"); PutResponse rsp; - auto c = GrpcClient::CreateGrpcClient(GRPC_TEST_SERVER_ADDR); + auto c = GrpcClient::CreateGrpcClient(serverAddress_); ASSERT_TRUE(c); auto s = c->Call("Test::test etcd kv put ", req, rsp, &etcdserverpb::KV::Stub::Put, 10); @@ -212,7 +219,7 @@ TEST_F(GrpcClientTest, GrpcEtcdKvPutSuccess2) req.set_key("test_key"); PutResponse rsp; - auto c = GrpcClient::CreateGrpcClient(GRPC_TEST_SERVER_ADDR); + auto c = GrpcClient::CreateGrpcClient(serverAddress_); ASSERT_TRUE(c); auto s = c->CallAsync("Test::test etcd kv async put", req, rsp, &etcdserverpb::KV::Stub::AsyncPut, 10); @@ -230,7 +237,7 @@ TEST_F(GrpcClientTest, GrpcEtcdKvPutSuccess3) req.set_key("test_key"); PutResponse rsp; - auto c = GrpcClient::CreateGrpcClient(GRPC_TEST_SERVER_ADDR); + auto c = GrpcClient::CreateGrpcClient(serverAddress_); ASSERT_TRUE(c); litebus::Promise donePromise; @@ -254,7 +261,7 @@ TEST_F(GrpcClientTest, GrpcEtcdKvPutSuccess3) TEST_F(GrpcClientTest, GrpcEtcdWatchCallFailed) { - auto c = GrpcClient::CreateGrpcClient(GRPC_TEST_SERVER_ADDR); + auto c = GrpcClient::CreateGrpcClient(serverAddress_); ASSERT_TRUE(c); using StubFunc = @@ -270,7 +277,7 @@ TEST_F(GrpcClientTest, GrpcEtcdWatchWriteSuccess) auto cReq = req.mutable_create_request(); cReq->set_watch_id(GRPC_TEST_WATCH_ID); - auto c = GrpcClient::CreateGrpcClient(GRPC_TEST_SERVER_ADDR); + auto c = GrpcClient::CreateGrpcClient(serverAddress_); ASSERT_TRUE(c); grpc::ClientContext context; diff --git a/functionsystem/tests/unit/common/rpc/stream/grpc_stream_test.cpp b/functionsystem/tests/unit/common/rpc/stream/grpc_stream_test.cpp index b3664dce46fa42fa2fbc1258548cf95ca7e5bc97..e1eff5d5997bf5d217895b5ad8550495875da4c2 100644 --- a/functionsystem/tests/unit/common/rpc/stream/grpc_stream_test.cpp +++ b/functionsystem/tests/unit/common/rpc/stream/grpc_stream_test.cpp @@ -20,14 +20,16 @@ #include #include -#include "rpc/stream/posix/control_client.h" -#include "rpc/stream/posix/control_server.h" -#include "status/status.h" +#include "utils/port_helper.h" +#include "common/rpc/stream/posix/control_client.h" +#include "common/rpc/stream/posix/control_server.h" +#include "common/status/status.h" #include "utils/future_test_helper.h" namespace functionsystem::test { using namespace functionsystem::grpc; using namespace runtime_rpc; + litebus::Future> InvokeHandler(const std::string &from, const std::shared_ptr &request) { @@ -79,7 +81,7 @@ litebus::Future> NotifyServerHandler(const std class StreamTest : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { REGISTER_FUNCTION_SYS_POSIX_CONTROL_HANDLER(StreamingMessage::kInvokeReq, &InvokeHandler); REGISTER_FUNCTION_SYS_POSIX_CONTROL_HANDLER(StreamingMessage::kCallResultReq, &CallResultHandler); @@ -87,10 +89,11 @@ public: REGISTER_RUNTIME_CONTROL_POSIX_HANDLER(StreamingMessage::kCallReq, &CallServerHandler); REGISTER_RUNTIME_CONTROL_POSIX_HANDLER(StreamingMessage::kNotifyReq, &NotifyServerHandler); + serverAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); StartServerAndClient(); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { ShutdownServerAndClient(); } @@ -100,25 +103,26 @@ public: service_ = std::make_shared(); litebus::Promise promise; thr_ = std::thread([promise]() { - std::string serverAddress("127.0.0.1:12345"); ::grpc::ServerBuilder builder; - builder.AddListeningPort(serverAddress, ::grpc::InsecureServerCredentials()); + YRLOG_INFO("Server listening on {}", serverAddress_); + builder.AddListeningPort(serverAddress_, ::grpc::InsecureServerCredentials()); builder.RegisterService(service_.get()); server_ = std::move(builder.BuildAndStart()); - std::cout << "Server listening on " << serverAddress << std::endl; promise.SetValue(true); server_->Wait(); std::cout << "Server exit." << std::endl; }); promise.GetFuture().Get(); - grpc::ControlClientConfig config{ .target = "127.0.0.1:12345", - .creds = ::grpc::InsecureChannelCredentials(), - .timeoutSec = 30, - .maxGrpcSize = 5 }; + grpc::ControlClientConfig config{ + .target = serverAddress_, + .creds = ::grpc::InsecureChannelCredentials(), + .timeoutSec = 30, + .maxGrpcSize = 5 + }; client_ = std::make_shared("tmpInstance", "runtimeID", config); client_->Start(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for client to connect to server + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for client to connect to server } static void ShutdownServerAndClient() @@ -141,6 +145,7 @@ public: protected: inline static std::thread thr_; + inline static std::string serverAddress_; inline static std::unique_ptr<::grpc::Server> server_; inline static std::shared_ptr service_; inline static std::shared_ptr client_; @@ -201,7 +206,7 @@ TEST_F(StreamTest, ServerFinishTest) auto future = service_->Send(request); EXPECT_TRUE(future.IsError()); - Restart(); // restart server and client + Restart(); // restart server and client } TEST_F(StreamTest, PosixInvokeClientInValidTest) @@ -284,7 +289,7 @@ TEST_F(StreamTest, PosixInvokeClientMsgSizeTest) ASSERT_AWAIT_SET(future); EXPECT_EQ(future.IsError(), true); - Restart(); // restart server and client + Restart(); // restart server and client } class InvocationService : public runtime_rpc::RuntimeRPC::Service { @@ -308,11 +313,14 @@ public: class StreamTestV2 : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { + serverAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + YRLOG_INFO("Start GRPC Server on net port: {}", serverAddress_); + litebus::Promise promise; auto thr = std::thread([promise]() { - std::string serverAddress("127.0.0.1:50000"); + std::string serverAddress(serverAddress_); ::grpc::ServerBuilder builder; builder.AddListeningPort(serverAddress, ::grpc::InsecureServerCredentials()); builder.RegisterService(&service_); @@ -324,14 +332,15 @@ public: }); thr.detach(); promise.GetFuture().Get(); - grpc::ControlClientConfig config{ .target = "127.0.0.1:50000", + grpc::ControlClientConfig config{ .target = serverAddress_, .creds = ::grpc::InsecureChannelCredentials(), .timeoutSec = 30, .maxGrpcSize = 4 }; client_ = std::make_shared("tmpInstance", "runtimeID", config); client_->Start(); } - static void TearDownTestCase() + + [[maybe_unused]] static void TearDownTestSuite() { server_->Shutdown(); client_->Stop(); @@ -339,7 +348,7 @@ public: void Invoke() { - grpc::ControlClientConfig config{ .target = "127.0.0.1:50000", + grpc::ControlClientConfig config{ .target = serverAddress_, .creds = ::grpc::InsecureChannelCredentials(), .timeoutSec = 30, .maxGrpcSize = 4 }; @@ -361,6 +370,7 @@ protected: inline static std::unique_ptr<::grpc::Server> server_; inline static InvocationService service_; inline static std::shared_ptr client_; + inline static std::string serverAddress_; }; TEST_F(StreamTestV2, PosixCallServerValidTest) @@ -374,5 +384,4 @@ TEST_F(StreamTestV2, PosixCallServerValidTest) EXPECT_EQ(result.has_callrsp(), true); EXPECT_EQ(result.callrsp().code(), common::ErrorCode::ERR_NONE); } - -} // namespace functionsystem::test +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/schedule_decision/aggregated_queue_test.cpp b/functionsystem/tests/unit/common/schedule_decision/aggregated_queue_test.cpp index 6d512b67832c07c14c3d7bd2e366a981a13f79d7..6b3e48bfea7a123949a0035e3682b09ae113e506 100644 --- a/functionsystem/tests/unit/common/schedule_decision/aggregated_queue_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/aggregated_queue_test.cpp @@ -18,7 +18,7 @@ #include "common/schedule_decision/queue/aggregated_queue.h" #include "common/schedule_decision/queue/schedule_queue.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/common/schedule_decision/preemption_controller_test.cpp b/functionsystem/tests/unit/common/schedule_decision/preemption_controller_test.cpp index 61cb66732f5142a533841b93c3d341a806208975..44b5bc6e379dc8d1f31aba1a31405a0d47d0f148 100644 --- a/functionsystem/tests/unit/common/schedule_decision/preemption_controller_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/preemption_controller_test.cpp @@ -99,6 +99,26 @@ TEST_F(PreemptionControllerTest, InvalidParemTest) EXPECT_EQ(result.status.StatusCode(), StatusCode::PARAMETER_ERROR); } +// test for unable to preempt with cross tenant +TEST_F(PreemptionControllerTest, UnableToPreemptWithCrossTenant) +{ + auto pod1 = view_utils::Get1DResourceUnit("pod1"); + resourceView_->AddResourceUnit(pod1); + auto instance1 = view_utils::Get1DInstance(); + instance1.set_unitid("pod1"); + instance1.mutable_scheduleoption()->set_priority(1); + instance1.add_labels("tenantId:123456"); + resourceView_->AddInstances({ { instance1.instanceid(), { instance1, nullptr } } }); + + auto scheduledInstance = GetInstanceWithResource("scheduledInstance", 5, 100.1, 100.1); + scheduledInstance.add_labels("tenantId:xxxxx"); + auto preemption = PreemptionController(); + auto unit = resourceView_->GetResourceView().Get(); + auto preContext = std::make_shared(); + auto result = preemption.PreemptDecision(preContext, scheduledInstance, *unit); + EXPECT_EQ(result.status.StatusCode(), StatusCode::DOMAIN_SCHEDULER_NO_PREEMPTABLE_INSTANCE); +} + // test for preemption failed with resource capacity not enough TEST_F(PreemptionControllerTest, PreemptionFailedWithCapNotEnough) { @@ -112,6 +132,30 @@ TEST_F(PreemptionControllerTest, PreemptionFailedWithCapNotEnough) EXPECT_EQ(result.status.StatusCode(), StatusCode::DOMAIN_SCHEDULER_NO_PREEMPTABLE_INSTANCE); } +// test for preemption failed with instance required anti affinity +TEST_F(PreemptionControllerTest, PreemptionFailedWithInstanceRequiredAntiAffinity) +{ + auto pod1 = view_utils::Get1DResourceUnit("pod1"); + resourceView_->AddResourceUnit(pod1); + auto instance1 = view_utils::Get1DInstance(); + instance1.set_unitid("pod1"); + instance1.mutable_scheduleoption()->set_priority(1); + instance1.mutable_scheduleoption()->set_preemptedallowed(true); + instance1.add_labels("key1"); + resourceView_->AddInstances({ { instance1.instanceid(), { instance1, nullptr } } }); + + auto scheduledInstance = GetInstanceWithResource("scheduledInstance", 5, 100.1, 100.1); + scheduledInstance.add_labels("tenantId:xxxxx"); + auto instanceAffinity = scheduledInstance.mutable_scheduleoption()->mutable_affinity()->mutable_instance(); + (*instanceAffinity->mutable_requiredantiaffinity()) = Selector(false, { { Exist("key1") } }); + + auto preemption = PreemptionController(); + auto unit = resourceView_->GetResourceView().Get(); + auto preContext = std::make_shared(); + auto result = preemption.PreemptDecision(preContext, scheduledInstance, *unit); + EXPECT_EQ(result.status.StatusCode(), StatusCode::DOMAIN_SCHEDULER_NO_PREEMPTABLE_INSTANCE); +} + // test for preemption failed with instance required affinity TEST_F(PreemptionControllerTest, PreemptionFailedWithInstanceRequiredAffinity) { @@ -125,8 +169,9 @@ TEST_F(PreemptionControllerTest, PreemptionFailedWithInstanceRequiredAffinity) resourceView_->AddInstances({ { instance1.instanceid(), { instance1, nullptr } } }); auto scheduledInstance = GetInstanceWithResource("scheduledInstance", 5, 100.1, 100.1); + scheduledInstance.add_labels("tenantId:xxxxx"); auto instanceAffinity = scheduledInstance.mutable_scheduleoption()->mutable_affinity()->mutable_instance(); - (*instanceAffinity->mutable_requiredantiaffinity()) = Selector(false, { { Exist("key1") } }); + (*instanceAffinity->mutable_requiredaffinity()) = Selector(false, { { Exist("key1") } }); auto preemption = PreemptionController(); auto unit = resourceView_->GetResourceView().Get(); diff --git a/functionsystem/tests/unit/common/schedule_decision/priority_scheduler_test.cpp b/functionsystem/tests/unit/common/schedule_decision/priority_scheduler_test.cpp index 45bee74f95ff7c846f4f6b4bbe4592c973d4d6ee..f1344c5c3bd6b8debae509d87b92793f41d0ecb6 100644 --- a/functionsystem/tests/unit/common/schedule_decision/priority_scheduler_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/priority_scheduler_test.cpp @@ -22,6 +22,7 @@ #include "common/schedule_decision/queue/queue_item.h" #include "common/scheduler_framework/utils/label_affinity_selector.h" #include "mocks/mock_schedule_performer.h" +#include "utils/future_test_helper.h" namespace functionsystem::test { using namespace ::testing; @@ -88,8 +89,7 @@ TEST_F(PrioritySchedulerTest, ConsumeCompleteTest) auto group2 = GroupItem::CreateGroupItem("group2"); scheduler->Enqueue(ins1); scheduler->Enqueue(ins2); - scheduler->Enqueue(group1); - scheduler->Enqueue(group2); + EXPECT_FALSE(scheduler->CheckIsRunningQueueEmpty()); EXPECT_TRUE(scheduler->CheckIsPendingQueueEmpty()); @@ -102,6 +102,10 @@ TEST_F(PrioritySchedulerTest, ConsumeCompleteTest) scheduler->ConsumeRunningQueue(); EXPECT_EQ(ins1->schedulePromise->GetFuture().Get().code, 0); EXPECT_EQ(ins2->schedulePromise->GetFuture().Get().code, StatusCode::INVALID_RESOURCE_PARAMETER); + + scheduler->Enqueue(group1); + scheduler->Enqueue(group2); + scheduler->ConsumeRunningQueue(); EXPECT_EQ(group1->groupPromise->GetFuture().Get().code, 0); EXPECT_EQ(group2->groupPromise->GetFuture().Get().code, StatusCode::INVALID_RESOURCE_PARAMETER); EXPECT_TRUE(scheduler->CheckIsRunningQueueEmpty()); @@ -180,8 +184,8 @@ TEST_F(PrioritySchedulerTest, ConsumeCancelTest) EXPECT_CALL(*mockGroupPerformer_, RollBack).WillOnce(Return(Status::OK())); scheduler->ConsumeRunningQueue(); - EXPECT_TRUE(ins1->schedulePromise->GetFuture().IsInit()); - EXPECT_TRUE(group1->groupPromise->GetFuture().IsInit()); + EXPECT_TRUE(ins1->schedulePromise->GetFuture().IsOK()); + EXPECT_TRUE(group1->groupPromise->GetFuture().IsOK()); } // FIFO and Fairness policy exhibit consistent behavior @@ -200,13 +204,14 @@ TEST_F(PrioritySchedulerTest, AggregatedConsumeCancelTest) scheduler->Enqueue(group1); auto group2 = GroupItem::CreateGroupItem("group2"); scheduler->Enqueue(group2); + std::shared_ptr captureIns = nullptr; EXPECT_CALL(*mockAggregatedSchedulePerformer_, DoSchedule) - .WillOnce(Invoke([]( + .WillOnce(Invoke([&captureIns]( const std::shared_ptr &context, const resource_view::ResourceViewInfo &resourceInfo, const std::shared_ptr &aggregatedItem) { - auto instance1 = aggregatedItem->reqQueue->front(); - instance1->cancelTag.SetValue("cancel"); + captureIns = aggregatedItem->reqQueue->front(); + captureIns->cancelTag.SetValue("cancel"); return std::make_shared > (std::initializer_list{ ScheduleResult{"", 0, ""} @@ -223,8 +228,8 @@ TEST_F(PrioritySchedulerTest, AggregatedConsumeCancelTest) EXPECT_CALL(*mockGroupPerformer_, RollBack).WillOnce(Return(Status::OK())); scheduler->ConsumeRunningQueue(); - EXPECT_TRUE(ins1->schedulePromise->GetFuture().IsInit()); - EXPECT_TRUE(group1->groupPromise->GetFuture().IsInit()); + EXPECT_TRUE(captureIns->schedulePromise->GetFuture().IsOK()); + EXPECT_TRUE(group2->groupPromise->GetFuture().IsOK()); } // FIFO and Fairness policy exhibit consistent behavior diff --git a/functionsystem/tests/unit/common/schedule_decision/schedule_benchmark_test.cpp b/functionsystem/tests/unit/common/schedule_decision/schedule_benchmark_test.cpp index 2925f5a642c9cc7bcd756e60bab2c0261084dbae..37c32af5b1b30596d5b5401263e29029d49d02ac 100644 --- a/functionsystem/tests/unit/common/schedule_decision/schedule_benchmark_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/schedule_benchmark_test.cpp @@ -38,7 +38,7 @@ #include "common/scheduler_framework/framework/policy.h" #include "common/scheduler_framework/framework/framework_impl.h" #include "common/schedule_plugin/common/preallocated_context.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/schedule_plugin/prefilter/default_prefilter/default_prefilter.h" #include "common/schedule_plugin/filter/default_filter/default_filter.h" diff --git a/functionsystem/tests/unit/common/schedule_decision/schedule_queue_test.cpp b/functionsystem/tests/unit/common/schedule_decision/schedule_queue_test.cpp index 74fb7f241eb88b11b353e4771ba93b1b9ca267a6..9506c5e739abb03f0b900aeef0d2a6434ae98838 100644 --- a/functionsystem/tests/unit/common/schedule_decision/schedule_queue_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/schedule_queue_test.cpp @@ -16,7 +16,7 @@ #include #include "common/schedule_decision/queue/schedule_queue.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" namespace functionsystem::test { using namespace schedule_decision; diff --git a/functionsystem/tests/unit/common/schedule_decision/time_sorted_queue_test.cpp b/functionsystem/tests/unit/common/schedule_decision/time_sorted_queue_test.cpp index b3ca83c2b2bf7b90a9b29e4b62acb19b4985ef13..79c82ca0e5796b04250e03baea6568f8a85a9f8f 100644 --- a/functionsystem/tests/unit/common/schedule_decision/time_sorted_queue_test.cpp +++ b/functionsystem/tests/unit/common/schedule_decision/time_sorted_queue_test.cpp @@ -1,6 +1,18 @@ /* -* Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. -*/ + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "gtest/gtest.h" #define private public diff --git a/functionsystem/tests/unit/common/schedule_framework/framework_impl_test.cpp b/functionsystem/tests/unit/common/schedule_framework/framework_impl_test.cpp index f3c69f00830532e6c004b8682cb1ef45f93584ec..9e518b31bffffe970c50600e1d2c403d9d854e90 100644 --- a/functionsystem/tests/unit/common/schedule_framework/framework_impl_test.cpp +++ b/functionsystem/tests/unit/common/schedule_framework/framework_impl_test.cpp @@ -19,7 +19,7 @@ #include -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" #include "common/scheduler_framework/framework/framework.h" #include "common/scheduler_framework/framework/policy.h" @@ -30,12 +30,12 @@ using namespace ::testing; std::unique_ptr fwk = nullptr; class FrameworkImplTest : public ::testing::Test { protected: - static void SetUpTestSuite() + [[maybe_unused]] static void SetUpTestSuite() { fwk = std::make_unique(); } - static void TearDownTestSuite() + [[maybe_unused]] static void TearDownTestSuite() { fwk = nullptr; } @@ -391,6 +391,56 @@ TEST_F(FrameworkImplTest, ScoreSortedTest) result.sortedFeasibleNodes.pop(); } +TEST_F(FrameworkImplTest, AddScoreTest) +{ + auto fw = std::make_unique(-1); + auto ctx = std::make_shared(); + auto instance = MakeDefaultTestInstanceInfo(); + auto resource = MakeMultiFragmentTestResourceUnit(1); + + auto mockPrefilter = DefaultPrefilter(resource); + auto mockFilter = std::make_shared(); + EXPECT_CALL(*mockFilter, GetPluginName()).WillRepeatedly(Return("mockFilter")); + EXPECT_CALL(*mockFilter, Filter(_, _, _)).WillRepeatedly(Return(Filtered{})); + + std::map scoreList = { { "0", 5 } }; + std::map scoreLis1 = { { "0", 30 } }; + + auto mockScore = std::make_shared(); + EXPECT_CALL(*mockScore, GetPluginName()).WillRepeatedly(Return("mockScore")); + auto nodeScore = NodeScore(5); + nodeScore.heteroProductName = "NPU/910B4"; + nodeScore.realIDs = { 0 }; + nodeScore.vectorAllocations.push_back(schedule_framework::VectorResourceAllocation{}); + nodeScore.vectorAllocations.push_back(schedule_framework::VectorResourceAllocation{}); + EXPECT_CALL(*mockScore, Score(_, _, _)).WillOnce(Return(nodeScore)); + + auto mockScore1 = std::make_shared(); + EXPECT_CALL(*mockScore1, GetPluginName()).WillRepeatedly(Return("mockScore1")); + auto nodeScore1 = NodeScore(30); + nodeScore1.heteroProductName = "NPU/310"; + nodeScore1.realIDs = { 1 }; + nodeScore1.vectorAllocations.push_back(schedule_framework::VectorResourceAllocation{}); + EXPECT_CALL(*mockScore1, Score(_, _, _)).WillOnce(Return(nodeScore1));; + fw->RegisterPolicy(mockPrefilter); + fw->RegisterPolicy(mockFilter); + fw->RegisterPolicy(mockScore); + fw->RegisterPolicy(mockScore1); + EXPECT_EQ(mockFilter->GetPluginType(), PolicyType::FILTER_POLICY); + EXPECT_EQ(mockScore->GetPluginType(), PolicyType::SCORE_POLICY); + + auto result = fw->SelectFeasible(ctx, instance, resource, 0); + EXPECT_EQ(result.sortedFeasibleNodes.size(), (size_t)1); + auto top = result.sortedFeasibleNodes.top(); + EXPECT_EQ(top.name, "0"); + EXPECT_EQ(top.score, scoreList[top.name] + scoreLis1[top.name]); + // validate heterogeneous info -- overwrite mode + EXPECT_EQ(top.heteroProductName, "NPU/310"); + EXPECT_EQ(top.realIDs, std::vector{1}); + // validate vector allocation info -- append mode + EXPECT_EQ(top.vectorAllocations.size(), 3); +} + // multi filter return diff available TEST_F(FrameworkImplTest, AvailableForRequestTest) { diff --git a/functionsystem/tests/unit/common/schedule_plugin/filter/disk_filter_test.cpp b/functionsystem/tests/unit/common/schedule_plugin/filter/disk_filter_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..57eb167c312295fdba6a6a341fe1b29ba8677837 --- /dev/null +++ b/functionsystem/tests/unit/common/schedule_plugin/filter/disk_filter_test.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/schedule_plugin/filter/disk_filter/disk_filter.h" + +#include +#include + +#include "common/resource_view/view_utils.h" +#include "common/schedule_plugin/common/preallocated_context.h" + +namespace functionsystem::test { +using namespace ::testing; +using namespace functionsystem::schedule_plugin::filter; +using namespace functionsystem::schedule_framework; + +const std::string DEFAULT_NODE_ID = "test_node"; + +class DiskFilterTest : public Test {}; + +// Test disk scheduling success due to sufficient resources +TEST(DiskFilterTest, SuccessWhenSufficientDiskResource) { + + DiskFilter filter; + + // Subcase 1: Exact match scenario + // Instance requests exactly the available disk capacity (100) + { + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::SUCCESS); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 1); + } + + // Subcase 2: Partial match scenario + // Resource view contains two disks: + // - First disk capacity (40) is insufficient for request (100) + // - Second disk capacity (200) can satisfy the request + { + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 40, 200 }); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0, 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::SUCCESS); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 1); + } + + // Subcase 3: Non-disk request + { + auto instance = view_utils::Get1DInstance(); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::SUCCESS); + EXPECT_FALSE(filter.Filter(preAllocated, instance, unit).isFatalErr); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 0); + } +} + + +// Test disk scheduling failure due to insufficient resources +TEST(DiskFilterTest, FailedWhenInsufficientDiskResource) +{ + DiskFilter filter; + + // Subcase 1: Initial empty disk resources (request 200) + { + auto instance = view_utils::Get1DInstanceWithDiskResource(200); + auto unit = view_utils::Get1DResourceUnit(); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::DISK_SCHEDULE_FAILED); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 0); + } + + // Subcase 2: Initial disk resources insufficient (request 200 > available 100) + { + auto instance = view_utils::Get1DInstanceWithDiskResource(200); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::DISK_SCHEDULE_FAILED); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 0); + } + + // Subcase 3: Disk insufficiency caused by pre-allocated resources + // (request 200 > remaining 0 && request 200 > remaining 100) + { + auto instance = view_utils::Get1DInstanceWithDiskResource(200); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 250, 200 }, DEFAULT_NODE_ID); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 250, 100 }, DEFAULT_NODE_ID); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::DISK_SCHEDULE_FAILED); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 0); + } + + + // Subcase 4: Zero-disk request + { + auto instance = view_utils::Get1DInstanceWithDiskResource(0); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0 }); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).status, StatusCode::PARAMETER_ERROR); + EXPECT_EQ(filter.Filter(preAllocated, instance, unit).availableForRequest, 0); + } +} + +// Abnormal scenario: Invalid context +TEST(DiskFilterTest, InvalidContext) { + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + DiskFilter filter; + auto status = filter.Filter(nullptr, instance, unit); + EXPECT_EQ(status.status, StatusCode::PARAMETER_ERROR); + EXPECT_TRUE(status.isFatalErr); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/schedule_plugin/filter/label_affinity_filter_test.cpp b/functionsystem/tests/unit/common/schedule_plugin/filter/label_affinity_filter_test.cpp index 7e0e44b9c4e4fd44265865c22a21f4218fabe109..5d4764888f2fa50b4a56c97586bed1e8afd8145d 100644 --- a/functionsystem/tests/unit/common/schedule_plugin/filter/label_affinity_filter_test.cpp +++ b/functionsystem/tests/unit/common/schedule_plugin/filter/label_affinity_filter_test.cpp @@ -19,9 +19,9 @@ #include #include -#include "constants.h" -#include "logs/logging.h" -#include "resource_type.h" +#include "common/constants/constants.h" +#include "common/logs/logging.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" #include "common/schedule_plugin/common/plugin_utils.h" #include "common/schedule_plugin/common/preallocated_context.h" @@ -55,6 +55,7 @@ TEST_F(LabelAffinityFilterTest, InstancRequiredAffinityInPodScopeTest) auto instance1 = view_utils::Get1DInstance(); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -118,10 +119,10 @@ TEST_F(LabelAffinityFilterTest, InstancRequiredAffinityInNodeScopeTest) auto agent3 = NewResourceUnit("agent3", { { "key2", "value2" } }); auto agent4 = NewResourceUnit("agent3", { { "key3", "value3" } }); auto local1 = NewResourceUnit("local1", {}); - agent1.set_ownerid(local1.id()); - agent2.set_ownerid(local1.id()); - agent3.set_ownerid(local1.id()); - agent4.set_ownerid(local1.id()); + agent1.set_ownerid(agent1.id()); + agent2.set_ownerid(agent2.id()); + agent3.set_ownerid(agent3.id()); + agent4.set_ownerid(agent4.id()); AddFragmentToUnit(local1, agent1); AddFragmentToUnit(local1, agent2); AddFragmentToUnit(local1, agent3); @@ -145,6 +146,7 @@ TEST_F(LabelAffinityFilterTest, InstancRequiredAffinityInNodeScopeTest) (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledresult(); preAllocated->ClearUnfeasible(); preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::LOCAL; schedule_framework::Filtered result; result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); @@ -167,9 +169,9 @@ TEST_F(LabelAffinityFilterTest, InstancPreferredAffinityTest) auto agent2 = NewResourceUnit("agent2", { { "key2", "value2" } }); auto agent3 = NewResourceUnit("agent3", { { "key3", "value3" } }); auto local1 = NewResourceUnit("local1", {}); - agent1.set_ownerid(local1.id()); - agent2.set_ownerid(local1.id()); - agent3.set_ownerid(local1.id()); + agent1.set_ownerid(agent1.id()); + agent2.set_ownerid(agent2.id()); + agent3.set_ownerid(agent3.id()); AddFragmentToUnit(local1, agent1); AddFragmentToUnit(local1, agent2); AddFragmentToUnit(local1, agent3); @@ -180,6 +182,7 @@ TEST_F(LabelAffinityFilterTest, InstancPreferredAffinityTest) auto affinity = Selector(true, { { Exist("key1") }, { Exist("key2") } }); (*instanceAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -236,7 +239,7 @@ TEST_F(LabelAffinityFilterTest, InstancPreferredAffinityTest) auto instanceAffinity = instance1.mutable_scheduleoption()->mutable_affinity()->mutable_instance(); instanceAffinity->set_scope(affinity::NODE); - preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::LOCAL; schedule_framework::Filtered result; @@ -269,6 +272,7 @@ TEST_F(LabelAffinityFilterTest, TopDomnSchedulingSkipPreferredOptimalScoreTest) auto affinity = Selector(false, { { Exist("key1") } }); (*instanceAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -320,6 +324,7 @@ TEST_F(LabelAffinityFilterTest, ResourceRequiredAffinityTest) auto instance1 = view_utils::Get1DInstance(); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -396,6 +401,7 @@ TEST_F(LabelAffinityFilterTest, ResourcePreferredAffinityTest) auto affinity = Selector(true, { { Exist("key1") }, { Exist("key2") } }); (*resourceAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -490,6 +496,7 @@ TEST_F(LabelAffinityFilterTest, AllowPreemptPreferredAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; schedule_framework::Filtered result; result = relaxedNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); @@ -506,6 +513,7 @@ TEST_F(LabelAffinityFilterTest, AllowPreemptPreferredAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; schedule_framework::Filtered result; result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); @@ -559,6 +567,7 @@ TEST_F(LabelAffinityFilterTest, NotAllowPreemptPreferredAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; schedule_framework::Filtered result; result = relaxedNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); @@ -574,6 +583,7 @@ TEST_F(LabelAffinityFilterTest, NotAllowPreemptPreferredAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; schedule_framework::Filtered result; result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); @@ -606,6 +616,7 @@ TEST_F(LabelAffinityFilterTest, DataPreferredAffinityTest) auto affinity = Selector(true, { { Exist("key1") }, { Exist("key2") } }); (*dataAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -679,6 +690,7 @@ TEST_F(LabelAffinityFilterTest, PendingAffinityTest) { auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -713,6 +725,68 @@ TEST_F(LabelAffinityFilterTest, PendingAffinityTest) } } +TEST_F(LabelAffinityFilterTest, TenantPreferredAffinityTest) +{ + LabelAffinityFilter relaxedNonRootFilterPlugin(true, false); + LabelAffinityFilter strictNonRootFilterPlugin(false, false); + + auto agent1 = NewResourceUnit("agent1", { { "tenantId", "tenantA" } }); + auto agent2 = NewResourceUnit("agent2", { { "tenantId", "tenantB" } }); + auto agent3 = NewResourceUnit("agent3", {}); + auto local1 = NewResourceUnit("local1", {}); + agent1.set_ownerid(local1.id()); + agent2.set_ownerid(local1.id()); + agent3.set_ownerid(local1.id()); + AddFragmentToUnit(local1, agent1); + AddFragmentToUnit(local1, agent2); + AddFragmentToUnit(local1, agent3); + + auto instance1 = view_utils::Get1DInstance(); + auto tenantAffinity = instance1.mutable_scheduleoption()->mutable_affinity()->mutable_inner()->mutable_tenant(); + auto tenantRequiredAntiAffinity = Selector(false, { { NotIn("tenantId", { "tenantA" }), Exist("tenantId") } }); + (*tenantAffinity->mutable_requiredantiaffinity()) = std::move(tenantRequiredAntiAffinity); + auto tenantPreferrdAffinity = Selector(true, { { In("tenantId", { "tenantA" }) } }); + (*tenantAffinity->mutable_preferredaffinity()) = std::move(tenantPreferrdAffinity); + + auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + auto pluginCtx = messages::PluginContext(); + pluginCtx.mutable_affinityctx()->set_maxscore(100); + std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; + ::google::protobuf::Map map(tmp.begin(), tmp.end()); + preAllocated->pluginCtx = ↦ + + // 1.relaxed filter + { + preAllocated->ClearUnfeasible(); + schedule_framework::Filtered result; + + result = relaxedNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); + EXPECT_EQ(result.status.IsOk(), true); + EXPECT_EQ(result.availableForRequest, -1); + result = relaxedNonRootFilterPlugin.Filter(preAllocated, instance1, agent2); + EXPECT_EQ(result.status.IsOk(), false); + result = relaxedNonRootFilterPlugin.Filter(preAllocated, instance1, agent3); + EXPECT_EQ(result.status.IsOk(), true); + } + + // 2.strict filter + { + (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledscore(); + (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledresult(); + preAllocated->ClearUnfeasible(); + schedule_framework::Filtered result; + + result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent1); + EXPECT_EQ(result.status.IsOk(), true); + EXPECT_EQ(result.availableForRequest, -1); + result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent2); + EXPECT_EQ(result.status.IsOk(), false); + result = strictNonRootFilterPlugin.Filter(preAllocated, instance1, agent3); + EXPECT_EQ(result.status.IsOk(), false); + } +} + TEST_F(LabelAffinityFilterTest, ResourceGroupAffinityTest) { LabelAffinityFilter strictNonRootFilterPlugin(false, false); @@ -735,6 +809,7 @@ TEST_F(LabelAffinityFilterTest, ResourceGroupAffinityTest) { auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; diff --git a/functionsystem/tests/unit/common/schedule_plugin/scorer/default_heterogeneous_scorer_test.cpp b/functionsystem/tests/unit/common/schedule_plugin/scorer/default_heterogeneous_scorer_test.cpp index 65b174428237590f7a094763a984e45485141ef9..eec8620f3fcc3b5a702217fb2ea863375bd9178b 100644 --- a/functionsystem/tests/unit/common/schedule_plugin/scorer/default_heterogeneous_scorer_test.cpp +++ b/functionsystem/tests/unit/common/schedule_plugin/scorer/default_heterogeneous_scorer_test.cpp @@ -71,16 +71,16 @@ TEST(DefaultHeterogeneousScorerTest, TestHeteroScoringHBMAndLatencyAndStreamInHe auto instance = view_utils::Get1DInstanceWithNpuResource(30, 20, 1, "NPU/Ascend910B"); auto preAllocated = std::make_shared(); auto score = scorer.Score(preAllocated, instance, unit); - EXPECT_EQ(score.score, 41); // ((40 - 30)/40 + 0 + 99) / 3; - EXPECT_EQ(score.realIDs[0], 2); + EXPECT_EQ(score.score, 33); // int(30/30*100 + 0 + 0) / 3; + EXPECT_EQ(score.realIDs[0], 5); EXPECT_EQ(score.heteroProductName, "NPU/Ascend910B"); // 2.regex instance = view_utils::Get1DInstanceWithNpuResource(30, 20, 1, "NPU/Ascend910.*"); preAllocated = std::make_shared(); score = scorer.Score(preAllocated, instance, unit); - EXPECT_EQ(score.score, 41); // ((40 - 30)/40 + 0 + 99) / 3; - EXPECT_EQ(score.realIDs[0], 2); + EXPECT_EQ(score.score, 33); // int(30/30*100 + 0 + 0) / 3; + EXPECT_EQ(score.realIDs[0], 5); EXPECT_EQ(score.heteroProductName, "NPU/Ascend910B"); } diff --git a/functionsystem/tests/unit/common/schedule_plugin/scorer/disk_scorer_test.cpp b/functionsystem/tests/unit/common/schedule_plugin/scorer/disk_scorer_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8ca320b48849c93ac423f694c64b8b3a04de41b1 --- /dev/null +++ b/functionsystem/tests/unit/common/schedule_plugin/scorer/disk_scorer_test.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/schedule_plugin/scorer/disk_scorer/disk_scorer.h" + +#include +#include + +#include "common/resource_view/view_utils.h" +#include "common/schedule_plugin/common/preallocated_context.h" + +namespace functionsystem::test { +using namespace ::testing; +using namespace functionsystem::schedule_plugin::score; +using namespace functionsystem::schedule_framework; + +const std::string DEFAULT_NODE_ID = "test_node"; + +class DiskScorerTest : public Test {}; + +// Tests disk scoring when 50 out of 200 disk units are pre-allocated: +// 1. Verifies score calculation: (1 - requested/remaining) * 100 +// 2. Checks if resource allocation selects correct mount point ("/tmp/abc2/") +// 3. Validates output vector allocation metadata +TEST(DiskScorerTest, ScoreWithPartiallyAllocatedDisk) { + DiskScorer scorer; + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 20, 100, 200 }, DEFAULT_NODE_ID); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0, 0, 50 }, DEFAULT_NODE_ID); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + auto score = scorer.Score(preAllocated, instance, unit); + EXPECT_EQ(score.score, 33); // (1-100/(200-50))*100 + EXPECT_EQ(score.vectorAllocations.size(), 1); + auto vectorAllocation = score.vectorAllocations[0]; + EXPECT_EQ(vectorAllocation.selectedIndices.size(), 1); + EXPECT_EQ(vectorAllocation.selectedIndices[0], 2); + EXPECT_EQ(vectorAllocation.selectedIndices.size(), 1); + EXPECT_EQ(vectorAllocation.type, resource_view::DISK_RESOURCE_NAME); + auto diskMountPoint = vectorAllocation.extendedInfo.find(resource_view::DISK_MOUNT_POINT); + ASSERT_NE(diskMountPoint, vectorAllocation.extendedInfo.end()); + EXPECT_EQ(diskMountPoint->second, "/tmp/abc2/"); +} + +// Test disk scoring with floating-point disk requirement(100.5) +// Verifies score calculation: (1 - requested/remaining) * 100 +TEST(DiskScorerTest, ScoreWithFloatDiskReq) { + DiskScorer scorer; + auto instance = view_utils::Get1DInstanceWithDiskResource(100.5); + (*instance.mutable_resources()->mutable_resources())[resource_view::DISK_RESOURCE_NAME] + .mutable_scalar()->set_value(100.5); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 20, 100, 200 }, DEFAULT_NODE_ID); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 0, 0, 50 }, DEFAULT_NODE_ID); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + auto score = scorer.Score(preAllocated, instance, unit); + EXPECT_EQ(score.score, 33); // int((1-100.5/(200-50))*100) + EXPECT_EQ(score.vectorAllocations.size(), 1); + auto vectorAllocation = score.vectorAllocations[0]; + EXPECT_EQ(vectorAllocation.selectedIndices.size(), 1); + EXPECT_EQ(vectorAllocation.selectedIndices[0], 2); + EXPECT_EQ(vectorAllocation.selectedIndices.size(), 1); + EXPECT_EQ(vectorAllocation.type, resource_view::DISK_RESOURCE_NAME); + + std::vector expectAlloc{ 0, 0, 100.5 }; + auto allocCategory = vectorAllocation.allocationValues.values().find(resource_view::DISK_RESOURCE_NAME); + ASSERT_NE(allocCategory, vectorAllocation.allocationValues.values().end()); + EXPECT_EQ(allocCategory->second.vectors_size(), 1); + auto valuesAlloc = allocCategory->second.vectors().begin()->second; + EXPECT_EQ(valuesAlloc.values_size(), expectAlloc.size()); + for (size_t i = 0; i< expectAlloc.size(); i++) { + EXPECT_EQ(expectAlloc[i], valuesAlloc.values(i)); + } +} + +// abnormal scenario: Tests disk scoring with insufficient disk resource +TEST(DiskScorerTest, ScoreWithInsufficientDisk) { + DiskScorer scorer; + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 20, 100, 200 }, DEFAULT_NODE_ID); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 20, 100, 200 }, DEFAULT_NODE_ID); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + auto score = scorer.Score(preAllocated, instance, unit); + EXPECT_EQ(score.score, -1); + EXPECT_TRUE(score.vectorAllocations.empty()); +} + +// Tests disk scoring with Non-disk req +TEST(DiskScorerTest, ScoreWithNonDiskReq) { + DiskScorer scorer; + auto instance = view_utils::Get1DInstance(); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 20, 100, 200 }, DEFAULT_NODE_ID); + + auto preAllocated = std::make_shared(); + resource_view::Resources rs = view_utils::GetDiskResources({ 20, 100, 200 }, DEFAULT_NODE_ID); + preAllocated->allocated[unit.id()].resource = std::move(rs); + + auto score = scorer.Score(preAllocated, instance, unit); + EXPECT_EQ(score.score, 0); +} + +// abnormal scenario: Tests disk scoring with Non-disk resource +TEST(DiskScorerTest, ScoreWithNonDiskResource) { + DiskScorer scorer; + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnit(); + + auto preAllocated = std::make_shared(); + auto score = scorer.Score(preAllocated, instance, unit); + EXPECT_EQ(score.score, 0); +} + +// Abnormal scenario: Invalid context +TEST(DiskScorerTest, InvalidContext) { + auto instance = view_utils::Get1DInstanceWithDiskResource(100); + auto unit = view_utils::Get1DResourceUnitWithDisk({ 100 }); + + DiskScorer scorer; + auto score = scorer.Score(nullptr, instance, unit); + EXPECT_EQ(score.score, 0); +} + +} \ No newline at end of file diff --git a/functionsystem/tests/unit/common/schedule_plugin/scorer/label_affinity_scorer_test.cpp b/functionsystem/tests/unit/common/schedule_plugin/scorer/label_affinity_scorer_test.cpp index 9a3af5d833afc78e225ab284e7af4d3af811c42b..b844c3c2443cbe0facc70dba4e2072b34738dde6 100644 --- a/functionsystem/tests/unit/common/schedule_plugin/scorer/label_affinity_scorer_test.cpp +++ b/functionsystem/tests/unit/common/schedule_plugin/scorer/label_affinity_scorer_test.cpp @@ -19,9 +19,9 @@ #include #include -#include "constants.h" -#include "logs/logging.h" -#include "resource_type.h" +#include "common/constants/constants.h" +#include "common/logs/logging.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" #include "common/schedule_plugin/common/plugin_utils.h" #include "common/schedule_plugin/common/preallocated_context.h" @@ -42,9 +42,9 @@ TEST_F(LabelAffinityScorerTest, InstancAffinityTest) auto agent2 = NewResourceUnit("agent2", { { "key2", "value2" } }); auto agent3 = NewResourceUnit("agent3", { { "key3", "value3" } }); auto local1 = NewResourceUnit("local1", {}); - agent1.set_ownerid(local1.id()); - agent2.set_ownerid(local1.id()); - agent3.set_ownerid(local1.id()); + agent1.set_ownerid(agent1.id()); + agent2.set_ownerid(agent2.id()); + agent3.set_ownerid(agent3.id()); AddFragmentToUnit(local1, agent1); AddFragmentToUnit(local1, agent2); AddFragmentToUnit(local1, agent3); @@ -60,6 +60,7 @@ TEST_F(LabelAffinityScorerTest, InstancAffinityTest) (*instanceAffinity->mutable_requiredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(300); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -82,7 +83,7 @@ TEST_F(LabelAffinityScorerTest, InstancAffinityTest) instanceAffinity->set_scope(affinity::NODE); (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledscore(); - preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::LOCAL; result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); EXPECT_EQ(result.score, 200); @@ -97,7 +98,7 @@ TEST_F(LabelAffinityScorerTest, InstancAffinityTest) instance1.mutable_scheduleoption()->clear_affinity(); (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledscore(); - preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::LOCAL; result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); EXPECT_EQ(result.score, 1); @@ -133,6 +134,7 @@ TEST_F(LabelAffinityScorerTest, ResourceAffinityTest) (*resourceAffinity->mutable_requiredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(300); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -144,9 +146,9 @@ TEST_F(LabelAffinityScorerTest, ResourceAffinityTest) // 1.preferredaffinity with priority { result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); - EXPECT_EQ(result.score, 300); + EXPECT_EQ(result.score, 10200); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent2); - EXPECT_EQ(result.score, 280); + EXPECT_EQ(result.score, 9190); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent3); EXPECT_EQ(result.score, 0); } @@ -160,9 +162,9 @@ TEST_F(LabelAffinityScorerTest, ResourceAffinityTest) (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledscore(); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); - EXPECT_EQ(result.score, 300); + EXPECT_EQ(result.score, 10200); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent2); - EXPECT_EQ(result.score, 290); + EXPECT_EQ(result.score, 9200); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent3); EXPECT_EQ(result.score, 0); } @@ -209,6 +211,7 @@ TEST_F(LabelAffinityScorerTest, PreemptAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); EXPECT_EQ(result.score, 6); @@ -230,6 +233,7 @@ TEST_F(LabelAffinityScorerTest, PreemptAffinityTest) preAllocated->allLocalLabels["NodeA"] = NodeA.nodelabels(); preAllocated->allLocalLabels["NodeB"] = NodeB.nodelabels(); preAllocated->allLocalLabels["NodeC"] = NodeC.nodelabels(); + preAllocated->schedulerLevel = resource_view::SCHEDULER_LEVEL::NON_ROOT_DOMAIN; (*preAllocated->pluginCtx)[LABEL_AFFINITY_PLUGIN].mutable_affinityctx()->clear_scheduledscore(); result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); @@ -241,6 +245,47 @@ TEST_F(LabelAffinityScorerTest, PreemptAffinityTest) } } +TEST_F(LabelAffinityScorerTest, TenantAffinityTest) +{ + LabelAffinityScorer relaxedScorerPlugin(true); + + auto agent1 = NewResourceUnit("agent1", { { "tenantId", "tenantA" } }); + auto agent2 = NewResourceUnit("agent2", { { "tenantId", "tenantB" } }); + auto agent3 = NewResourceUnit("agent3", {}); + auto local1 = NewResourceUnit("local1", {}); + agent1.set_ownerid(local1.id()); + agent2.set_ownerid(local1.id()); + agent3.set_ownerid(local1.id()); + AddFragmentToUnit(local1, agent1); + AddFragmentToUnit(local1, agent2); + AddFragmentToUnit(local1, agent3); + + auto instance1 = view_utils::Get1DInstance(); + auto tenantAffinity = instance1.mutable_scheduleoption()->mutable_affinity()->mutable_inner()->mutable_tenant(); + auto tenantRequiredAntiAffinity = Selector(false, { { NotIn("tenantId", { "tenantA" }), Exist("tenantId") } }); + (*tenantAffinity->mutable_requiredantiaffinity()) = std::move(tenantRequiredAntiAffinity); + auto tenantPreferrdAffinity = Selector(true, { { In("tenantId", { "tenantA" }) } }); + (*tenantAffinity->mutable_preferredaffinity()) = std::move(tenantPreferrdAffinity); + + auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); + auto pluginCtx = messages::PluginContext(); + pluginCtx.mutable_affinityctx()->set_maxscore(100); + std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; + ::google::protobuf::Map map(tmp.begin(), tmp.end()); + preAllocated->pluginCtx = ↦ + + preAllocated->ClearUnfeasible(); + schedule_framework::NodeScore result(0); + + result = relaxedScorerPlugin.Score(preAllocated, instance1, agent1); + EXPECT_EQ(result.score, 100); + result = relaxedScorerPlugin.Score(preAllocated, instance1, agent2); + EXPECT_EQ(result.score, 0); + result = relaxedScorerPlugin.Score(preAllocated, instance1, agent3); + EXPECT_EQ(result.score, 0); +} + TEST_F(LabelAffinityScorerTest, DataAffinityTest) { LabelAffinityScorer relaxedScorerPlugin(true); @@ -261,6 +306,7 @@ TEST_F(LabelAffinityScorerTest, DataAffinityTest) auto affinity = Selector(true, { { Exist("key1") }, { Exist("key2") } }); (*dataAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(100); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -299,6 +345,7 @@ TEST_F(LabelAffinityScorerTest, SkipPreferredScoreTest) auto affinity = Selector(true, { { Exist("key1") }, { Exist("key2") } }); (*instanceAffinity->mutable_preferredaffinity()) = std::move(affinity); auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(666); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; @@ -387,6 +434,7 @@ TEST_F(LabelAffinityScorerTest, MultiAffinityTest) (*dataAffinity->mutable_preferredaffinity()) = affinity; auto preAllocated = std::make_shared(); + preAllocated->allLocalLabels[local1.id()] = local1.nodelabels(); auto pluginCtx = messages::PluginContext(); pluginCtx.mutable_affinityctx()->set_maxscore(400); std::map tmp{ { LABEL_AFFINITY_PLUGIN, pluginCtx } }; diff --git a/functionsystem/tests/unit/common/status/status_test.cpp b/functionsystem/tests/unit/common/status/status_test.cpp index a587aff1519629aadbcac06aca2678c53dfe2d6b..0ca1c7c1a817ee70d9298fd53bb7940c052fc870 100644 --- a/functionsystem/tests/unit/common/status/status_test.cpp +++ b/functionsystem/tests/unit/common/status/status_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "status/status.h" +#include "common/status/status.h" #include #include diff --git a/functionsystem/tests/unit/common/utils/actor_worker_test.cpp b/functionsystem/tests/unit/common/utils/actor_worker_test.cpp index e90082cdbd7c9d8d7fe6de835e80b75f0fad1b2b..66acc75ff47cbc2e0944c7f8eab371b0ee4e9ba7 100644 --- a/functionsystem/tests/unit/common/utils/actor_worker_test.cpp +++ b/functionsystem/tests/unit/common/utils/actor_worker_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "actor_worker.h" +#include "common/utils/actor_worker.h" #include diff --git a/functionsystem/tests/unit/common/utils/aksk_content_test.cpp b/functionsystem/tests/unit/common/utils/aksk_content_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3819aeafe0dc9bc1d3bceae2ba4e40679df231b8 --- /dev/null +++ b/functionsystem/tests/unit/common/utils/aksk_content_test.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/utils/aksk_content.h" +#include +#include + +namespace functionsystem::test { +class AKSKContentTest : public ::testing::Test {}; + +AKSKContent NewContent() +{ + AKSKContent content; + content.tenantID = "tenant"; + content.secretKey = "secret"; + content.accessKey = "access"; + content.dataKey = "data"; + content.expiredTimeStamp = static_cast(std::time(nullptr)) + 3000; + content.status = Status::OK(); + return content; +} + +TEST_F(AKSKContentTest, AKSKContentValid) +{ + auto content = NewContent(); + EXPECT_EQ(content.IsValid(3).IsOk(), true); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_status_err) +{ + auto content = NewContent(); + content.status = Status(StatusCode::FAILED); + EXPECT_FALSE(content.IsValid().IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_TenantID) +{ + auto content = NewContent(); + content.tenantID = std::string(); + EXPECT_FALSE(content.IsValid().IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_AccessKey) +{ + auto content = NewContent(); + content.accessKey = std::string(); + EXPECT_FALSE(content.IsValid().IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_SecretKey) +{ + auto content = NewContent(); + content.secretKey = std::string(); + EXPECT_FALSE(content.IsValid().IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_DataKey) +{ + auto content = NewContent(); + content.dataKey = std::string(); + EXPECT_FALSE(content.IsValid().IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_Expired) +{ + auto content = NewContent(); + content.expiredTimeStamp = 1; + EXPECT_FALSE(content.IsValid(1).IsOk()); +} + +TEST_F(AKSKContentTest, AKSKContentInValid_Copy) +{ + auto content = NewContent(); + auto copy = content.Copy(); + EXPECT_EQ(content.tenantID, copy->tenantID); + EXPECT_EQ(content.accessKey, copy->accessKey); + EXPECT_EQ(content.secretKey, copy->secretKey); + EXPECT_EQ(content.dataKey, copy->dataKey); + EXPECT_EQ(content.expiredTimeStamp, copy->expiredTimeStamp); +} +} \ No newline at end of file diff --git a/functionsystem/tests/unit/common/utils/files_test.cpp b/functionsystem/tests/unit/common/utils/files_test.cpp index 425ddde001ed7b498e4dc1debffb68f7ab0dcdad..4e815a09f46f774af5383e841986667aeca0c704 100644 --- a/functionsystem/tests/unit/common/utils/files_test.cpp +++ b/functionsystem/tests/unit/common/utils/files_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "files.h" +#include "common/utils/files.h" #include diff --git a/functionsystem/tests/unit/common/utils/grpc_auth_test.cpp b/functionsystem/tests/unit/common/utils/grpc_auth_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a910a7ba9f0880a1c04214451d768a017c764c64 --- /dev/null +++ b/functionsystem/tests/unit/common/utils/grpc_auth_test.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/aksk/aksk_util.h" +#include "gtest/gtest.h" + +namespace functionsystem::test { +class GrpcAuthTest : public ::testing::Test { +protected: + void SetUp() override + { + } + + void TearDown() override + { + } +}; + +TEST_F(GrpcAuthTest, SignAndVerifyStreamingMessageTest) +{ + std::string accessKey = "access_key"; + SensitiveValue secretKey("secret_key"); + auto message = std::make_shared(); + message->set_messageid("message_id"); + + // fail to sign empty StreamingMessage + EXPECT_FALSE(SignStreamingMessage(accessKey, secretKey, message)); + EXPECT_FALSE(message->metadata().contains("access_key")); + EXPECT_FALSE(message->metadata().contains("signature")); + EXPECT_FALSE(message->metadata().contains("timestamp")); + + // success sign + message->mutable_callreq()->set_requestid("123"); + EXPECT_TRUE(SignStreamingMessage(accessKey, secretKey, message)); + EXPECT_TRUE(message->metadata().contains("access_key")); + EXPECT_EQ(message->metadata().at("access_key"), accessKey); + EXPECT_TRUE(message->metadata().contains("signature")); + EXPECT_TRUE(message->metadata().contains("timestamp")); + + // success verify + EXPECT_TRUE(VerifyStreamingMessage(accessKey, secretKey, message)); + + EXPECT_FALSE(VerifyStreamingMessage("fake_access_key", secretKey, message)); + + SensitiveValue fakeSecretKey("fake_secret_key"); + EXPECT_FALSE(VerifyStreamingMessage(accessKey, fakeSecretKey, message)); + + // tamper message + message->mutable_callreq()->set_requestid("1234"); + EXPECT_FALSE(VerifyStreamingMessage(accessKey, secretKey, message)); + + // tamper message type + message->clear_callreq(); + message->mutable_callresultack()->set_message("123"); + EXPECT_FALSE(VerifyStreamingMessage(accessKey, secretKey, message)); + + message->Clear(); + EXPECT_FALSE(VerifyStreamingMessage(accessKey, secretKey, message)); +} + +TEST_F(GrpcAuthTest, SignAndVerifyTimestampTest) +{ + std::string accessKey = "access_key"; + SensitiveValue secretKey("secret_key"); + auto timestamp = litebus::time::GetCurrentUTCTime(); + auto signature = SignTimestamp(accessKey, secretKey, timestamp); + EXPECT_FALSE(signature.empty()); + + EXPECT_TRUE(VerifyTimestamp(accessKey, secretKey, timestamp, signature)); + + EXPECT_FALSE(VerifyTimestamp("fake_access_key", secretKey, timestamp, signature)); + + SensitiveValue fakeSecretKey("fake_secret_key"); + EXPECT_FALSE(VerifyTimestamp(accessKey, fakeSecretKey, timestamp, signature)); + + auto fakeTimestamp = litebus::time::GetCurrentUTCTime(); + EXPECT_FALSE(VerifyTimestamp(accessKey, fakeSecretKey, fakeTimestamp, signature)); + + EXPECT_FALSE(VerifyTimestamp(accessKey, fakeSecretKey, timestamp, "fake_signature")); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/utils/request_sync_helper_test.cpp b/functionsystem/tests/unit/common/utils/request_sync_helper_test.cpp index f6c66075aaf6bf58bb9b920029751d96540c5a64..0516404c0083459de3f83cb569bec893acd30019 100644 --- a/functionsystem/tests/unit/common/utils/request_sync_helper_test.cpp +++ b/functionsystem/tests/unit/common/utils/request_sync_helper_test.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "request_sync_helper.h" +#include "common/utils/request_sync_helper.h" #include diff --git a/functionsystem/tests/unit/common/utils/struct_transfer_test.cpp b/functionsystem/tests/unit/common/utils/struct_transfer_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..543c109e7a6a000adf7f815ac66048e1b312078f --- /dev/null +++ b/functionsystem/tests/unit/common/utils/struct_transfer_test.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/utils/struct_transfer.h" + +#include "gtest/gtest.h" +#include "common/resource_view/view_utils.h" + +namespace functionsystem::test { +const std::string DEFAULT_UNIT_ID = "test_node"; +const std::string DEFAULT_NODE_UUID = "test_uuid"; +const std::string DEFAULT_DISK_MOUNT_PATH = "/tmp/abc/"; +const int DEFAULT_DISK_INDEX = 1; +const std::string DEFAULT_VENDOR = "NPU"; +const std::string DEFAULT_CARD_TYPE = "NPU/Ascend310"; +const std::string RUNTIME_ENV_PREFIX = "func-"; +const std::vector DEFAULT_VECTOR_RESOURCE = { 0, 100, 0 }; +class StructTransferTest : public ::testing::Test {}; + +TEST_F(StructTransferTest, StaticFunction_TRUE) +{ + resources::InstanceInfo instanceInfo; + + auto *options = instanceInfo.mutable_createoptions(); + options->insert({ RESOURCE_OWNER_KEY, STATIC_FUNCTION_OWNER_VALUE }); + + EXPECT_TRUE(IsStaticFunctionInstance(instanceInfo)); +} + +TEST_F(StructTransferTest, StaticFunction_FALSE) +{ + resources::InstanceInfo instanceInfo; + EXPECT_FALSE(IsStaticFunctionInstance(instanceInfo)); +} + +void SetValidDiskAllocation(schedule_decision::ScheduleResult &result) +{ + schedule_framework::VectorResourceAllocation vectorAllocation; + vectorAllocation.type = resource_view::DISK_RESOURCE_NAME; + vectorAllocation.selectedIndices = { DEFAULT_DISK_INDEX }; + + auto &allocCategory = (*vectorAllocation.allocationValues.mutable_values())[resource_view::DISK_RESOURCE_NAME]; + for (const auto &value: DEFAULT_VECTOR_RESOURCE) { + (*allocCategory.mutable_vectors())[DEFAULT_NODE_UUID].add_values(value); + } + + vectorAllocation.extendedInfo[resource_view::DISK_MOUNT_POINT] = DEFAULT_DISK_MOUNT_PATH; + result.vectorAllocations.push_back(vectorAllocation); +} + +void SetValidHeteroAllocation(schedule_decision::ScheduleResult &result) +{ + result.realIDs = { DEFAULT_DISK_INDEX }; + + auto &allocCategory = (*result.allocatedVectors[DEFAULT_CARD_TYPE] + .mutable_values())[resource_view::HETEROGENEOUS_MEM_KEY]; + for (const auto &value: DEFAULT_VECTOR_RESOURCE) { + (*allocCategory.mutable_vectors())[DEFAULT_NODE_UUID].add_values(value); + } +} + +TEST_F(StructTransferTest, MergeScheduleResultToRequest) +{ + auto req = std::make_shared(); + auto scheduleInstance = view_utils::Get1DInstanceWithNpuResource(1); + *req->mutable_instance() = scheduleInstance; + req->set_requestid(scheduleInstance.requestid()); + req->set_traceid("traceID_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString()); + + schedule_decision::ScheduleResult result; + result.id = DEFAULT_UNIT_ID; + result.unitID = DEFAULT_UNIT_ID; + SetValidDiskAllocation(result); + SetValidHeteroAllocation(result); + + MergeScheduleResultToRequest(req, result); + + // validate basic info + EXPECT_EQ(req->instance().functionagentid(), result.id); + EXPECT_EQ(req->instance().unitid(), result.unitID); + const auto& schedulerChain = req->instance().schedulerchain(); + EXPECT_EQ(schedulerChain.size(), 1); + EXPECT_EQ(schedulerChain[0], result.id); + + const auto& createOptions = req->instance().createoptions(); + // validate heterogenous device id + auto heteroDeviceIds = createOptions.find("func-NPU-DEVICE-IDS"); + ASSERT_NE(heteroDeviceIds, createOptions.end()); + EXPECT_EQ(heteroDeviceIds->second, "1"); + + // validate heterogenous resource allocation + const auto& resources = req->instance().resources().resources(); + auto heteroResource = resources.find(DEFAULT_CARD_TYPE); + ASSERT_NE(heteroResource, resources.end()); + EXPECT_EQ(heteroResource->second.type(), resource_view::ValueType::Value_Type_VECTORS); + EXPECT_EQ(heteroResource->second.name(), DEFAULT_CARD_TYPE); + + const auto& heteroVectors = heteroResource->second.vectors().values(); + auto heteroCategory = heteroVectors.find(resource_view::HETEROGENEOUS_MEM_KEY); + ASSERT_NE(heteroCategory, heteroVectors.end()); + + auto hetegroVectors = heteroCategory->second.vectors().find(DEFAULT_NODE_UUID); + ASSERT_NE(hetegroVectors, heteroCategory->second.vectors().end()); + + EXPECT_EQ(hetegroVectors->second.values_size(), DEFAULT_VECTOR_RESOURCE.size()); + for (int i = 0; i < hetegroVectors->second.values_size(); i++) { + EXPECT_EQ(DEFAULT_VECTOR_RESOURCE[i], hetegroVectors->second.values(i)); + } + + // validate disk mount point + auto envKey = RUNTIME_ENV_PREFIX + resource_view::DISK_MOUNT_POINT; + auto diskMountPoint = createOptions.find(envKey); + ASSERT_NE(diskMountPoint, createOptions.end()); + EXPECT_EQ(diskMountPoint->second, DEFAULT_DISK_MOUNT_PATH); + + // validate disk resource allocation + auto diskResource = resources.find(resource_view::DISK_RESOURCE_NAME); + ASSERT_NE(diskResource, resources.end()); + EXPECT_EQ(diskResource->second.type(), resource_view::ValueType::Value_Type_VECTORS); + EXPECT_EQ(diskResource->second.name(), resource_view::DISK_RESOURCE_NAME); + + const auto& vectors = diskResource->second.vectors().values(); + auto diskCategory = vectors.find(resource_view::DISK_RESOURCE_NAME); + ASSERT_NE(diskCategory, vectors.end()); + + auto diskVectors = diskCategory->second.vectors().find(DEFAULT_NODE_UUID); + ASSERT_NE(diskVectors, diskCategory->second.vectors().end()); + + EXPECT_EQ(diskVectors->second.values_size(), DEFAULT_VECTOR_RESOURCE.size()); + for (int i = 0; i < diskVectors->second.values_size(); i++) { + EXPECT_EQ(DEFAULT_VECTOR_RESOURCE[i], diskVectors->second.values(i)); + } + +} + +TEST_F(StructTransferTest, SetMonopolyAffinityOptScope) +{ + auto scheduleReq = std::make_shared(); + + // 1.When instance affinity scope is not set --> scope == Node + { + auto instance = view_utils::Get1DInstance(); + instance.mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + auto createReq = CreateRequest(); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::NODE); + } + + // 2.When instance affinity is set to POD scope --> scope == Node + { + auto instance = view_utils::Get1DInstance(); + instance.mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + auto createReq = CreateRequest(); + createReq.mutable_schedulingops()->mutable_scheduleaffinity()->mutable_instance()->set_scope(affinity::POD); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::NODE); + } + + // 3.When instance affinity is set to Node scope --> scope == Node + { + auto instance = view_utils::Get1DInstance(); + instance.mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + auto createReq = CreateRequest(); + createReq.mutable_schedulingops()->mutable_scheduleaffinity()->mutable_instance()->set_scope(affinity::NODE); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::NODE); + } +} + +TEST_F(StructTransferTest, SetSharedAffinityOptScope) +{ + auto scheduleReq = std::make_shared(); + + // 1.When instance affinity scope is not set --> scope defaults to POD + { + auto instance = view_utils::Get1DInstance(); + auto createReq = CreateRequest(); + instance.mutable_scheduleoption()->set_schedpolicyname("shared"); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::POD); + } + + // 2.When instance affinity is set to POD scope --> scope == POD + { + auto instance = view_utils::Get1DInstance(); + instance.mutable_scheduleoption()->set_schedpolicyname("shared"); + auto createReq = CreateRequest(); + createReq.mutable_schedulingops()->mutable_scheduleaffinity()->mutable_instance()->set_scope(affinity::POD); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::POD); + } + + // 3. When instance affinity is set to NODE scope --> scope == NODE + { + auto instance = view_utils::Get1DInstance(); + instance.mutable_scheduleoption()->set_schedpolicyname("shared"); + auto createReq = CreateRequest(); + createReq.mutable_schedulingops()->mutable_scheduleaffinity()->mutable_instance()->set_scope(affinity::NODE); + SetAffinityOpt(instance, createReq, scheduleReq); + EXPECT_EQ(instance.scheduleoption().affinity().instance().scope(), affinity::NODE); + } +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/utils/utils_test.cpp b/functionsystem/tests/unit/common/utils/utils_test.cpp index 566194a1b251b28aabafad3bc59f4437c53bacb1..d9e25a33dc3258b5d1148a05ce1081ec7541508e 100644 --- a/functionsystem/tests/unit/common/utils/utils_test.cpp +++ b/functionsystem/tests/unit/common/utils/utils_test.cpp @@ -23,12 +23,14 @@ #include "common/utils/exception.h" #include "common/utils/exec_utils.h" -#include "files.h" -#include "meta_store_kv_operation.h" -#include "param_check.h" +#include "common/utils/files.h" +#include "common/utils/meta_store_kv_operation.h" +#include "common/utils/param_check.h" #include "common/utils/path.h" -#include "ssl_config.h" +#include "common/utils/ssl_config.h" #include "common/utils/struct_transfer.h" +#include "common/utils/random_number.h" +#include "common/utils/sensitive_value.h" #include "exec/exec.hpp" #include "utils/future_test_helper.h" @@ -734,4 +736,48 @@ TEST_F(UtilsTest, ParseValueFromKey) funcName = TrimKeyPrefix("/test/yr/functions/business/yrk/tenant/0/function/0-system-faascontroller/version/$latest", "/test"); EXPECT_EQ("/yr/functions/business/yrk/tenant/0/function/0-system-faascontroller/version/$latest", funcName); } + +/** +* @tc.name : ShuffleStringByDelimiter_ShouldReturnEmptyString_WhenInputIsEmpty +* @tc.number: ShuffleStringByDelimiter_Test_001 +* @tc.desc : Test ShuffleStringByDelimiter function with empty input string. + */ +TEST_F(UtilsTest, ShuffleStringByDelimiter_ShouldReturnEmptyString_WhenInputIsEmpty) { + std::string input = ""; + std::string pattern = ","; + std::string result = ShuffleStringByDelimiter(input, pattern); + EXPECT_EQ(result, ""); +} + +/** +* @tc.name : ShuffleStringByDelimiter_ShouldReturnSameString_WhenInputHasNoDelimiter +* @tc.number: ShuffleStringByDelimiter_Test_002 +* @tc.desc : Test ShuffleStringByDelimiter function with input string that has no delimiter. + */ +TEST_F(UtilsTest, ShuffleStringByDelimiter_ShouldReturnSameString_WhenInputHasNoDelimiter) { + std::string input = "127.0.0.1:2379"; + std::string pattern = ","; + std::string result = ShuffleStringByDelimiter(input, pattern); + EXPECT_EQ(result, "127.0.0.1:2379"); + result = ShuffleStringByDelimiter("ds-core-etcd.default.svc.cluster.local:2379", pattern); + EXPECT_EQ(result, "ds-core-etcd.default.svc.cluster.local:2379"); +} + +/** +* @tc.name : ShuffleStringByDelimiter_ShouldReturnShuffledString_WhenInputHasMultipleDelimiters +* @tc.number: ShuffleStringByDelimiter_Test_003 +* @tc.desc : Test ShuffleStringByDelimiter function with input string that has multiple delimiters. + */ +TEST_F(UtilsTest, ShuffleStringByDelimiter_ShouldReturnShuffledString_WhenInputHasMultipleDelimiters) { + std::string input = "127.0.0.1:2379,127.0.0.2:2379,127.0.0.3:2379"; + std::string pattern = ","; + std::string result = ShuffleStringByDelimiter(input, pattern); + // Since the result is shuffled, we cannot predict the exact output. + // However, we can check if the result contains the same elements. + std::vector expected = {"127.0.0.1:2379", "127.0.0.2:2379", "127.0.0.3:2379"}; + std::vector resultVec = litebus::strings::Split(result, ","); + std::sort(expected.begin(), expected.end()); + std::sort(resultVec.begin(), resultVec.end()); + EXPECT_EQ(resultVec, expected); +} } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/common/yaml_tool/yaml_tool_test.cpp b/functionsystem/tests/unit/common/yaml_tool/yaml_tool_test.cpp index d99d19e1a66b663d9e31ded7a8919906c3364bcf..64724cf6d54460ad5499fa6207730ca98774edeb 100644 --- a/functionsystem/tests/unit/common/yaml_tool/yaml_tool_test.cpp +++ b/functionsystem/tests/unit/common/yaml_tool/yaml_tool_test.cpp @@ -100,6 +100,7 @@ TEST_F(YamlToolTest, GetFunctionMetaSuccess) YRLOG_INFO("entryFile: {}", functionMeta.funcMetaData.entryFile); YRLOG_INFO("hookHandler: {}", PrintHookHandler(functionMeta.funcMetaData.hookHandler)); YRLOG_INFO("version: {}", functionMeta.funcMetaData.version); + YRLOG_INFO("tenantID: {}", functionMeta.funcMetaData.tenantId); YRLOG_INFO("storage: {}", functionMeta.codeMetaData.storageType); YRLOG_INFO("bucketID: {}", functionMeta.codeMetaData.bucketID); YRLOG_INFO("objectID: {}", functionMeta.codeMetaData.objectID); diff --git a/functionsystem/tests/unit/domain_scheduler/create_agent_decision/create_agent_decision_test.cpp b/functionsystem/tests/unit/domain_scheduler/create_agent_decision/create_agent_decision_test.cpp index 2f02370d54404f6c4f5b0faaa585cb771088d00a..1ef8dfe3d507860139bb1c751cf54f74b877b76a 100644 --- a/functionsystem/tests/unit/domain_scheduler/create_agent_decision/create_agent_decision_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/create_agent_decision/create_agent_decision_test.cpp @@ -18,7 +18,7 @@ #include -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/domain_scheduler/domain_group_control/domain_group_ctrl_test.cpp b/functionsystem/tests/unit/domain_scheduler/domain_group_control/domain_group_ctrl_test.cpp index ce311a427d7cb35981b714dfb3f8b597f4a7a838..4cf4034264abefaf48ed9cee1e0aa417f202af4c 100644 --- a/functionsystem/tests/unit/domain_scheduler/domain_group_control/domain_group_ctrl_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/domain_group_control/domain_group_ctrl_test.cpp @@ -181,7 +181,7 @@ TEST_F(DomainGroupCtrlTest, ResourceNotEnoughTimeout) auto groupInfo = NewGroupInfo(1); schedule_decision::GroupScheduleResult result; result.code = StatusCode::RESOURCE_NOT_ENOUGH; - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillRepeatedly(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillRepeatedly(Return(AsyncReturn(result))); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, domainGroupCtrlActor_->GetAID(), groupInfo); ASSERT_AWAIT_READY(future); @@ -198,18 +198,18 @@ TEST_F(DomainGroupCtrlTest, ScheduleFailedAfterReserveFailure) } litebus::Promise promise; promise.SetFailed(StatusCode::ERR_GROUP_SCHEDULE_FAILED); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)).WillOnce(Return(promise.GetFuture())); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))).WillOnce(Return(AsyncReturn(promise.GetFuture()))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); auto response1 = std::make_shared(); response1->set_code(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve) - .WillOnce(Return(response)) - .WillOnce(Return(response1)) - .WillOnce(Return(response1)); + .WillOnce(Return(AsyncReturn(response))) + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response1))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto groupInfo = NewGroupInfo(100); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, domainGroupCtrlActor_->GetAID(), groupInfo); @@ -230,10 +230,10 @@ TEST_F(DomainGroupCtrlTest, ReserveRollback) schedule_decision::GroupScheduleResult noEnough; noEnough.code = StatusCode::RESOURCE_NOT_ENOUGH; EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)) - .WillOnce(Return(result)) - .WillOnce(Return(result)) - .WillOnce(Return(noEnough)) - .WillRepeatedly(Return(failure)); + .WillOnce(Return(AsyncReturn(result))) + .WillOnce(Return(AsyncReturn(result))) + .WillOnce(Return(AsyncReturn(noEnough))) + .WillRepeatedly(Return(AsyncReturn(failure))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); @@ -241,16 +241,16 @@ TEST_F(DomainGroupCtrlTest, ReserveRollback) response1->set_code(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve) // first round to reserve - .WillOnce(Return(response)) - .WillOnce(Return(response1)) - .WillOnce(Return(response1)) + .WillOnce(Return(AsyncReturn(response))) + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response1))) // second round to reserve - .WillOnce(Return(response)) - .WillOnce(Return(response)) - .WillOnce(Return(response1)); + .WillOnce(Return(AsyncReturn(response))) + .WillOnce(Return(AsyncReturn(response))) + .WillOnce(Return(AsyncReturn(response1))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto groupInfo = NewGroupInfo(1); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, domainGroupCtrlActor_->GetAID(), groupInfo); @@ -268,25 +268,25 @@ TEST_F(DomainGroupCtrlTest, BindRollback) } litebus::Promise promise; promise.SetFailed(StatusCode::ERR_GROUP_SCHEDULE_FAILED); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)).WillOnce(Return(promise.GetFuture())); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))).WillOnce(Return(AsyncReturn(promise.GetFuture()))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); auto response1 = std::make_shared(); response1->set_code(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillRepeatedly(Return(response)); + .WillRepeatedly(Return(AsyncReturn(response))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); EXPECT_CALL(*mockUnderlayerSchedMgr_, Bind) - .WillOnce(Return(Status::OK())) - .WillOnce(Return(Status(StatusCode::ERR_INNER_COMMUNICATION))) - .WillOnce(Return(Status(StatusCode::ERR_INNER_COMMUNICATION))); + .WillOnce(Return(AsyncReturn(Status::OK()))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_INNER_COMMUNICATION)))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_INNER_COMMUNICATION)))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnBind) - .WillOnce(Return(Status::OK())) - .WillOnce(Return(Status(StatusCode::ERR_INNER_COMMUNICATION))) - .WillOnce(Return(Status(StatusCode::ERR_INNER_COMMUNICATION))); + .WillOnce(Return(AsyncReturn(Status::OK()))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_INNER_COMMUNICATION)))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_INNER_COMMUNICATION)))); auto groupInfo = NewGroupInfo(100); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, @@ -303,23 +303,23 @@ TEST_F(DomainGroupCtrlTest, LocalAbnormalBindRollback) for (int i = 0; i < INSTANCE_NUM; ++i) { (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillRepeatedly(Return(response)); + .WillRepeatedly(Return(AsyncReturn(response))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); EXPECT_CALL(*mockUnderlayerSchedMgr_, Bind) - .WillOnce(Return(Status::OK())) - .WillOnce(Return(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER))) - .WillOnce(Return(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER))); + .WillOnce(Return(AsyncReturn(Status::OK()))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER)))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER)))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnBind) - .WillOnce(Return(Status::OK())) - .WillOnce(Return(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER))) - .WillOnce(Return(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER))); + .WillOnce(Return(AsyncReturn(Status::OK()))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER)))) + .WillOnce(Return(AsyncReturn(Status(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER)))); auto groupInfo = NewGroupInfo(100); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, @@ -336,16 +336,16 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleSuccessful) for (int i = 0; i < INSTANCE_NUM; ++i) { (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillRepeatedly(Return(response)); + .WillRepeatedly(Return(AsyncReturn(response))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); EXPECT_CALL(*mockUnderlayerSchedMgr_, Bind) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnBind).Times(0); @@ -363,15 +363,15 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleRangeInstanceSuccessful) for (int i = 0; i < INSTANCE_NUM; ++i) { result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillRepeatedly(Return(response)); + .WillRepeatedly(Return(AsyncReturn(response))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); EXPECT_CALL(*mockUnderlayerSchedMgr_, Bind) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnBind).Times(0); auto groupInfo = NewRangeInstanceScheduleGroupInfo(100, 3, 1, 1); @@ -386,7 +386,7 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleRangeInstanceFailed_ResourceNotEnoughTi auto groupInfo = NewRangeInstanceScheduleGroupInfo(1, 3, 1, 1); schedule_decision::GroupScheduleResult result; result.code = StatusCode::RESOURCE_NOT_ENOUGH; - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillRepeatedly(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillRepeatedly(Return(AsyncReturn(result))); auto future = litebus::Async(localSchedSrvStub_->GetAID(), &LocalSchedSrvStub::ForwardGroupSchedule, domainGroupCtrlActor_->GetAID(), groupInfo); ASSERT_AWAIT_READY(future); @@ -411,8 +411,8 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleRangeInstanceReserveCallBackThenSuccess schedule_decision::GroupScheduleResult noEnough; noEnough.code = StatusCode::RESOURCE_NOT_ENOUGH; EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)) - .WillOnce(Return(result)) - .WillOnce(Return(noEnough)); + .WillOnce(Return(AsyncReturn(result))) + .WillOnce(Return(AsyncReturn(noEnough))); auto response = std::make_shared(); response->set_code(StatusCode::SUCCESS); @@ -420,14 +420,14 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleRangeInstanceReserveCallBackThenSuccess response1->set_code(StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve) // first round to reserve - .WillOnce(Return(response)) - .WillOnce(Return(response1)) - .WillOnce(Return(response1)); + .WillOnce(Return(AsyncReturn(response))) + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response1))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockUnderlayerSchedMgr_, Bind) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnBind).Times(0); auto groupInfo = NewRangeInstanceScheduleGroupInfo(100, 3, 1, 1); @@ -441,7 +441,7 @@ TEST_F(DomainGroupCtrlTest, GroupScheduleRangeInstanceReserveCallBackThenSuccess TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) { auto genResult = [](std::shared_ptr ctx, std::vector reserveResult) { - for (int i = 0; i < INSTANCE_NUM; i++) { + for (int i = 0; i < reserveResult.size(); i++) { if (!reserveResult[i]) { ctx->failedReserve.emplace(ctx->requests[i]->requestid()); } @@ -455,11 +455,11 @@ TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); // failed success success genResult(ctx, {false, true, true}); - domainGroupCtrlActor_->RollbackContext(ctx); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); EXPECT_EQ(ctx->lastReservedInd, -1); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(3) .WillRepeatedly(Return(Status::OK())); - auto future = domainGroupCtrlActor_->RollbackRangeReserve(ctx->tryScheduleResults, ctx); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); ASSERT_AWAIT_READY(future); EXPECT_EQ(ctx->lastReservedInd, -1); } @@ -471,11 +471,11 @@ TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); // failed success success genResult(ctx, {true, true, false}); - domainGroupCtrlActor_->RollbackContext(ctx); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); EXPECT_EQ(ctx->lastReservedInd, -1); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(3) .WillRepeatedly(Return(Status::OK())); - auto future = domainGroupCtrlActor_->RollbackRangeReserve(ctx->tryScheduleResults, ctx); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); ASSERT_AWAIT_READY(future); EXPECT_EQ(ctx->lastReservedInd, -1); } @@ -486,11 +486,11 @@ TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); // success success false genResult(ctx, {true, true, false}); - domainGroupCtrlActor_->RollbackContext(ctx); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); EXPECT_EQ(ctx->lastReservedInd, 1); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(1) - .WillRepeatedly(Return(Status::OK())); - auto future = domainGroupCtrlActor_->RollbackRangeReserve(ctx->tryScheduleResults, ctx); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); ASSERT_AWAIT_READY(future); EXPECT_EQ(ctx->lastReservedInd, 1); } @@ -499,11 +499,11 @@ TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); // success success false genResult(ctx, {true, true, false}); - domainGroupCtrlActor_->RollbackContext(ctx); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); EXPECT_EQ(ctx->lastReservedInd, 1); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(1) - .WillRepeatedly(Return(Status::OK())); - auto future = domainGroupCtrlActor_->RollbackRangeReserve(ctx->tryScheduleResults, ctx); + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); ASSERT_AWAIT_READY(future); EXPECT_EQ(ctx->lastReservedInd, 1); } @@ -512,11 +512,34 @@ TEST_F(DomainGroupCtrlTest, RollBackInstanceAfterLastestSuccessfulReserved) auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); // success success success genResult(ctx, {true, true, true}); - domainGroupCtrlActor_->RollbackContext(ctx); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); EXPECT_EQ(ctx->lastReservedInd, 2); - EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0) - .WillRepeatedly(Return(Status::OK())); - auto future = domainGroupCtrlActor_->RollbackRangeReserve(ctx->tryScheduleResults, ctx); + EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(ctx->lastReservedInd, 2); + } + { + auto groupInfo = NewRangeInstanceScheduleGroupInfo(1, 5, 3, 2); + auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); + genResult(ctx, {true, false, true, false, true}); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); + EXPECT_EQ(ctx->lastReservedInd, 2); + EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(3) + .WillRepeatedly(Return(Status::OK())); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(ctx->lastReservedInd, 2); + } + { + auto groupInfo = NewRangeInstanceScheduleGroupInfo(1, 5, 3, 2); + auto ctx = domainGroupCtrlActor_->NewGroupContext(groupInfo); + genResult(ctx, {true, true, true, false, true}); + auto rollbacked = domainGroupCtrlActor_->RollbackContext(ctx); + EXPECT_EQ(ctx->lastReservedInd, 2); + EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(2) + .WillRepeatedly(Return(Status::OK())); + auto future = domainGroupCtrlActor_->RollbackRangeReserve(rollbacked, ctx->tryScheduleResults, ctx); ASSERT_AWAIT_READY(future); EXPECT_EQ(ctx->lastReservedInd, 2); } @@ -550,7 +573,7 @@ TEST_F(DomainGroupCtrlTest, TestReleaseUnusedReserve) .mutable_groupschedctx()->set_reserved("test2"); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(1) - .WillOnce(Return(Status::OK())); + .WillOnce(Return(AsyncReturn(Status::OK()))); domainGroupCtrlActor_->ReleaseUnusedReserve(ctx->tryScheduleResults, ctx); EXPECT_EQ(ctx->lastReservedInd, 0); } @@ -664,7 +687,7 @@ TEST_F(DomainGroupCtrlTest, SfmdGroupScheduleSuccessful) // single node { EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(result))); auto groupInfo = NewSfmdGroupInfo(100); auto response1 = NewScheduleResponse(selectedNodeId1); @@ -678,9 +701,9 @@ TEST_F(DomainGroupCtrlTest, SfmdGroupScheduleSuccessful) (*response3->mutable_scheduleresult()->add_devices()) = std::move(NewHeteroDeviceInfo(103, "0.0.0.3")); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillOnce(Return(response1)) - .WillOnce(Return(response2)) - .WillOnce(Return(response3)); + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response2))) + .WillOnce(Return(AsyncReturn(response3))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); std::vector> scheduleReqs; @@ -750,7 +773,7 @@ TEST_F(DomainGroupCtrlTest, SfmdGroupScheduleSuccessful) // multi node { - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); auto groupInfo = NewSfmdGroupInfo(100); auto response1 = NewScheduleResponse(selectedNodeId1); @@ -767,9 +790,9 @@ TEST_F(DomainGroupCtrlTest, SfmdGroupScheduleSuccessful) response3->set_instanceid(groupInfo->requests(2).instance().instanceid()); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillOnce(Return(response1)) - .WillOnce(Return(response2)) - .WillOnce(Return(response3)); + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response2))) + .WillOnce(Return(AsyncReturn(response3))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); std::vector> scheduleReqs; @@ -852,11 +875,11 @@ TEST_F(DomainGroupCtrlTest, SfmdGroupScheduleSuccessful) expectedInsRankIds[response3->instanceid()] = { 0, 2 }; for (const auto &req : scheduleReqs) { - auto &instanceId = req->instance().instanceid(); - if (!google::protobuf::util::JsonStringToMessage(req->instance().createoptions().at( - "FUNCTION_GROUP_RUNNING_INFO"), &functionGroupRunningInfo).ok()) { + auto &instanceId = req->instance().instanceid(); + if (!google::protobuf::util::JsonStringToMessage(req->instance().createoptions().at( + "FUNCTION_GROUP_RUNNING_INFO"), &functionGroupRunningInfo).ok()) { EXPECT_EQ(1, 0); - } + } EXPECT_EQ(expectedInsRankIds[instanceId].find(functionGroupRunningInfo.instancerankid()) != expectedInsRankIds[instanceId].end(), true); } @@ -879,7 +902,7 @@ TEST_F(DomainGroupCtrlTest, HeteroGroupSchedulerWithResourceGroup) EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(result))); auto groupInfo = NewSfmdGroupInfo(100); for (auto &req : *groupInfo->mutable_requests()) { req.mutable_instance()->mutable_scheduleoption()->set_target(resources::CreateTarget::RESOURCE_GROUP); @@ -896,9 +919,9 @@ TEST_F(DomainGroupCtrlTest, HeteroGroupSchedulerWithResourceGroup) (*response3->mutable_scheduleresult()->add_devices()) = std::move(NewHeteroDeviceInfo(103, "0.0.0.3")); EXPECT_CALL(*mockUnderlayerSchedMgr_, Reserve).Times(3) - .WillOnce(Return(response1)) - .WillOnce(Return(response2)) - .WillOnce(Return(response3)); + .WillOnce(Return(AsyncReturn(response1))) + .WillOnce(Return(AsyncReturn(response2))) + .WillOnce(Return(AsyncReturn(response3))); EXPECT_CALL(*mockUnderlayerSchedMgr_, UnReserve).Times(0); std::vector> scheduleReqs; diff --git a/functionsystem/tests/unit/domain_scheduler/domain_scheduler_service/domain_sched_srv_test.cpp b/functionsystem/tests/unit/domain_scheduler/domain_scheduler_service/domain_sched_srv_test.cpp index 262a834749abff24df9003de750078f74d71d616..a25c4d9acc257764f987c4c565205ce79d4eef3f 100644 --- a/functionsystem/tests/unit/domain_scheduler/domain_scheduler_service/domain_sched_srv_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/domain_scheduler_service/domain_sched_srv_test.cpp @@ -19,12 +19,12 @@ #include -#include "constants.h" +#include "common/constants/constants.h" #include "common/constants/metastore_keys.h" #include "common/etcd_service/etcd_service_driver.h" #include "common/explorer/etcd_explorer_actor.h" #include "common/explorer/explorer.h" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "meta_store_client/meta_store_client.h" #include "common/resource_view/view_utils.h" #include "mocks/mock_domain_group_ctrl.h" @@ -82,7 +82,7 @@ private: class DomainSchedSrvTest : public ::testing::Test { protected: inline static std::string metaStoreServerHost_; - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { const auto &address = litebus::GetLitebusAddress(); address_ = address.ip + ":" + std::to_string(address.port); @@ -93,7 +93,7 @@ protected: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -136,9 +136,9 @@ protected: { globalStub->SetResponseLeader(upDomainName, address_); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); LeaderResponse response{ .status = Status::OK(), .header = {}, .kv = std::make_pair(DEFAULT_MASTER_ELECTION_KEY, address_) }; @@ -185,9 +185,9 @@ TEST_F(DomainSchedSrvTest, RegisterToGlobalTimeout) InitCase("RegisterToGlobalTimeout", 5, 2); auto unit = std::make_shared(); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto domainSchedSrv = DomainSchedSrv(domainSchedSrvActor_->GetAID()); // test global not start LeaderResponse response{ .status = Status::OK(), @@ -207,9 +207,9 @@ TEST_F(DomainSchedSrvTest, RegisterToGlobalNormalAndNotifyAbnormal) EXPECT_CALL(*mockUnderlayerSchedMgr_, UpdateUnderlayerTopo).WillOnce(Return()); auto unit = std::make_shared(); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); // test global start auto globalStub = std::make_shared(DOMAIN_SCHED_MGR_ACTOR_NAME); @@ -268,7 +268,7 @@ public: void PingPongLostHelper(const litebus::AID &lostDst) { - PingPongLost(lostDst, HeartbeatConnection::LOST); + PingPongLost(lostDst); } void UpdateLeaderHelper(const std::string &name, const std::string &address) @@ -283,7 +283,7 @@ public: TEST_F(DomainSchedSrvTest, NotifyAbnormalFailWithNoGlobalAndUplayer) { DomainSchedSrvActorHelper domainSchedSrvActorHelper("test", metaStoreServerHost_); - litebus::AID connDst("conn", "127.0.0.1:12345"); + litebus::AID connDst("conn", "127.0.0.1:12345"); // ���ᷢ����ʵ������ domainSchedSrvActorHelper.SetUplayerHelper(connDst); domainSchedSrvActorHelper.SetGlobalHelper(connDst); @@ -297,7 +297,7 @@ TEST_F(DomainSchedSrvTest, NotifyAbnormalFailWithNoGlobalAndUplayer) TEST_F(DomainSchedSrvTest, NotifyWorkerFailWithNoGlobalAndUplayer) { DomainSchedSrvActorHelper domainSchedSrvActorHelper("test", metaStoreServerHost_); - litebus::AID connDst("conn", "127.0.0.1:12345"); + litebus::AID connDst("conn", "127.0.0.1:12345"); // ���ᷢ����ʵ������ domainSchedSrvActorHelper.SetUplayerHelper(connDst); domainSchedSrvActorHelper.SetGlobalHelper(connDst); @@ -310,8 +310,8 @@ TEST_F(DomainSchedSrvTest, NotifyWorkerFailWithNoGlobalAndUplayer) TEST_F(DomainSchedSrvTest, PingPongLostFail) { DomainSchedSrvActorHelper domainSchedSrvActorHelper("test", metaStoreServerHost_); - litebus::AID connDst("conn", "127.0.0.1:12345"); - litebus::AID lostDst("lost", "127.0.0.1:12345"); + litebus::AID connDst("conn", "127.0.0.1:12345"); // ���������Mock���� + litebus::AID lostDst("lost", "127.0.0.1:12345"); // ���������Mock���� domainSchedSrvActorHelper.SetUplayerHelper(lostDst); EXPECT_CALL(domainSchedSrvActorHelper, RegisterToLeader()).WillRepeatedly(Return()); @@ -390,13 +390,13 @@ TEST_F(DomainSchedSrvTest, PutReadyAgent) auto domainUnit2 = GenUnitByFragment(agentUnits2); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillOnce(Return(std::make_shared(domainUnit1))) - .WillOnce(Return(std::make_shared(domainUnit1))) - .WillOnce(Return(std::make_shared(domainUnit2))) - .WillRepeatedly(Return(std::make_shared(domainUnit1))); + .WillOnce(Return(AsyncReturn(std::make_shared(domainUnit1)))) + .WillOnce(Return(AsyncReturn(std::make_shared(domainUnit1)))) + .WillOnce(Return(AsyncReturn(std::make_shared(domainUnit2)))) + .WillRepeatedly(Return(AsyncReturn(std::make_shared(domainUnit1)))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared(domainUnit1))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(domainUnit1)))); uint32_t putCnt = 0; uint32_t readyResCnt = 0; @@ -418,7 +418,9 @@ TEST_F(DomainSchedSrvTest, PutReadyAgent) MetaStoreClient client(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }); client.Init(); WatchOption option = { .prefix = true, .prevKv = true, .revision = 0 }; - auto syncer = []() -> litebus::Future { return SyncResult{ Status::OK(), 0 }; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; auto watcher = client.Watch(READY_AGENT_CNT_KEY, option, observer, syncer).Get(); ASSERT_AWAIT_TRUE([&watcher]() -> bool { return watcher->GetWatchId() == 0; }); @@ -460,7 +462,7 @@ TEST_F(DomainSchedSrvTest, ReceiveLeaderTopoFromGlobalToRegister) .WillRepeatedly(testing::DoAll(FutureArg<1>(&name), FutureArg<2>(&msg))); resource_view::ResourceUnit successRet; successRet.set_id("ReceiveLeaderTopoFromGlobalToRegister"); - EXPECT_CALL(*primary_, GetSerializedResourceView()).WillOnce(Return(successRet.SerializeAsString())); + EXPECT_CALL(*primary_, GetSerializedResourceView()).WillOnce(Return(AsyncReturn(successRet.SerializeAsString()))); RegisterUplayer("d1", "ReceiveLeaderTopoFromGlobalToRegister", globalStub, leadStub, domainSchedSrv); ASSERT_AWAIT_READY_FOR(name, 5000); EXPECT_EQ(name.Get(), "UpdateResources"); @@ -524,7 +526,7 @@ TEST_F(DomainSchedSrvTest, UpdateSchedTopoView) EXPECT_EQ(1, 0); })); auto unit = std::make_shared(); - EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(unit)); + EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(AsyncReturn(unit))); litebus::Async(globalStub->GetAID(), &UplayerActor::SendRequest, domainSchedSrvActor_->GetAID(), "UpdateSchedTopoView", topo.SerializeAsString()); litebus::Terminate(globalStub->GetAID()); @@ -557,7 +559,7 @@ TEST_F(DomainSchedSrvTest, UpdateSchedTopoViewWithNoHeader) EXPECT_EQ(1, 0); })); auto unit = std::make_shared(); - EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(unit)); + EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(AsyncReturn(unit))); litebus::Async(globalStub->GetAID(), &UplayerActor::SendRequest, domainSchedSrvActor_->GetAID(), "UpdateSchedTopoView", topo.SerializeAsString()); litebus::Terminate(globalStub->GetAID()); @@ -579,7 +581,7 @@ TEST_F(DomainSchedSrvTest, UpdateSchedTopoViewWithPareseFail) litebus::Future topo; EXPECT_CALL(*mockUnderlayerSchedMgr_, UpdateUnderlayerTopo(_)).WillRepeatedly(testing::DoAll(FutureArg<0>(&topo))); auto unit = std::make_shared(); - EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(unit)); + EXPECT_CALL(*primary_, GetFullResourceView()).WillRepeatedly(Return(AsyncReturn(unit))); litebus::Async(globalStub->GetAID(), &UplayerActor::SendRequest, domainSchedSrvActor_->GetAID(), "UpdateSchedTopoView", "test"); @@ -603,8 +605,8 @@ TEST_F(DomainSchedSrvTest, PullResources) auto failed = litebus::Future(); failed.SetFailed(StatusCode::FAILED); EXPECT_CALL(*primary_, GetSerializedResourceView()) - .WillOnce(Return(successRet.SerializeAsString())) - .WillOnce(Return(failed)); + .WillOnce(Return(AsyncReturn(successRet.SerializeAsString()))) + .WillOnce(Return(AsyncReturn(failed))); litebus::Future name; litebus::Future msg; @@ -736,9 +738,9 @@ TEST_F(DomainSchedSrvTest, ScheduleSuccessful) rsp->set_code(0); EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(rsp)); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto req = std::make_shared(); req->set_requestid("request"); @@ -777,11 +779,11 @@ TEST_F(DomainSchedSrvTest, ScheduleFailed) auto rsp = std::make_shared(); rsp->set_requestid("request"); rsp->set_code(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(rsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(rsp))); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto req = std::make_shared(); req->set_requestid("request"); @@ -881,7 +883,7 @@ TEST_F(DomainSchedSrvTest, QueryAgentInfo) auto domainUnit1 = GenUnitByFragment(agentUnits, "domain1"); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillOnce(Return(std::make_shared(domainUnit1))); + .WillOnce(Return(AsyncReturn(std::make_shared(domainUnit1)))); auto req = std::make_shared(); req->set_requestid("request"); @@ -1003,10 +1005,10 @@ TEST_F(DomainSchedSrvTest, GetSchedulingQueue) .WillOnce(testing::DoAll(FutureArg<1>(&name), FutureArg<2>(&msg))); std::vector> instanceRequest = GetInstanceRequest(); - EXPECT_CALL(*mockInstanceCtrl_, GetSchedulerQueue()).WillOnce(Return(instanceRequest)); + EXPECT_CALL(*mockInstanceCtrl_, GetSchedulerQueue()).WillOnce(Return(AsyncReturn(instanceRequest))); std::vector> groupRequest = GetGroupRequest(); - EXPECT_CALL(*mockGroupCtrl_, GetRequests()).WillOnce(Return(groupRequest)); + EXPECT_CALL(*mockGroupCtrl_, GetRequests()).WillOnce(Return(AsyncReturn(groupRequest))); auto req = std::make_shared(); std::string requestId = "requestIdIdId"; @@ -1042,9 +1044,9 @@ TEST_F(DomainSchedSrvTest, QueryResourceInfo) invalid.set_status(static_cast(UnitStatus::TO_BE_DELETED)); (*unit.mutable_fragment())["invalid"] = invalid; EXPECT_CALL(*primary_, GetResourceViewCopy()) - .WillRepeatedly(Return(std::make_shared(unit))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(unit)))); EXPECT_CALL(*virtual_, GetResourceViewCopy()) - .WillRepeatedly(Return(std::make_shared(unit))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(unit)))); auto req = std::make_shared(); req->set_requestid("request"); diff --git a/functionsystem/tests/unit/domain_scheduler/flags/flag_test.cpp b/functionsystem/tests/unit/domain_scheduler/flags/flag_test.cpp index 81e963fc68bef279e8d9cef95a38bb02ec904dd1..846d0038c24be4e1ec595cad2a1c6cb363d6f1ee 100644 --- a/functionsystem/tests/unit/domain_scheduler/flags/flag_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/flags/flag_test.cpp @@ -17,7 +17,7 @@ #include #include -#include "logs/logging.h" +#include "common/logs/logging.h" #include "domain_scheduler/flags/flags.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/domain_scheduler/instance_control/instance_ctrl_test.cpp b/functionsystem/tests/unit/domain_scheduler/instance_control/instance_ctrl_test.cpp index 0d489409b7336bc8544360eedd5a2445ac944436..6800c8037a9f123b8deed1bd9b027c4e00636104 100644 --- a/functionsystem/tests/unit/domain_scheduler/instance_control/instance_ctrl_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/instance_control/instance_ctrl_test.cpp @@ -19,7 +19,7 @@ #include #include -#include "constants.h" +#include "common/constants/constants.h" #include "domain_scheduler/instance_control/instance_ctrl_actor.h" #include "common/schedule_decision/schedule_recorder/schedule_recorder.h" #include "mocks/mock_domain_sched_srv.h" @@ -28,6 +28,7 @@ #include "mocks/mock_scaler_actor.h" #include "mocks/mock_shared_client.h" #include "mocks/mock_shared_client_manager_proxy.h" +#include "mocks/mock_resource_view.h" #include "utils/future_test_helper.h" namespace functionsystem::test { @@ -41,7 +42,13 @@ public: instanceCtrl_ = std::make_shared("DomainInstanceCtrlTest"); mockScheduler_ = std::make_shared(); mockUnderlayerScheMgr_ = std::make_shared(); + auto resourceViewMgr_ = std::make_shared(); + primary_ = MockResourceView::CreateMockResourceView(); + virtual_ = MockResourceView::CreateMockResourceView(); + resourceViewMgr_->primary_ = primary_; + resourceViewMgr_->virtual_ = virtual_; instanceCtrl_->BindScheduler(mockScheduler_); + instanceCtrl_->BindResourceView(resourceViewMgr_); instanceCtrl_->BindUnderlayerMgr(mockUnderlayerScheMgr_); instanceCtrl_->BindScheduleRecorder(schedule_decision::ScheduleRecorder::CreateScheduleRecorder()); litebus::Spawn(instanceCtrl_); @@ -57,6 +64,8 @@ protected: std::shared_ptr instanceCtrl_; std::shared_ptr mockScheduler_; std::shared_ptr mockUnderlayerScheMgr_; + std::shared_ptr primary_; + std::shared_ptr virtual_; }; /** @@ -76,12 +85,16 @@ TEST_F(DomainInstanceCtrlTest, ScheduleInstanceSuccessful) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -92,6 +105,9 @@ TEST_F(DomainInstanceCtrlTest, ScheduleInstanceSuccessful) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); } /** @@ -111,12 +127,16 @@ TEST_F(DomainInstanceCtrlTest, ScheduleInstanceVersionWrong) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(StatusCode::INSTANCE_TRANSACTION_WRONG_VERSION); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -127,6 +147,10 @@ TEST_F(DomainInstanceCtrlTest, ScheduleInstanceVersionWrong) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), StatusCode::INSTANCE_TRANSACTION_WRONG_VERSION); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); } /** @@ -152,7 +176,7 @@ TEST_F(DomainInstanceCtrlTest, InsufficientResource) ScheduleResult resourceNotEnough{ "", RESOURCE_NOT_ENOUGH, "resources not enough" }; ScheduleResult invalidParam{ "", PARAMETER_ERROR, "parameter error" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillRepeatedly(Return(resourceNotEnough)); + .WillRepeatedly(Return(AsyncReturn(resourceNotEnough))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -167,7 +191,7 @@ TEST_F(DomainInstanceCtrlTest, InsufficientResource) EXPECT_THAT(rsp->message(), testing::HasSubstr("resources not enough")); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillRepeatedly(Return(invalidParam)); + .WillRepeatedly(Return(AsyncReturn(invalidParam))); future = instanceCtrl.Schedule(req); ASSERT_AWAIT_READY_FOR(future, 1000); rsp = future.Get(); @@ -176,6 +200,7 @@ TEST_F(DomainInstanceCtrlTest, InsufficientResource) EXPECT_THAT( rsp->message(), testing::HasSubstr("invalid resource parameter, request resource is greater than each node's max resource")); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); } /** @@ -195,7 +220,7 @@ TEST_F(DomainInstanceCtrlTest, SuccessfullyAfterRetries) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillOnce(Return(AsyncReturn(result))); auto pro = litebus::Promise>(); pro.SetFailed(StatusCode::REQUEST_TIME_OUT); @@ -204,9 +229,14 @@ TEST_F(DomainInstanceCtrlTest, SuccessfullyAfterRetries) mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)) - .WillOnce(Return(failedFuture)) - .WillOnce(Return(failedFuture)) - .WillOnce(Return(mockSchedRsp)); + .WillOnce(Return(AsyncReturn(failedFuture))) + .WillOnce(Return(AsyncReturn(failedFuture))) + .WillOnce(Return(AsyncReturn(mockSchedRsp))); + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).Times(3).WillRepeatedly(Return(litebus::Future{snapshot})); + auto req = std::make_shared(); req->set_requestid(requestID); @@ -217,6 +247,7 @@ TEST_F(DomainInstanceCtrlTest, SuccessfullyAfterRetries) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); } /** @@ -237,7 +268,7 @@ TEST_F(DomainInstanceCtrlTest, ReschedulingTriggeredByRetriesFailed) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(AsyncReturn(result))); auto pro = litebus::Promise>(); pro.SetFailed(StatusCode::REQUEST_TIME_OUT); @@ -246,10 +277,21 @@ TEST_F(DomainInstanceCtrlTest, ReschedulingTriggeredByRetriesFailed) mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)) - .WillOnce(Return(failedFuture)) - .WillOnce(Return(failedFuture)) - .WillOnce(Return(failedFuture)) - .WillOnce(Return(mockSchedRsp)); + .WillOnce(Return(AsyncReturn(failedFuture))) + .WillOnce(Return(AsyncReturn(failedFuture))) + .WillOnce(Return(AsyncReturn(failedFuture))) + .WillOnce(Return(AsyncReturn(mockSchedRsp))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + resource_view::PullResourceRequest snapshot2; + snapshot2.set_localviewinittime("INITTIME1"); + snapshot2.set_version(5); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})) + .WillOnce(Return(litebus::Future{snapshot})) + .WillOnce(Return(litebus::Future{snapshot})) + .WillOnce(Return(litebus::Future{snapshot2})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -260,6 +302,9 @@ TEST_F(DomainInstanceCtrlTest, ReschedulingTriggeredByRetriesFailed) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 5); } /** @@ -280,7 +325,7 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingAfterConflict) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(StatusCode::RESOURCE_NOT_ENOUGH); @@ -289,8 +334,16 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingAfterConflict) mockSuccessRsp->set_code(0); mockSuccessRsp->set_requestid(requestID); EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)) - .WillOnce(Return(mockSchedRsp)) - .WillOnce(Return(mockSuccessRsp)); + .WillOnce(Return(AsyncReturn(mockSchedRsp))) + .WillOnce(Return(AsyncReturn(mockSuccessRsp))); + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + resource_view::PullResourceRequest snapshot2; + snapshot2.set_localviewinittime("INITTIME1"); + snapshot2.set_version(5); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})) + .WillOnce(Return(litebus::Future{snapshot2})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -301,6 +354,9 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingAfterConflict) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 5); } /** @@ -321,7 +377,7 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingFailedAfterConflict) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; - EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(result)); + EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)).WillRepeatedly(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(StatusCode::RESOURCE_NOT_ENOUGH); @@ -340,6 +396,10 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingFailedAfterConflict) rsp->set_code(StatusCode::RESOURCE_NOT_ENOUGH); return rsp; })); + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillRepeatedly(Return(litebus::Future{snapshot})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -352,6 +412,9 @@ TEST_F(DomainInstanceCtrlTest, ReSchedulingFailedAfterConflict) // retry attempts. EXPECT_EQ(rsp->code(), StatusCode::ERR_SCHEDULE_CANCELED); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); } /** @@ -382,13 +445,18 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentSuccess) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); messages::CreateAgentResponse createAgentRsp; createAgentRsp.set_requestid(requestID); @@ -407,6 +475,9 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentSuccess) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -429,13 +500,18 @@ TEST_F(DomainInstanceCtrlTest, AffinityCreateAgent) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(result))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); messages::CreateAgentResponse createAgentRsp; createAgentRsp.set_requestid(requestID); @@ -458,6 +534,9 @@ TEST_F(DomainInstanceCtrlTest, AffinityCreateAgent) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -489,8 +568,8 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentFailed) ScheduleResult resourceNotEnough{ "", RESOURCE_NOT_ENOUGH, "resources not enough" }; ScheduleResult invalidParam{ "", PARAMETER_ERROR, "parameter error" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(invalidParam)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(invalidParam))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -519,6 +598,7 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentFailed) rsp = future.Get(); EXPECT_EQ(rsp->code(), -1); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -553,14 +633,19 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentRetrySuccess) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(result))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -580,6 +665,9 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentRetrySuccess) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -612,9 +700,9 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentRetryFailed) ScheduleResult resourceNotEnough{ "", RESOURCE_NOT_ENOUGH, "resources not enough" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -637,6 +725,7 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentRetryFailed) EXPECT_EQ(rsp->requestid(), requestID); YRLOG_INFO("err msg: {}", rsp->message()); EXPECT_THAT(rsp->message(), testing::HasSubstr("resources not enough")); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -667,10 +756,10 @@ TEST_F(DomainInstanceCtrlTest, MonopolyRetry) ScheduleResult resourceNotEnough{ "", RESOURCE_NOT_ENOUGH, "resources not enough" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -688,18 +777,26 @@ TEST_F(DomainInstanceCtrlTest, MonopolyRetry) std::string mockSelectedName = "selected"; ScheduleResult ok{ mockSelectedName, SUCCESS, "success" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(resourceNotEnough)) - .WillOnce(Return(ok)); + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(resourceNotEnough))) + .WillOnce(Return(AsyncReturn(ok))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); future = instanceCtrl.Schedule(req); ASSERT_AWAIT_READY_FOR(future, 1000); rsp = future.Get(); EXPECT_EQ(rsp->code(), StatusCode::SUCCESS); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); } TEST_F(DomainInstanceCtrlTest, AffinityRetry) @@ -714,10 +811,10 @@ TEST_F(DomainInstanceCtrlTest, AffinityRetry) ScheduleResult labelNotFound{ "", AFFINITY_SCHEDULE_FAILED, "affinity schedule failed" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(labelNotFound)) - .WillOnce(Return(labelNotFound)) - .WillOnce(Return(labelNotFound)) - .WillOnce(Return(labelNotFound)); + .WillOnce(Return(AsyncReturn(labelNotFound))) + .WillOnce(Return(AsyncReturn(labelNotFound))) + .WillOnce(Return(AsyncReturn(labelNotFound))) + .WillOnce(Return(AsyncReturn(labelNotFound))); auto req = std::make_shared(); req->set_requestid(requestID); @@ -739,18 +836,26 @@ TEST_F(DomainInstanceCtrlTest, AffinityRetry) std::string mockSelectedName = "selected"; ScheduleResult ok{ mockSelectedName, SUCCESS, "success" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(labelNotFound)) - .WillOnce(Return(labelNotFound)) - .WillOnce(Return(ok)); + .WillOnce(Return(AsyncReturn(labelNotFound))) + .WillOnce(Return(AsyncReturn(labelNotFound))) + .WillOnce(Return(AsyncReturn(ok))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); future = instanceCtrl.Schedule(req); ASSERT_AWAIT_READY_FOR(future, 2000); rsp = future.Get(); EXPECT_EQ(rsp->code(), StatusCode::SUCCESS); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); } /** @@ -782,14 +887,19 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentByPoolIDAffinityFailed) std::string mockSelectedName = "selected"; ScheduleResult result{ mockSelectedName, 0, "" }; EXPECT_CALL(*mockScheduler_, ScheduleDecision(_, _)) - .WillOnce(Return(affinityFailed)) - .WillOnce(Return(affinityFailed)) - .WillOnce(Return(result)); + .WillOnce(Return(AsyncReturn(affinityFailed))) + .WillOnce(Return(AsyncReturn(affinityFailed))) + .WillOnce(Return(AsyncReturn(result))); auto mockSchedRsp = std::make_shared(); mockSchedRsp->set_code(0); mockSchedRsp->set_requestid(requestID); - EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(mockSchedRsp)); + EXPECT_CALL(*mockUnderlayerScheMgr_, DispatchSchedule(mockSelectedName, _)).WillOnce(Return(AsyncReturn(mockSchedRsp))); + + resource_view::PullResourceRequest snapshot; + snapshot.set_localviewinittime("INITTIME1"); + snapshot.set_version(2); + EXPECT_CALL(*primary_, GetUnitSnapshotInfo(_)).WillOnce(Return(litebus::Future{snapshot})); auto req = std::make_shared(); req->set_requestid(requestID); @@ -807,6 +917,9 @@ TEST_F(DomainInstanceCtrlTest, CreateAgentByPoolIDAffinityFailed) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), 0); EXPECT_EQ(rsp->requestid(), requestID); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); + EXPECT_EQ(req->unitsnapshot().localviewinittime(), "INITTIME1"); + EXPECT_EQ(req->unitsnapshot().version(), 2); litebus::Terminate(mockScaler->GetAID()); litebus::Await(mockScaler->GetAID()); @@ -826,5 +939,6 @@ TEST_F(DomainInstanceCtrlTest, ScheduleTimeoutCancel) auto rsp = future.Get(); EXPECT_EQ(rsp->code(), StatusCode::ERR_SCHEDULE_CANCELED); EXPECT_EQ(rsp->requestid(), "req"); + EXPECT_TRUE(instanceCtrl_->cancelTag_.find(rsp->requestid())==instanceCtrl_->cancelTag_.end()); } } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/domain_scheduler/preemption_controller/preemption_controller_test.cpp b/functionsystem/tests/unit/domain_scheduler/preemption_controller/preemption_controller_test.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e9bb1da130e97f11e9cc3278d851be45dd292202 100644 --- a/functionsystem/tests/unit/domain_scheduler/preemption_controller/preemption_controller_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/preemption_controller/preemption_controller_test.cpp @@ -0,0 +1,16 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + diff --git a/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_test.cpp b/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_test.cpp index 9c8443881d03434e7b5cdf7f36e3b0a717759d36..83a6bbe44e9484c2defa190351350dc6ef1e99a7 100644 --- a/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_test.cpp +++ b/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_test.cpp @@ -22,7 +22,7 @@ #include "common/constants/actor_name.h" #include "common/resource_view/view_utils.h" -#include "status/status.h" +#include "common/status/status.h" #include "domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr.h" #include "domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_actor.h" #include "mocks/mock_domain_instance_ctrl.h" @@ -38,7 +38,7 @@ using ::testing::Return; using namespace domain_scheduler; class UnderlayerSchedMgrTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { auto address = litebus::GetLitebusAddress(); address_ = address.ip + ":" + std::to_string(address.port); @@ -72,10 +72,10 @@ protected: litebus::Future msgValue; EXPECT_CALL(*mockUnderlayerActor, MockRegistered(_, _, _)) .WillOnce(testing::DoAll(FutureArg<1>(&msgName), FutureArg<2>(&msgValue))); - EXPECT_CALL(*primary_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*virtual_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*primary_, DeleteLocalResourceView(_)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*virtual_, DeleteLocalResourceView(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*primary_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*virtual_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*primary_, DeleteLocalResourceView(_)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*virtual_, DeleteLocalResourceView(_)).WillOnce(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(1)).Times(1); messages::ScheduleTopology topo; auto member = topo.add_members(); @@ -127,10 +127,10 @@ TEST_F(UnderlayerSchedMgrTest, UnderlayerRegisterAlreadyRegistered) litebus::Future msgValue; EXPECT_CALL(*mockUnderlayerActor, MockRegistered(_, _, _)) .WillRepeatedly(testing::DoAll(FutureArg<1>(&msgName), FutureArg<2>(&msgValue))); - EXPECT_CALL(*primary_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*virtual_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*primary_, DeleteLocalResourceView(_)).WillOnce(Return(Status::OK())); - EXPECT_CALL(*virtual_, DeleteLocalResourceView(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*primary_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*virtual_, AddResourceUnitWithUrl(_, _)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*primary_, DeleteLocalResourceView(_)).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*virtual_, DeleteLocalResourceView(_)).WillOnce(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(1)).Times(1); messages::ScheduleTopology topo; @@ -205,32 +205,6 @@ TEST_F(UnderlayerSchedMgrTest, UnderlayerRegisterFailWhenParseReq) litebus::Await(mockUnderlayerActor); } -TEST_F(UnderlayerSchedMgrTest, UnderlayerExit) -{ - UnderlayerSchedMgr underlayer(underlayerSchedMgrActor_->GetAID()); - auto mockUnderlayerActor = std::make_shared("WillRegister"); - litebus::Spawn(mockUnderlayerActor); - UnderlayerRegiter(mockUnderlayerActor, underlayer); - litebus::Promise pro; - auto fut = pro.GetFuture(); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)) - .WillOnce(::testing::Invoke([pro](const messages::NotifySchedAbnormalRequest &req) -> litebus::Future { - pro.SetValue(req); - return Status::OK(); - })); - litebus::Future times; - EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(_)) - .WillOnce(::testing::DoAll(FutureArg<0>(×), Return())); - mockUnderlayerActor->ClosePingPong(); - litebus::Terminate(mockUnderlayerActor->GetAID()); - litebus::Await(mockUnderlayerActor); - - EXPECT_AWAIT_READY_FOR(fut, 1000); - EXPECT_EQ(fut.Get().schedname(), "WillRegister"); - EXPECT_AWAIT_READY(times); - EXPECT_EQ(times.Get(), (uint32_t)0); -} - TEST_F(UnderlayerSchedMgrTest, ForwardScheduleSuccessful) { auto mockUnderlayerActor = std::make_shared("Forwarder"); @@ -240,7 +214,7 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleSuccessful) successRsp->set_code(0); successRsp->set_requestid("request"); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(successRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(successRsp))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseForwardSchedule(_, _, _)) @@ -269,7 +243,7 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleFailWhenScheduleFail) litebus::Future> failRsp; failRsp.SetFailed(100); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(failRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(failRsp))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseForwardSchedule(_, _, _)) @@ -306,7 +280,7 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleFailWhenVersionWrong) versionWrongRsp->set_code(StatusCode::INSTANCE_TRANSACTION_WRONG_VERSION); versionWrongRsp->set_requestid("request"); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(versionWrongRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(versionWrongRsp))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseForwardSchedule(_, _, _)) @@ -357,7 +331,7 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleWithUpdateResourceSuccessful) successRsp->set_code(0); successRsp->set_requestid("request"); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(successRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(successRsp))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseForwardSchedule(_, _, _)) @@ -386,12 +360,12 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleFailedToForwardUplayerFail) auto failedRsp = std::make_shared(); failedRsp->set_code(2); failedRsp->set_requestid("request1"); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(failedRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(failedRsp))); auto upfailRsp = std::make_shared(); upfailRsp->set_code(StatusCode::DOMAIN_SCHEDULER_FORWARD_ERR); upfailRsp->set_requestid("request1"); - EXPECT_CALL(*mockDomainSrv_, ForwardSchedule(_)).WillOnce(Return(upfailRsp)); + EXPECT_CALL(*mockDomainSrv_, ForwardSchedule(_)).WillOnce(Return(AsyncReturn(upfailRsp))); messages::ScheduleRequest req; req.set_requestid("request1"); @@ -422,12 +396,12 @@ TEST_F(UnderlayerSchedMgrTest, ForwardScheduleFailedToForwardUplayerSuccess) auto failedRsp = std::make_shared(); failedRsp->set_code(2); failedRsp->set_requestid("request1"); - EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(failedRsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule(_)).WillOnce(Return(AsyncReturn(failedRsp))); auto successRsp = std::make_shared(); successRsp->set_code(0); successRsp->set_requestid("request1"); - EXPECT_CALL(*mockDomainSrv_, ForwardSchedule(_)).WillOnce(Return(successRsp)); + EXPECT_CALL(*mockDomainSrv_, ForwardSchedule(_)).WillOnce(Return(AsyncReturn(successRsp))); messages::ScheduleRequest req; req.set_requestid("request1"); @@ -561,9 +535,9 @@ TEST_F(UnderlayerSchedMgrTest, NotifyAbnormalSuccess) failStatus.SetFailed(100); litebus::Future msg2; EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)) - .WillOnce(testing::DoAll(FutureArg<0>(&msg), Return(Status::OK()))) - .WillOnce(testing::DoAll(FutureArg<0>(&msg1), Return(failStatus))) - .WillRepeatedly(testing::DoAll(FutureArg<0>(&msg2), Return(Status::OK()))); + .WillOnce(testing::DoAll(FutureArg<0>(&msg), Return(AsyncReturn(Status::OK())))) + .WillOnce(testing::DoAll(FutureArg<0>(&msg1), Return(AsyncReturn(failStatus)))) + .WillRepeatedly(testing::DoAll(FutureArg<0>(&msg2), Return(AsyncReturn(Status::OK())))); messages::NotifySchedAbnormalRequest req; req.set_schedname("request"); @@ -588,7 +562,7 @@ TEST_F(UnderlayerSchedMgrTest, NotifySchedAbnormalSuccess) auto mockUnderlayerActor = std::make_shared("NotifySchedAbnormal"); litebus::Spawn(mockUnderlayerActor); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseNotifySchedAbnormal(_, _, _)) @@ -613,7 +587,7 @@ TEST_F(UnderlayerSchedMgrTest, NotifyWorkerStatusSuccess) { auto mockUnderlayerActor = std::make_shared("NotifySchedAbnormal"); litebus::Spawn(mockUnderlayerActor); - EXPECT_CALL(*mockDomainSrv_, NotifyWorkerStatus(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifyWorkerStatus(_)).WillOnce(Return(AsyncReturn(Status::OK()))); litebus::Future msg; EXPECT_CALL(*mockUnderlayerActor, MockResponseNotifyWorkerStatus(_, _, _)) .WillOnce(testing::DoAll(FutureArg<2>(&msg))); @@ -643,7 +617,7 @@ TEST_F(UnderlayerSchedMgrTest, UnfinishedscheduleRequest) auto mockUnderlayerActor = std::make_shared("WillRegister"); litebus::Spawn(mockUnderlayerActor); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); litebus::Promise pro; EXPECT_CALL(*mockUnderlayerActor, MockSchedule(_, _, _)) .WillOnce(::testing::Invoke([pro](const litebus::AID &, std::string, std::string) { @@ -749,7 +723,7 @@ TEST_F(UnderlayerSchedMgrTest, Reserve) EXPECT_CALL(*mockLocalGroupCtrl, MockReserve).WillRepeatedly(Return("xxxxx")); mockUnderlayerActor->ClosePingPong(); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(0)).Times(1); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); auto future = underlayer.Reserve("WillRegister", req); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get()->code(), (int32_t)StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); @@ -789,7 +763,7 @@ TEST_F(UnderlayerSchedMgrTest, UnReserve) EXPECT_CALL(*mockLocalGroupCtrl, MockUnReserve).WillRepeatedly(Return("xxxxx")); mockUnderlayerActor->ClosePingPong(); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(0)).Times(1); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); auto future = underlayer.UnReserve("WillRegister", req); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get().StatusCode(), (int32_t)StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); @@ -829,7 +803,7 @@ TEST_F(UnderlayerSchedMgrTest, Bind) EXPECT_CALL(*mockLocalGroupCtrl, MockBind).WillRepeatedly(Return("xxxxx")); mockUnderlayerActor->ClosePingPong(); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(0)).Times(1); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); auto future = underlayer.Bind("WillRegister", req); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get().StatusCode(), (int32_t)StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); @@ -869,7 +843,7 @@ TEST_F(UnderlayerSchedMgrTest, UnBind) EXPECT_CALL(*mockLocalGroupCtrl, MockUnBind).WillRepeatedly(Return("xxxxx")); mockUnderlayerActor->ClosePingPong(); EXPECT_CALL(*mockInstanceCtrl_, UpdateMaxSchedRetryTimes(0)).Times(1); - EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockDomainSrv_, NotifySchedAbnormal(_)).WillOnce(Return(AsyncReturn(Status::OK()))); auto future = underlayer.UnBind("WillRegister", req); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get().StatusCode(), (int32_t)StatusCode::DOMAIN_SCHEDULER_UNAVAILABLE_SCHEDULER); diff --git a/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_stub.h b/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_stub.h index 19380773d65e9bcbfa5bc806d38f20a65a3b9900..c4811a9db65d696a2352b3bf27aaf2537287f6b1 100644 --- a/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_stub.h +++ b/functionsystem/tests/unit/domain_scheduler/underlayer_scheduler_manager/underlayer_stub.h @@ -22,7 +22,7 @@ #include #include -#include "heartbeat/ping_pong_driver.h" +#include "common/heartbeat/heartbeat_client.h" #include "domain_scheduler/underlayer_scheduler_manager/underlayer_sched_mgr_actor.h" namespace functionsystem::test { @@ -82,9 +82,16 @@ public: void ClosePingPong() { + pingpong_->Stop(); pingpong_ = nullptr; } + void StartPingPong() + { + (void)pingpong_->Start( + litebus::AID(HEARTBEAT_OBSERVER_BASENAME + DOMAIN_UNDERLAYER_SCHED_MGR_ACTOR_NAME, GetAID().Url())); + } + void DeletePodResponse(const litebus::AID &from, std::string &&name, std::string &&msg) { auto deletePodResponse = std::make_shared(); @@ -117,13 +124,11 @@ protected: Receive("ResponseNotifyWorkerStatus", &MockUnderlayer::ResponseNotifyWorkerStatus); Receive("DeletePodResponse", &MockUnderlayer::DeletePodResponse); Receive("PreemptInstances", &MockUnderlayer::PreemptInstance); - pingpong_ = std::make_unique(GetAID().Name(), 6000, - // while connection lost, try to register - [this](const litebus::AID &lostDst, HeartbeatConnection) {}); + pingpong_ = std::make_unique(GetAID().Name(), [this](const litebus::AID &lostDst) {}); } private: - std::unique_ptr pingpong_; + std::unique_ptr pingpong_; }; class MockLocalGroupCtrl : public litebus::ActorBase { @@ -170,6 +175,6 @@ protected: Receive("UnBind", &MockLocalGroupCtrl::UnBind); } }; -} +} // namespace functionsystem::test #endif // TEST_UNIT_DOMAIN_SCHEDULER_UNDERLAYER_SCHEDULER_MANAGER_UNDERLAYER_STUB_H diff --git a/functionsystem/tests/unit/function_agent/CMakeLists.txt b/functionsystem/tests/unit/function_agent/CMakeLists.txt index 7182b2e9b8466717d413b83e032662a425f9d368..b01bf4f7418d29d8db971580b7af31ab71b6782d 100644 --- a/functionsystem/tests/unit/function_agent/CMakeLists.txt +++ b/functionsystem/tests/unit/function_agent/CMakeLists.txt @@ -15,5 +15,6 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR} FUNCTION_AGENT_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/agent_service FUNCTION_AGENT_TEST_SRCS) aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/common FUNCTION_AGENT_TEST_SRCS) +aux_source_directory(${CMAKE_CURRENT_LIST_DIR}/network FUNCTION_AGENT_TEST_SRCS) target_sources(${UNIT_TEST_MODULE} PRIVATE ${FUNCTION_AGENT_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_agent/agent_service/agent_service_actor_test.cpp b/functionsystem/tests/unit/function_agent/agent_service/agent_service_actor_test.cpp index 677cf5e4435b1f313611458c33c125ec1d4ca2de..fe7c05cf102e00a7c0aaaae2d3d9f0c907bf146b 100644 --- a/functionsystem/tests/unit/function_agent/agent_service/agent_service_actor_test.cpp +++ b/functionsystem/tests/unit/function_agent/agent_service/agent_service_actor_test.cpp @@ -23,20 +23,23 @@ #include "agent_service_test_actor.h" #include "common/constants/actor_name.h" -#include "logs/logging.h" -#include "metadata/metadata.h" +#include "common/logs/logging.h" +#include "common/metadata/metadata.h" #define private public // only for test -#include "metrics/metrics_adapter.h" +#include "common/metrics/metrics_adapter.h" #undef private // reset +#include "common/types/instance_state.h" #include "common/utils/exec_utils.h" #include "common/utils/hash_util.h" -#include "hex/hex.h" +#include "common/hex/hex.h" #include "common/utils/struct_transfer.h" #include "function_agent/code_deployer/copy_deployer.h" #include "function_agent/code_deployer/local_deployer.h" #include "function_agent/code_deployer/working_dir_deployer.h" +#include "function_agent/code_deployer/shared_dir_deployer.h" #include "function_agent/common/constants.h" #include "mocks/mock_agent_s3_deployer.h" +#include "mocks/mock_test_agent_s3_deployer.h" #include "mocks/mock_exec_utils.h" #include "utils/future_test_helper.h" @@ -62,6 +65,11 @@ namespace { const std::string TEST_LAYER_OBJECT_ID = "testObjectID-layer"; const std::string TEST_LAYER_OBJECT_ID_2 = "testObjectID-layer2"; const std::string LOCAL_DEPLOY_DIR = "/home/local/test"; + const std::string TEST_WRONG_FIREWALL_CONFIG = R"([{"firewallConfig": {"chain": "OUTPUT", "table": "", "operation": "add", "target": "127.0.0.1", "args": "-j ACCEPT"}}])"; + const std::string TEST_WRONG_TUNNEL_CONFIG = R"([{"tunnelConfig": {"tunnelName": "", "remoteIP": "127.0.0.1", "mode": "ipip"}}])"; + const std::string TEST_WRONG_ROUTE_CONFIG = R"([{"routeConfig": {"gateway": "127.0.0.1", "cidr": ""}}])"; + const std::string TEST_WRONG_PROBER_CONFIG = R"([{"protocol": "ICMP", "address": "127.0.0.1", "interval": 1, "timeout": 1, "failureThreshold": 1}])"; + const std::string TEST_PROBER_CONFIG = R"([{"protocol": "ICMP", "address": "127.0.0.1", "interval": 1, "timeout": 1, "failureThreshold": 1}])"; const std::string TEST_PODIP_IPSET_NAME = "test-podip-whitelist"; // length cannot exceed 31 const std::string TEST_TENANT_ID = "tenant001"; @@ -88,6 +96,7 @@ public: { auto deployer = std::make_shared(); auto workingDirDeployer = std::make_shared(); + auto sharedDirDeployer = std::make_shared(); auto s3Config = std::make_shared(); messages::CodePackageThresholds codePackageThresholds; auto mockDeployer = std::make_shared(s3Config, codePackageThresholds); @@ -100,9 +109,11 @@ public: dstActor_->SetDeployers(function_agent::S3_STORAGE_TYPE, mockDeployer); dstActor_->SetDeployers(function_agent::LOCAL_STORAGE_TYPE, deployer); dstActor_->SetDeployers(function_agent::WORKING_DIR_STORAGE_TYPE, workingDirDeployer); + dstActor_->SetDeployers(SHARED_DIR_STORAGE_TYPE, sharedDirDeployer); dstActor_->isRegisterCompleted_ = true; dstActor_->isUnitTestSituation_ = true; dstActor_->SetIpsetName(TEST_PODIP_IPSET_NAME); + dstActor_->SetPingTimeoutMs(1000); std::shared_ptr isolation = std::make_shared(dstActor_->GetIpsetName()); commandRunner_ = std::make_shared(); isolation->SetCommandRunner(commandRunner_); @@ -118,6 +129,7 @@ public: testFuncAgentMgrActor_->actorMessageList_.emplace("UpdateCred"); testFuncAgentMgrActor_->actorMessageList_.emplace("SetNetworkIsolationRequest"); testFuncAgentMgrActor_->actorMessageList_.emplace("QueryDebugInstanceInfos"); + testFuncAgentMgrActor_->actorMessageList_.emplace("NotifyFunctionStatusChange"); litebus::Spawn(testFuncAgentMgrActor_, true); testMetricsActor_ = std::make_shared("testMetricsActor"); @@ -372,6 +384,62 @@ TEST_F(AgentServiceActorTest, DeployInstanceErrorRequest) } +/** + * Feature: AgentServiceActor--DeployInstanceSetNetworkFailed + * Description: deploy instance fail with error network configuration + * Steps: + * 1. Create wrong networkConfig json value and set createoptions in DeployInstanceRequest and then send request + * 2. Create empty networkConfig json value and wrong proberConfig json value and set createoptions in DeployInstanceRequest + * and then send request + * 3. Create empty networkConfig json value and set createoptions in DeployInstanceRequest and then send request + * and simulate RuntimeManager to send err StartInstanceResponse + * Expectation: + * 1. Cause set network err, gentServiceActor will send DeployInstanceResponse with errcode FUNC_AGENT_SET_NETWORK_ERROR + * back to FunctionAgentMgrActor but not send StartInstance request to RuntimeManager + * 2. Cause set network err, gentServiceActor will send DeployInstanceResponse with errcode FUNC_AGENT_NETWORK_WORK_ERROR + * back to FunctionAgentMgrActor but not send StartInstance request to RuntimeManager + * 3. RuntimeManager will receive StartInstance request from AgentServiceActor, but FunctionAgentMgrActor won't receive DeployInstanceResponse + */ +TEST_F(AgentServiceActorTest, DeployInstanceSetNetworkFailed) +{ + messages::DeployInstanceRequest deployInstance; + deployInstance.set_requestid(TEST_REQUEST_ID); + deployInstance.set_instanceid(TEST_INSTANCE_ID); + deployInstance.mutable_funcdeployspec()->set_storagetype(function_agent::LOCAL_STORAGE_TYPE); + EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse).WillRepeatedly(Return("invalid msg")); // send err response + auto deployer = std::make_shared(); + dstActor_->SetDeployers(function_agent::LOCAL_STORAGE_TYPE, deployer); + // error network config --> SetFireWall Failed + deployInstance.mutable_createoptions()->operator[]("networkConfig") = TEST_WRONG_FIREWALL_CONFIG; + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "DeployInstance", + std::move(deployInstance.SerializeAsString())); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->code(), StatusCode::FUNC_AGENT_SET_NETWORK_ERROR); + testFuncAgentMgrActor_->ResetDeployInstanceResponse(); + EXPECT_EQ(testRuntimeManager_->GetReceivedStartInstanceRequest(), false); + // error prober config + deployInstance.mutable_createoptions()->operator[]("networkConfig") = ""; + deployInstance.mutable_createoptions()->operator[]("proberConfig") = "invalid config $$"; + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "DeployInstance", + std::move(deployInstance.SerializeAsString())); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->code(), StatusCode::FUNC_AGENT_NETWORK_WORK_ERROR); + testFuncAgentMgrActor_->ResetDeployInstanceResponse(); + EXPECT_EQ(testRuntimeManager_->GetReceivedStartInstanceRequest(), false); + // success + deployInstance.mutable_createoptions()->erase("proberConfig"); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "DeployInstance", + std::move(deployInstance.SerializeAsString())); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), ""); + EXPECT_EQ(testRuntimeManager_->GetReceivedStartInstanceRequest(), true); +} + /** * Feature: AgentServiceActor--DeployInstanceAboutRuntimeMgrRegistration * Description: deploy instance with and without Runtime Manager Registration @@ -698,6 +766,20 @@ TEST_F(AgentServiceActorTest, RepeatedlyDeployInstanceWithOneLayersAndDelegateVi EXPECT_TRUE(litebus::os::Rmdir(delegateDestination).IsNone()); } +std::string DELEGATE_LAYER_DESTINATION = "/home/test/layer/testUserLibCodeBucketID/testUserLibCodeObjectID"; +std::string DELEGATE_DESTINATION = "/home/test/layer/func/testUserCodeBucketID/testUserCodeObjectID"; +std::string DESTINATION = "/home/layer/func/" + TEST_BUCKET_ID + "/" + TEST_OBJECT_ID; + +void BuildCreateOptions(const std::unique_ptr &request, const std::string &code, + const std::string &layer) +{ + request->mutable_createoptions()->erase("DELEGATE_DOWNLOAD"); + request->mutable_createoptions()->erase("DELEGATE_LAYER_DOWNLOAD"); + // add delegate code + request->mutable_createoptions()->insert({ "DELEGATE_DOWNLOAD", code }); + request->mutable_createoptions()->insert({ "DELEGATE_LAYER_DOWNLOAD", layer }); +} + /** * Feature: AgentServiceActor--DeployInstanceWithDelegateCode * Description: Deploy instance with user delegate code and lib @@ -706,13 +788,15 @@ TEST_F(AgentServiceActorTest, RepeatedlyDeployInstanceWithOneLayersAndDelegateVi * 2. set DELEGATE_DOWNLOAD, DELEGATE_LAYER_DOWNLOAD and S3_DEPLOY_DIR * 3. Mock FunctionAgentMgrActor to send DeployInstance, mock RuntimeManager to return StartInstanceReponse, and mock * S3Deployer to download code packages and create dir. - * 4. First send, DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD both have hostName, token, temporayAccessKey and temporarySecretKey - * 5. Second send, DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD both have token, temporayAccessKey and temporarySecretKey, without hostName + * 4. First send, DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD both have hostName, token, temporayAccessKey and + * temporarySecretKey + * 5. Second send, DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD both have token, temporayAccessKey and + * temporarySecretKey, without hostName * 6. Third send, DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD both just have hostName * 7. Forth send, DELEGATE_DOWNLOAD with local file * Expectation: - * 1. First deploy, runtime code, delegate lib code and delegate code should create dir respectively, each one has one code refer, - * FunctionAgentMgrActor will receive DeployInstanceResponse + * 1. First deploy, runtime code, delegate lib code and delegate code should create dir respectively, each one has one + * code refer, FunctionAgentMgrActor will receive DeployInstanceResponse * 2. Second deploy, runtime code, delegate lib code and delegate code have two code refer respectively, * FunctionAgentMgrActor will receive DeployInstanceResponse again * 4. Third deploy, runtime code, delegate lib code and delegate code have three code refer respectively, @@ -730,110 +814,87 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithDelegateCode) spec->set_deploydir("/home"); spec->set_bucketid(TEST_BUCKET_ID); spec->set_objectid(TEST_OBJECT_ID); - // add delegate code - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_DOWNLOAD", - R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "hostName":"xx", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx","sha256":"","sha512":"aaaaaaaa"})" }); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_LAYER_DOWNLOAD", - R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx","sha256":"","sha512":"aaaaaaaa"}])" }); - // set extractly layer deploy dir - deployInstanceReq->mutable_createoptions()->insert( - { "S3_DEPLOY_DIR", - "/home/test" }); - std::string destination = "/home/layer/func/" + TEST_BUCKET_ID + "/" + TEST_OBJECT_ID; - std::string delegateLayerDestination = "/home/test/layer/testUserLibCodeBucketID/testUserLibCodeObjectID"; - std::string delegateDestination = "/home/test/layer/func/testUserCodeBucketID/testUserCodeObjectID"; - litebus::os::Rmdir(destination); - litebus::os::Rmdir(delegateLayerDestination); - litebus::os::Rmdir(delegateDestination); + + BuildCreateOptions( + deployInstanceReq, + R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "hostName":"xx", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx","sha256":"","sha512":"aaaaaaaa"})", + R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx","sha256":"","sha512":"aaaaaaaa"}])"); + deployInstanceReq->mutable_createoptions()->insert({ "S3_DEPLOY_DIR", "/home/test" }); + litebus::os::Rmdir(DESTINATION); + litebus::os::Rmdir(DELEGATE_LAYER_DESTINATION); + litebus::os::Rmdir(DELEGATE_DESTINATION); messages::StartInstanceResponse startInstanceResponse; startInstanceResponse.set_requestid(TEST_REQUEST_ID); EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) .WillOnce(Return(startInstanceResponse.SerializeAsString())); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); - ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(destination); }); - ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(delegateLayerDestination); }); - ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(delegateDestination); }); + ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(DESTINATION); }); + ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(DELEGATE_LAYER_DESTINATION); }); + ASSERT_AWAIT_TRUE([&]() -> bool { return litebus::os::ExistPath(DELEGATE_DESTINATION); }); std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), destination), static_cast(1)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateLayerDestination), static_cast(1)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateDestination), static_cast(1)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DESTINATION), static_cast(1)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_LAYER_DESTINATION), static_cast(1)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_DESTINATION), static_cast(1)); EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID); testFuncAgentMgrActor_->ResetDeployInstanceResponse(); // set DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD with empty hostName - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_LAYER_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_DOWNLOAD", - R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx"})" }); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_LAYER_DOWNLOAD", - R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx"}])" }); + BuildCreateOptions( + deployInstanceReq, + R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx"})", + R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "securityToken":"xxx", "temporayAccessKey":"xxx", "temporarySecretKey":"xxx"}])"); startInstanceResponse.set_requestid(TEST_REQUEST_ID_2); EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) .WillOnce(Return(startInstanceResponse.SerializeAsString())); deployInstanceReq->set_requestid(TEST_REQUEST_ID_2); deployInstanceReq->set_instanceid(TEST_INSTANCE_ID_2); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), destination), static_cast(2)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateLayerDestination), static_cast(2)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateDestination), static_cast(2)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DESTINATION), static_cast(2)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_LAYER_DESTINATION), static_cast(2)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_DESTINATION), static_cast(2)); EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID_2); testFuncAgentMgrActor_->ResetDeployInstanceResponse(); // set DELEGATE_DOWNLOAD and DELEGATE_LAYER_DOWNLOAD with just hostName - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_LAYER_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_DOWNLOAD", - R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "hostName":"xx"})" }); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_LAYER_DOWNLOAD", - R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx"}])" }); + BuildCreateOptions( + deployInstanceReq, + R"({"appId":"userCode", "bucketId":"testUserCodeBucketID", "objectId":"testUserCodeObjectID", "hostName":"xx"})", + R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx"}])"); startInstanceResponse.set_requestid(TEST_REQUEST_ID_3); EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) .WillOnce(Return(startInstanceResponse.SerializeAsString())); deployInstanceReq->set_requestid(TEST_REQUEST_ID_3); deployInstanceReq->set_instanceid(TEST_INSTANCE_ID_3); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), destination), static_cast(3)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateLayerDestination), static_cast(3)); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), delegateDestination), static_cast(3)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DESTINATION), static_cast(3)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_LAYER_DESTINATION), static_cast(3)); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), DELEGATE_DESTINATION), static_cast(3)); EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID_3); - // set DELEGATE_DOWNLOAD with local file - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->erase("DELEGATE_LAYER_DOWNLOAD"); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_DOWNLOAD", - R"({"appId":"", "bucketId":"", "objectId":"", "hostName":"xx", "storage_type": "local", "code_path": "/home/test/function-packages"})" }); - deployInstanceReq->mutable_createoptions()->insert( - { "DELEGATE_LAYER_DOWNLOAD", - R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx"}])" }); + BuildCreateOptions( + deployInstanceReq, + R"({"appId":"", "bucketId":"", "objectId":"", "hostName":"xx", "storage_type": "local", "code_path": "/home/test/function-packages"})", + R"([{"appId":"userCode-layer", "bucketId":"testUserLibCodeBucketID", "objectId":"testUserLibCodeObjectID", "hostName":"xx"}])"); startInstanceResponse.set_requestid("testRequestID4"); EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) .WillOnce(Return(startInstanceResponse.SerializeAsString())); deployInstanceReq->set_requestid("testRequestID4"); deployInstanceReq->set_instanceid("testInstanceID4"); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); - ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == "testRequestID4"; }); - EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), "/home/test/function-packages"), static_cast(1)); + ASSERT_AWAIT_TRUE( + [&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == "testRequestID4"; }); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), "/home/test/function-packages"), + static_cast(1)); - EXPECT_TRUE(litebus::os::Rmdir(destination).IsNone()); - EXPECT_TRUE(litebus::os::Rmdir(delegateLayerDestination).IsNone()); - EXPECT_TRUE(litebus::os::Rmdir(delegateDestination).IsNone()); + EXPECT_TRUE(litebus::os::Rmdir(DESTINATION).IsNone()); + EXPECT_TRUE(litebus::os::Rmdir(DELEGATE_LAYER_DESTINATION).IsNone()); + EXPECT_TRUE(litebus::os::Rmdir(DELEGATE_DESTINATION).IsNone()); } /** @@ -989,6 +1050,36 @@ TEST_F(AgentServiceActorTest, KillInstanceWithRespose) EXPECT_EQ(dstActor_->monopolyUsed_, true); } +TEST_F(AgentServiceActorTest, KillInstanceWithResposeCancelHeartbeat) +{ + auto killInstanceReq = std::make_unique(); + killInstanceReq->set_requestid(TEST_REQUEST_ID); + killInstanceReq->set_storagetype(function_agent::LOCAL_STORAGE_TYPE); + killInstanceReq->set_ismonopoly(true); + dstActor_->SetDeployers(function_agent::LOCAL_STORAGE_TYPE, std::make_shared()); + dstActor_->enableRestartForReuse_ = true; + messages::Registered registered; + dstActor_->StartPingPong(registered); + auto status = dstActor_->pingPongDriver_->GetStatus(); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsOk()); + + messages::StopInstanceResponse stopInstanceResponse; + stopInstanceResponse.set_code(StatusCode::SUCCESS); + stopInstanceResponse.set_requestid(TEST_REQUEST_ID); + EXPECT_CALL(*testRuntimeManager_, MockStopInstanceResponse) + .WillOnce(Return(stopInstanceResponse.SerializeAsString())); + + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "KillInstance", + std::move(killInstanceReq->SerializeAsString())); + ASSERT_AWAIT_TRUE([&]() { return testRuntimeManager_->GetReceivedStopInstanceRequest(); }); + EXPECT_EQ(dstActor_->monopolyUsed_, true); + status = dstActor_->pingPongDriver_->GetStatus(); + ASSERT_AWAIT_READY(status); + EXPECT_EQ(status.Get().StatusCode(), StatusCode::FAILED); + EXPECT_THAT(status.Get().ToString(), testing::HasSubstr("HeartbeatClient is stopped")); +} + TEST_F(AgentServiceActorTest, KillInstanceWithoutRuntimeMgrRegistration) { auto killInstanceReq = std::make_unique(); @@ -1652,6 +1743,36 @@ TEST_F(AgentServiceActorTest, UpdateTokenTest) [&]() { return testFuncAgentMgrActor_->GetUpdateTokenResponse()->requestid() == TEST_REQUEST_ID; }); } +/** + * Feature: AgentServiceActor--ProtectedSetNetworkTest + * Description: Config network, including firewall, tunnel and route + * Steps: + * 1. Construct wrong firewall config and set network + * 2. Construct wrong tunnel config and set network + * 2. Construct wrong route config and set network + * 2. Construct empty network config and set network + * Expectation: + * 1. Set firewall cause err, return false + * 1. Set tunnel cause err, return false + * 1. Set route cause err, return false + * 1. Skip set network, return true + */ +TEST_F(AgentServiceActorTest, ProtectedSetNetworkTest) +{ + bool state = false; + if (getenv("POD_IP") == nullptr) { + setenv("POD_IP", "127.0.0.1", 0); + state = true; + } + EXPECT_EQ(dstActor_->ProtectedSetNetwork(NetworkTool::ParseNetworkConfig(TEST_WRONG_FIREWALL_CONFIG)), false); + EXPECT_EQ(dstActor_->ProtectedSetNetwork(NetworkTool::ParseNetworkConfig(TEST_WRONG_TUNNEL_CONFIG)), false); + EXPECT_EQ(dstActor_->ProtectedSetNetwork(NetworkTool::ParseNetworkConfig(TEST_WRONG_ROUTE_CONFIG)), false); + EXPECT_EQ(dstActor_->ProtectedSetNetwork(NetworkTool::ParseNetworkConfig("")), true); + if (state) { + unsetenv("POD_IP"); + } +} + /** * Feature: AgentServiceActor--StartPingPongSuccess * Description: When AgentServiceActor registered, start pingpong to receive heartbeat from FunctionAgentMgrActor @@ -1683,13 +1804,13 @@ TEST_F(AgentServiceActorTest, TimeOutEventTest) RegisterInfo registerInfo; registerInfo.registeredPromise = litebus::Promise(); dstActor_->SetRegisterInfo(registerInfo); - dstActor_->TimeOutEvent(HeartbeatConnection::LOST); + dstActor_->TimeOutEvent(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); messages::Registered registered; auto registerResponseFuture = dstActor_->StartPingPong(registered); registerInfo.registeredPromise.SetFailed(static_cast(StatusCode::FUNC_AGENT_PING_PONG_IS_NULL)); dstActor_->SetRegisterInfo(registerInfo); - dstActor_->TimeOutEvent(HeartbeatConnection::LOST); + dstActor_->TimeOutEvent(); EXPECT_TRUE(dstActor_->GetPingPongDriver() != nullptr); } @@ -2172,9 +2293,16 @@ TEST_F(AgentServiceActorTest, RegisterAgentFailedTest) TEST_F(AgentServiceActorTest, GracefulShutdown) { + testFuncAgentMgrActor_->ResetReceivedUpdateAgentStatus(); auto fut = litebus::Async(dstActor_->GetAID(), &AgentServiceActor::GracefulShutdown); testRuntimeManager_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "GracefulShutdownFinish", ""); + EXPECT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetReceivedUpdateAgentStatus(); }); + messages::UpdateAgentStatusResponse updateAgentStatusRsp; + updateAgentStatusRsp.set_requestid(testFuncAgentMgrActor_->GetUpdateAgentStatusRequest()->requestid()); + EXPECT_CALL(*testFuncAgentMgrActor_, MockUpdateAgentStatusResponse) + .WillOnce(Return(updateAgentStatusRsp.SerializeAsString())); + dstActor_->TimeOutEvent(); EXPECT_TRUE(fut.Get()); } @@ -2182,7 +2310,7 @@ TEST_F(AgentServiceActorTest, RestartForReuse) { dstActor_->monopolyUsed_ = true; dstActor_->enableRestartForReuse_ = true; - litebus::Async(dstActor_->GetAID(), &AgentServiceActor::TimeOutEvent, HeartbeatConnection::LOST); + litebus::Async(dstActor_->GetAID(), &AgentServiceActor::TimeOutEvent); EXPECT_TRUE(dstActor_->runtimeManagerGracefulShutdown_.GetFuture().Get()); } @@ -2197,6 +2325,12 @@ TEST_F(AgentServiceActorTest, SetNetworkIsolationPodIpSuccessAddDelete) CommandExecResult result5; result5.output = "Name: test-podip-whitelist\nMembers:\n192.168.1.1"; result5.error = ""; + EXPECT_CALL(*commandRunner_, ExecuteCommandWrapper(_)) + .WillOnce(Return(result)) // ipset list result #1 + .WillOnce(Return(result3)) // #3 + .WillOnce(Return(result5)); // #5 + + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)).WillRepeatedly(Return(true)); // #2, #4 // add more messages::SetNetworkIsolationRequest req; @@ -2206,7 +2340,7 @@ TEST_F(AgentServiceActorTest, SetNetworkIsolationPodIpSuccessAddDelete) (*req.mutable_rules()->Add()) = "192.168.2.1"; auto response = testFuncAgentMgrActor_->GetSetNetworkIsolationResponse(); - response->set_code(StatusCode::SUCCESS); // must do reset + response->set_code(StatusCode::FAILED); // must do reset response->set_requestid(""); // must do reset testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "SetNetworkIsolationRequest", // #1, #2 @@ -2214,7 +2348,7 @@ TEST_F(AgentServiceActorTest, SetNetworkIsolationPodIpSuccessAddDelete) EXPECT_AWAIT_TRUE([&]() -> bool { return response->requestid() == TEST_REQUEST_ID; }); - EXPECT_EQ(response->code(), StatusCode::FAILED); + EXPECT_EQ(response->code(), StatusCode::SUCCESS); // delete messages::SetNetworkIsolationRequest req2; @@ -2222,16 +2356,72 @@ TEST_F(AgentServiceActorTest, SetNetworkIsolationPodIpSuccessAddDelete) req2.set_ruletype(messages::RuleType::IPSET_DELETE); (*req2.mutable_rules()->Add()) = "192.168.2.1"; - response->set_code(StatusCode::SUCCESS); // must do reset + response->set_code(StatusCode::FAILED); // must do reset response->set_requestid(""); // must do reset testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "SetNetworkIsolationRequest", // #3, #4 req2.SerializeAsString()); EXPECT_AWAIT_TRUE([&]() -> bool { - return response->code() == StatusCode::FAILED; + return response->code() == StatusCode::SUCCESS; }); + + auto ipsetIsolation = dstActor_->GetIpsetIpv4NetworkIsolation(); + EXPECT_EQ(ipsetIsolation->GetAllRules().size(), static_cast(1)); // #5 } +TEST_F(AgentServiceActorTest, SetNetworkIsolationRuleFlush) +{ + CommandExecResult result; + result.output = "Name: test-podip-whitelist\nMembers:\n"; + result.error = ""; + CommandExecResult result3; + result3.output = "Name: test-podip-whitelist\nMembers:\n192.168.1.1"; + result3.error = ""; + EXPECT_CALL(*commandRunner_, ExecuteCommandWrapper(_)) + .WillOnce(Return(result)) // #1 + .WillOnce(Return(result3)) // #3 + .WillOnce(Return(result)); // #5 + + EXPECT_CALL(*commandRunner_, CheckAndRunCommandWrapper(_)) + .WillOnce(Return(true)) // #2 + .WillOnce(Return(true)); // #4 + + // add one + messages::SetNetworkIsolationRequest req; + req.set_requestid(TEST_REQUEST_ID); + req.set_ruletype(messages::RuleType::IPSET_ADD); + (*req.mutable_rules()->Add()) = "192.168.1.1"; + + const auto response = testFuncAgentMgrActor_->GetSetNetworkIsolationResponse(); + response->set_code(StatusCode::FAILED); // must do reset + response->set_requestid(""); // must do reset + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "SetNetworkIsolationRequest", // #1, #2 + req.SerializeAsString()); + EXPECT_AWAIT_TRUE([&]() -> bool { + return response->requestid() == TEST_REQUEST_ID; + }); + EXPECT_AWAIT_TRUE([&]() -> bool { + return response->code() == StatusCode::SUCCESS; + }); + + // flush + messages::SetNetworkIsolationRequest req2; + req2.set_requestid(TEST_REQUEST_ID_3); + req2.set_ruletype(messages::RuleType::IPSET_FLUSH); + + response->set_code(StatusCode::FAILED); // must do reset + response->set_requestid(""); // must do reset + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "SetNetworkIsolationRequest", // #3, #4 + req2.SerializeAsString()); + EXPECT_AWAIT_TRUE([&]() -> bool { + return response->code() == StatusCode::SUCCESS; + }); + + const auto ipsetIsolation = dstActor_->GetIpsetIpv4NetworkIsolation(); + EXPECT_EQ(ipsetIsolation->GetAllRules().size(), static_cast(0)); // #5 +} TEST_F(AgentServiceActorTest, RegisteredEvictedTest) { @@ -2320,7 +2510,7 @@ TEST_F(AgentServiceActorTest, PythonRuntime_Support_WorkingDirFileZip_WithOut_En { PrepareWorkingDir("/tmp/working_dir-tmp"); auto deployInstanceReq = std::make_unique(); - deployInstanceReq->set_requestid(TEST_REQUEST_ID); // as appID + deployInstanceReq->set_requestid(TEST_REQUEST_ID); // as appID deployInstanceReq->set_instanceid(TEST_INSTANCE_ID); deployInstanceReq->set_language("/usr/bin/python3.9"); auto spec = deployInstanceReq->mutable_funcdeployspec(); @@ -2330,16 +2520,17 @@ TEST_F(AgentServiceActorTest, PythonRuntime_Support_WorkingDirFileZip_WithOut_En auto destination = "/home/sn/function/package/xxxz/app/working_dir/" + CalculateFileMD5(workingDirFile.substr(7)); (void)litebus::os::Rmdir(deployDir); spec->set_deploydir(deployDir); - std::string optionDetail = "{\"appId\":\"userWorkingDirCode001\", \"storage_type\":\"working_dir\", \"code_path\":\""; + std::string optionDetail = + "{\"appId\":\"userWorkingDirCode001\", \"storage_type\":\"working_dir\", \"code_path\":\""; optionDetail += workingDirFile; optionDetail += "\"}"; - deployInstanceReq->mutable_createoptions()->insert({"DELEGATE_DOWNLOAD", optionDetail}); - deployInstanceReq->mutable_createoptions()->insert({CONDA_CONFIG, "{'test_conda_config': 'confit_content'}"}); - deployInstanceReq->mutable_createoptions()->insert({CONDA_COMMAND, "conda create -n test_env python=3.11"}); + deployInstanceReq->mutable_createoptions()->insert({ "DELEGATE_DOWNLOAD", optionDetail }); + deployInstanceReq->mutable_createoptions()->insert({ CONDA_CONFIG, "{'test_conda_config': 'confit_content'}" }); + deployInstanceReq->mutable_createoptions()->insert({ CONDA_COMMAND, "conda create -n test_env python=3.11" }); std::string testCondaPrefix = "/tmp/conda"; std::string testCondaDefaultEnv = "env_name_copy"; - deployInstanceReq->mutable_createoptions()->insert({CONDA_PREFIX, testCondaPrefix}); - deployInstanceReq->mutable_createoptions()->insert({CONDA_DEFAULT_ENV, testCondaDefaultEnv}); + deployInstanceReq->mutable_createoptions()->insert({ CONDA_PREFIX, testCondaPrefix }); + deployInstanceReq->mutable_createoptions()->insert({ CONDA_DEFAULT_ENV, testCondaDefaultEnv }); deployInstanceReq->set_tenantid(TEST_TENANT_ID); messages::StartInstanceResponse startInstanceResponse; startInstanceResponse.set_code(StatusCode::SUCCESS); @@ -2349,19 +2540,20 @@ TEST_F(AgentServiceActorTest, PythonRuntime_Support_WorkingDirFileZip_WithOut_En .WillOnce(Return(startInstanceResponse.SerializeAsString())); testFuncAgentMgrActor_->ResetDeployInstanceResponse(); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); - ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == TEST_REQUEST_ID; }); - EXPECT_TRUE(litebus::os::ExistPath(destination)); // app deployed + ASSERT_AWAIT_TRUE( + [&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == TEST_REQUEST_ID; }); + EXPECT_TRUE(litebus::os::ExistPath(destination)); // app deployed auto startInstanceRequest = std::make_shared(); startInstanceRequest->ParseFromString(testRuntimeManager_->promiseOfStartInstanceRequest.GetFuture().Get()); YRLOG_DEBUG(startInstanceRequest->ShortDebugString()); - EXPECT_EQ(startInstanceRequest->runtimeinstanceinfo().runtimeconfig().posixenvs().find(UNZIPPED_WORKING_DIR)->second, - destination); // startInstance param posixenvs should contains UNZIPPED_WORKING_DIR + EXPECT_EQ( + startInstanceRequest->runtimeinstanceinfo().runtimeconfig().posixenvs().find(UNZIPPED_WORKING_DIR)->second, + destination); // startInstance param posixenvs should contain UNZIPPED_WORKING_DIR EXPECT_EQ(startInstanceRequest->runtimeinstanceinfo().runtimeconfig().posixenvs().find(YR_WORKING_DIR)->second, - workingDirFile); // startInstance param posixenvs should contains YR_WORKING_DIR + workingDirFile); // startInstance param posixenvs should contain YR_WORKING_DIR EXPECT_EQ(startInstanceRequest->runtimeinstanceinfo().runtimeconfig().posixenvs().find(YR_TENANT_ID)->second, TEST_TENANT_ID); auto iter = startInstanceRequest->runtimeinstanceinfo().deploymentconfig().deployoptions().end(); @@ -2450,12 +2642,7 @@ TEST_F(AgentServiceActorTest, AppDriver_Support_DeployInstanceWithWorkingDirDepl return testRuntimeManager_->GetReceivedStopInstanceRequest() && testFuncAgentMgrActor_->GetKillInstanceResponse()->code() == StatusCode::SUCCESS; }); - - messages::CleanStatusRequest cleanStatusRequest; - cleanStatusRequest.set_name(TEST_AGENT_ID); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "CleanStatus", - std::move(cleanStatusRequest.SerializeAsString())); + dstActor_->RemoveCodePackageAsync(); ASSERT_AWAIT_TRUE([&]() -> bool { return !litebus::os::ExistPath(destination); }); // clean after app killed DestroyWorkingDir("/tmp/working_dir-tmp"); } @@ -2654,7 +2841,40 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDirDeployer_Ray_Serve_Wit workingDirFile); // startInstance param posixenvs should contains YR_WORKING_DIR EXPECT_EQ(startInstanceRequest->runtimeinstanceinfo().runtimeconfig().posixenvs().find(YR_APP_MODE)->second, "false"); + DestroyWorkingDir("/tmp/working_dir-tmp"); +} +TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDir_ErrorInputFile_Createoption_WorkingDirFile) +{ + PrepareWorkingDir("/tmp/working_dir-tmp"); + auto deployInstanceReq = std::make_unique(); + deployInstanceReq->set_requestid(TEST_REQUEST_ID); // as appID + deployInstanceReq->set_instanceid(TEST_INSTANCE_ID); + deployInstanceReq->set_language("posix-custom-runtime"); + auto appEntryPoint = "python script.py"; + deployInstanceReq->set_entryfile(appEntryPoint); // app entrypoint set from proxy + auto spec = deployInstanceReq->mutable_funcdeployspec(); + spec->set_storagetype(function_agent::WORKING_DIR_STORAGE_TYPE); + auto deployDir = "/home/sn/function/package/xxxz"; + auto destination = "/home/sn/function/package/xxxz/app/working_dir/" + TEST_INSTANCE_ID; + (void)litebus::os::Rmdir(deployDir); + EXPECT_TRUE(!litebus::os::ExistPath(destination)); + spec->set_deploydir(deployDir); + deployInstanceReq->mutable_createoptions()->insert({ APP_ENTRYPOINT, appEntryPoint }); + // file is not zip file , it is dir + deployInstanceReq->mutable_createoptions()->insert( + { "DELEGATE_DOWNLOAD", + R"({"appId":"userWorkingDirCode001", "storage_type":"working_dir", "code_path":"file:///tmp/working_dir-tmp/"})" }); + EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse).Times(0); + testFuncAgentMgrActor_->ResetDeployInstanceResponse(); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", + std::move(deployInstanceReq->SerializeAsString())); + ASSERT_AWAIT_TRUE([&]() -> bool { + return (testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == TEST_REQUEST_ID + && testFuncAgentMgrActor_->GetDeployInstanceResponse()->code() + == StatusCode::FUNC_AGENT_INVALID_WORKING_DIR_FILE); + }); + EXPECT_TRUE(!litebus::os::ExistPath(destination)); DestroyWorkingDir("/tmp/working_dir-tmp"); } @@ -2672,6 +2892,7 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDir_ErrorInput_Createopti auto deployDir = "/home/sn/function/package/xxxz"; auto destination = "/home/sn/function/package/xxxz/app/working_dir/" + TEST_INSTANCE_ID; (void)litebus::os::Rmdir(deployDir); + EXPECT_TRUE(!litebus::os::ExistPath(destination)); spec->set_deploydir(deployDir); deployInstanceReq->mutable_createoptions()->insert({ APP_ENTRYPOINT, appEntryPoint }); deployInstanceReq->mutable_createoptions()->insert( @@ -2692,76 +2913,10 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDir_ErrorInput_Createopti "CleanStatus", std::move(cleanStatusRequest.SerializeAsString())); ASSERT_AWAIT_TRUE([&]() -> bool { return !litebus::os::ExistPath(destination); }); // app deploy error + EXPECT_TRUE(!litebus::os::ExistPath(destination)); DestroyWorkingDir("/tmp/working_dir-tmp"); } -TEST_F(AgentServiceActorTest, ParallelDeployInstanceWithS3Deployer) -{ - auto deployInstanceReq1 = GetDeployInstanceRequest("req-11111", "instance1-150000","testBucketID11", "testObjectID11"); - auto deployInstanceReq2 = GetDeployInstanceRequest("req-11112", "instance2-150000","testBucketID12", "testObjectID12"); - auto deployInstanceReq3 = GetDeployInstanceRequest("req-11113", "instance3-150000","testBucketID13", "testObjectID13"); - std::string destination1 = "/home/layer/func/testBucketID11/testObjectID11"; - std::string destination2 = "/home/layer/func/testBucketID12/testObjectID12"; - std::string destination3 = "/home/layer/func/testBucketID13/testObjectID13"; - litebus::os::Rmdir(destination1); - litebus::os::Rmdir(destination2); - litebus::os::Rmdir(destination3); - - messages::StartInstanceResponse startInstanceResponse1; - startInstanceResponse1.set_code(StatusCode::SUCCESS); - startInstanceResponse1.set_requestid("req-11111"); - startInstanceResponse1.mutable_startruntimeinstanceresponse()->set_runtimeid("test-runtime-111"); - messages::StartInstanceResponse startInstanceResponse2; - startInstanceResponse2.set_code(StatusCode::SUCCESS); - startInstanceResponse2.set_requestid("req-11112"); - startInstanceResponse2.mutable_startruntimeinstanceresponse()->set_runtimeid("test-runtime-112"); - messages::StartInstanceResponse startInstanceResponse3; - startInstanceResponse3.set_code(StatusCode::SUCCESS); - startInstanceResponse3.set_requestid("req-11113"); - startInstanceResponse3.mutable_startruntimeinstanceresponse()->set_runtimeid("test-runtime-113"); - EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) - .WillOnce(Return(startInstanceResponse1.SerializeAsString())) - .WillOnce(Return(startInstanceResponse2.SerializeAsString())) - .WillOnce(Return(startInstanceResponse3.SerializeAsString())); - testFuncAgentMgrActor_->ResetDeployInstanceResponse(); - auto start = std::chrono::steady_clock::now(); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", - std::move(deployInstanceReq1->SerializeAsString())); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", - std::move(deployInstanceReq2->SerializeAsString())); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), - "DeployInstance", - std::move(deployInstanceReq3->SerializeAsString())); - ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponseMap().size() == 3; }); - EXPECT_TRUE(litebus::os::ExistPath(destination1)); - EXPECT_TRUE(litebus::os::ExistPath(destination2)); - EXPECT_TRUE(litebus::os::ExistPath(destination3)); - auto end = std::chrono::steady_clock::now(); - EXPECT_TRUE(std::chrono::duration_cast(end - start).count() <= 200); - litebus::os::Rmdir(destination1); - litebus::os::Rmdir(destination2); - litebus::os::Rmdir(destination3); -} - -TEST_F(AgentServiceActorTest, ConfigCodeAgingTimeTest) -{ - dstActor_->codePackageThresholds_.set_codeagingtime(10); - dstActor_->codeReferInfos_->clear(); - dstActor_->AddCodeRefer("/tmp/test1", "testIns001", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); - dstActor_->AddCodeRefer("/tmp/test2", "testIns002", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); - dstActor_->AddCodeRefer("/tmp/test2", "testIns003", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); - dstActor_->AddCodeRefer("/tmp/test3", "testIns004", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); - dstActor_->DeleteFunction("/tmp/test2", "testIns003"); - dstActor_->DeleteFunction("/tmp/test3", "testIns004"); - (*dstActor_->codeReferInfos_)["/tmp/test1"].lastAccessTimestamp = 1700000; - (*dstActor_->codeReferInfos_)["/tmp/test3"].lastAccessTimestamp = 1700000; - dstActor_->RemoveCodePackageAsync(); - EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test1") != dstActor_->codeReferInfos_->end()); - EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test2") != dstActor_->codeReferInfos_->end()); - EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test3") == dstActor_->codeReferInfos_->end()); -} TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDirCpp) { @@ -2774,11 +2929,12 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDirCpp) auto spec = deployInstanceReq->mutable_funcdeployspec(); spec->set_storagetype(function_agent::WORKING_DIR_STORAGE_TYPE); auto deployDir = "/home/sn/function/package/xxxz"; - std::string destination = "/tmp/working_dir-tmp/file.zip"; + auto destination = "/tmp/working_dir-tmp/"; (void)litebus::os::Rmdir(deployDir); EXPECT_TRUE(litebus::os::ExistPath(destination)); spec->set_deploydir(deployDir); + // path deployInstanceReq->mutable_createoptions()->insert( { "DELEGATE_DOWNLOAD", R"({"appId":"userWorkingDirCode001", "storage_type":"working_dir", "code_path":"/tmp/working_dir-tmp/"})" }); @@ -2790,7 +2946,8 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDirCpp) .WillOnce(Return(startInstanceResponse.SerializeAsString())); testFuncAgentMgrActor_->ResetDeployInstanceResponse(); - testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), + "DeployInstance", std::move(deployInstanceReq->SerializeAsString())); ASSERT_AWAIT_TRUE( [&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid() == TEST_REQUEST_ID; }); @@ -2806,4 +2963,193 @@ TEST_F(AgentServiceActorTest, DeployInstanceWithWorkingDirCpp) destination); // startInstance param posixenvs should contain YR_WORKING_DIR DestroyWorkingDir("/tmp/working_dir-tmp"); } + +TEST_F(AgentServiceActorTest, SendS3Alarm) +{ + metrics::MetricsAdapter::GetInstance().enabledInstruments_.insert(metrics::YRInstrument::YR_S3_ALARM); + + auto request = std::make_shared(); + auto promise = std::make_shared>(); + + auto s3Config = std::make_shared(); + messages::CodePackageThresholds codePackageThresholds; + auto mockTestDeployer = std::make_shared(s3Config, codePackageThresholds); + dstActor_->SetDeployers(function_agent::S3_STORAGE_TYPE, mockTestDeployer); + dstActor_->retryDownloadInterval_ = 50; + EXPECT_CALL(*mockTestDeployer, Deploy).WillRepeatedly(Return(DeployResult{.status = Status(StatusCode::FUNC_AGENT_OBS_ERROR_NEED_RETRY)})); + dstActor_->DownloadCode(request, mockTestDeployer, promise, 1); + ASSERT_AWAIT_TRUE([&]() -> bool { + promise->GetFuture().Get(); + auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + return alarmMap.find(metrics::S3_ALARM) != alarmMap.end(); + }); +} + + +TEST_F(AgentServiceActorTest, ConfigCodeAgingTimeTest) +{ + dstActor_->codePackageThresholds_.set_codeagingtime(10); + dstActor_->codeReferInfos_->clear(); + dstActor_->AddCodeRefer("/tmp/test1", "testIns001", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); + dstActor_->AddCodeRefer("/tmp/test2", "testIns002", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); + dstActor_->AddCodeRefer("/tmp/test2", "testIns003", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); + dstActor_->AddCodeRefer("/tmp/test3", "testIns004", dstActor_->deployers_[function_agent::S3_STORAGE_TYPE]); + dstActor_->DeleteFunction("/tmp/test2", "testIns003"); + dstActor_->DeleteFunction("/tmp/test3", "testIns004"); + (*dstActor_->codeReferInfos_)["/tmp/test1"].lastAccessTimestamp = 1700000; + (*dstActor_->codeReferInfos_)["/tmp/test3"].lastAccessTimestamp = 1700000; + dstActor_->RemoveCodePackageAsync(); + EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test1") != dstActor_->codeReferInfos_->end()); + EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test2") != dstActor_->codeReferInfos_->end()); + EXPECT_TRUE(dstActor_->codeReferInfos_->find("/tmp/test3") == dstActor_->codeReferInfos_->end()); +} + +const std::string STATIC_FUNCTION_META_JSON = R"delim({"funcMetaData":{"layers":[],"name":"0@functest@functest","description":"empty function","functionUrn":"sn:cn:yrk:12345678901234561234567890123456:function:0@functest@functest","reversedConcurrency":0,"tenantId":"12345678901234561234567890123456","tags":null,"functionUpdateTime":"","functionVersionUrn":"sn:cn:yrk:12345678901234561234567890123456:function:0@functest@functest:latest","revisionId":"20250410134701376","codeSize":2911,"codeSha512":"9508f59eb7231bb7577ecc123c5c27317e6563a434dd83d33501f55dfe76b80b58228da0d8f6e37f6cf6d54ccd6dba0eb506bdc10a6acc2ee43e7aa83028b8e3","handler":"handler.my_handler","runtime":"python3.9","timeout":900,"version":"latest","versionDescription":"latest","deadLetterConfig":"","businessId":"yrk","functionType":"","func_id":"","func_name":"functest","domain_id":"","project_name":"","service":"functest","dependencies":"","enable_cloud_debug":"","isStatefulFunction":false,"isBridgeFunction":false,"isStreamEnable":false,"type":"","created":"2025-04-09 08:58:48.655 UTC","enable_auth_in_header":false,"dns_domain_cfg":null,"vpcTriggerImage":""},"codeMetaData":{"codeUploadType":"s3","sha512":"9508f59eb7231bb7577ecc123c5c27317e6563a434dd83d33501f55dfe76b80b58228da0d8f6e37f6cf6d54ccd6dba0eb506bdc10a6acc2ee43e7aa83028b8e3","storage_type":"s3","code_path":"","appId":"61022","bucketId":"bucket-test-log1","objectId":"0@functest@functest-1744292821408","bucketUrl":"http://bucket-test-log1.hwcloudtest.cn:18085","code_type":"","code_url":"","code_filename":"","func_code":{"file":"","link":""}},"envMetaData":{"envKey":"abca007b01f5b1f1fc2e9b3a:86c55b48dc81cac2813f080bd214ea83bf8d79430892c54fdbcd353b29daf165d342943a45411b4b83b01cce27b3a0a51a22d09b6fee1fd729872e5e46893640d5bd870e1051eeb7c90e7e31df5af1ee","environment":"757be5c1d3725bedf25692ea:e4a8c7eeb322fcc5397a498ca9bd83f45597","encrypted_user_data":"","cryptoAlgorithm":"GCM"},"resourceMetaData":{"cpu":600,"memory":512,"gpu_memory":0,"enable_dynamic_memory":false,"customResources":"","enable_tmp_expansion":false,"ephemeral_storage":0},"instanceMetaData":{"maxInstance":100,"minInstance":0,"concurrentNum":100,"instanceType":"","idleMode":false,"poolLabel":"","poolId":"","scalePolicy":"staticFunction"},"extendedMetaData":{"image_name":"","role":{"xrole":"","app_xrole":""},"func_vpc":null,"endpoint_tenant_vpc":null,"mount_config":null,"strategy_config":{"concurrency":0},"extend_config":"","initializer":{"initializer_handler":"handler.init","initializer_timeout":30},"pre_stop":{"pre_stop_handler":"","pre_stop_timeout":0},"heartbeat":{"heartbeat_handler":""},"enterprise_project_id":"","log_tank_service":{"logGroupId":"","logStreamId":""},"tracing_config":{"tracing_ak":"","tracing_sk":"","project_name":""},"custom_container_config":{"control_path":"","image":"","command":null,"args":null,"working_dir":"","uid":0,"gid":0},"async_config_loaded":false,"restore_hook":{},"network_controller":{"disable_public_network":false,"trigger_access_vpcs":null},"user_agency":{"accessKey":"","secretKey":"","token":""}}})delim"; + +TEST_F(AgentServiceActorTest, CreateStaticFunctionInstanceTest) +{ + const std::string user_dir = "/home/static-function-test"; + + auto res = dstActor_->CreateStaticFunctionInstance(); + EXPECT_EQ(res.Get().StatusCode(), -1); // env not exist + + litebus::os::SetEnv("SF_CONFIG_PATH", user_dir + "/functionMeta.json"); + + litebus::os::SetEnv("SF_SCHEDULE_TIMEOUT_MS", "300"); + litebus::os::SetEnv("SF_INSTANCE_TYPE_NOTE", "reserved"); + litebus::os::SetEnv("SF_DELEGATE_DIRECTORY_INFO", "/tmp"); + litebus::os::SetEnv("SF_INVOKE_LABELS", ""); + litebus::os::SetEnv("SF_FUNCTION_SIGNATURE", "20250822095005"); + + litebus::os::SetEnv("POD_DEPLOYMENT_NAME", "pod-deployment-name"); + litebus::os::SetEnv("POD_NAMESPACE", "pod-namespace"); + litebus::os::SetEnv("POD_NAME", "pod-name"); + + (void)litebus::os::Rmdir(user_dir); + + res = dstActor_->CreateStaticFunctionInstance(); + EXPECT_EQ(res.Get().StatusCode(), -1); // function meta file not exist + + (void)litebus::os::Mkdir(user_dir); // why /home/test/config/ ? + + Write(user_dir + "/functionMeta.json", ""); + res = dstActor_->CreateStaticFunctionInstance(); + EXPECT_EQ(res.Get().StatusCode(), -1); // function meta file is empty + + + Write(user_dir + "/functionMeta.json", STATIC_FUNCTION_META_JSON); + bool isFinished = false; + EXPECT_CALL(*testFuncAgentMgrActor_, MockStaticFunctionScheduleResponse) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(""))); + dstActor_->retryScheduleInterval_ = 50; + + res = dstActor_->CreateStaticFunctionInstance(); + EXPECT_EQ(res.IsOK(), true); + + ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetReceivedScheduleRequest(); }); + ASSERT_AWAIT_TRUE([&]() -> bool { return isFinished; }); + auto req = testFuncAgentMgrActor_->GetScheduleRequest(); + ASSERT_NE(req, nullptr); + EXPECT_NE(req->instance().scheduleoption().resourceselector().find(RESOURCE_OWNER_KEY), + req->instance().scheduleoption().resourceselector().end()); + EXPECT_EQ(req->instance().scheduleoption().resourceselector().find(RESOURCE_OWNER_KEY)->second, TEST_AGENT_ID); + + testFuncAgentMgrActor_->ResetReceivedScheduleRequest(); + ::messages::ScheduleResponse response; + response.set_code(static_cast(StatusCode::SUCCESS)); + response.set_requestid(testFuncAgentMgrActor_->GetScheduleRequest()->requestid()); + response.set_instanceid("instance1"); + EXPECT_CALL(*testFuncAgentMgrActor_, MockStaticFunctionScheduleResponse) + .WillRepeatedly(Return(response.SerializeAsString())); + ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetReceivedScheduleRequest(); }); + ASSERT_AWAIT_TRUE([&]() -> bool { return dstActor_->scheduleResponsePromise_->GetFuture().IsOK(); }); + + litebus::os::Rmdir(user_dir); +} + +TEST_F(AgentServiceActorTest, NotifyFunctionStatusTest) +{ + auto staticFunctionChangeRequest = std::make_shared(); + std::string instanceID = "Test-InstanceID"; + staticFunctionChangeRequest->set_instanceid(instanceID); + staticFunctionChangeRequest->set_status(static_cast(InstanceState::RUNNING)); + + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "NotifyFunctionStatusChange", + staticFunctionChangeRequest->SerializeAsString()); + + ASSERT_AWAIT_TRUE([&]() -> bool { + return dstActor_->instanceHealthyMap_.find(instanceID) != dstActor_->instanceHealthyMap_.end(); + }); + EXPECT_TRUE(dstActor_->instanceHealthyMap_[instanceID] == static_cast(InstanceState::RUNNING)); + + staticFunctionChangeRequest->set_status(static_cast(InstanceState::EXITING)); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "NotifyFunctionStatusChange", + staticFunctionChangeRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { + return dstActor_->instanceHealthyMap_.find(instanceID) == dstActor_->instanceHealthyMap_.end(); + }); + + staticFunctionChangeRequest->set_status(static_cast(InstanceState::FATAL)); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "NotifyFunctionStatusChange", + staticFunctionChangeRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { + return dstActor_->instanceHealthyMap_.find(instanceID) == dstActor_->instanceHealthyMap_.end(); + }); +} + +TEST_F(AgentServiceActorTest, CheckReadinessTest) +{ + EXPECT_TRUE(dstActor_->IsAgentReadiness().IsError()); + + dstActor_->instanceHealthyMap_["xx"] = static_cast(InstanceState::CREATING); + EXPECT_TRUE(dstActor_->IsAgentReadiness().IsError()); + + dstActor_->instanceHealthyMap_["xx"] = static_cast(InstanceState::RUNNING); + const auto future = dstActor_->IsAgentReadiness(); // SetValue + EXPECT_TRUE(future.IsOK() && future.Get().IsOk()); +} + +TEST_F(AgentServiceActorTest, DeployInstanceWithSharedDir) +{ + std::string sharedDir = "test_abc"; + std::string dest = "/dcache/shared/" + sharedDir; + auto deployInstanceReq = std::make_unique(); + deployInstanceReq->set_requestid(TEST_REQUEST_ID); + deployInstanceReq->set_instanceid(TEST_INSTANCE_ID); + auto spec = deployInstanceReq->mutable_funcdeployspec(); + spec->set_storagetype(function_agent::LOCAL_STORAGE_TYPE); + deployInstanceReq->mutable_createoptions()->insert({ "DELEGATE_SHARED_DIRECTORY", sharedDir }); + deployInstanceReq->mutable_createoptions()->insert({ "DELEGATE_SHARED_DIRECTORY_TTL", "9223372036854775808" }); + litebus::os::Rmdir(dest); + messages::StartInstanceResponse startInstanceResponse; + startInstanceResponse.set_requestid(TEST_REQUEST_ID); + EXPECT_CALL(*testRuntimeManager_, MockStartInstanceResponse) + .WillOnce(Return(startInstanceResponse.SerializeAsString())); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", + std::move(deployInstanceReq->SerializeAsString())); + ASSERT_AWAIT_TRUE_FOR([&]() -> bool { return litebus::os::ExistPath(dest); }, 5000); + EXPECT_EQ(dstActor_->deployers_[SHARED_DIR_STORAGE_TYPE]->GetTTL(dest), std::numeric_limits::max()); + EXPECT_EQ(JudgeCodeReferNum(dstActor_->GetCodeReferManager(), dest), static_cast(1)); + EXPECT_EQ(testFuncAgentMgrActor_->GetDeployInstanceResponse()->requestid(), TEST_REQUEST_ID); + testFuncAgentMgrActor_->ResetDeployInstanceResponse(); + EXPECT_TRUE(litebus::os::Rmdir(dest).IsNone()); +} + +TEST_F(AgentServiceActorTest, DeployInstanceWithSharedDirAndInvalidTTL) +{ + std::string sharedDir = "test_abc"; + std::string dest = "/dcache/shared/" + sharedDir; + auto deployInstanceReq = std::make_unique(); + deployInstanceReq->set_requestid(TEST_REQUEST_ID); + deployInstanceReq->set_instanceid(TEST_INSTANCE_ID); + auto spec = deployInstanceReq->mutable_funcdeployspec(); + spec->set_storagetype(function_agent::LOCAL_STORAGE_TYPE); + deployInstanceReq->mutable_createoptions()->insert({ "DELEGATE_SHARED_DIRECTORY", sharedDir }); + deployInstanceReq->mutable_createoptions()->insert({ "DELEGATE_SHARED_DIRECTORY_TTL", "abc" }); + messages::StartInstanceResponse startInstanceResponse; + startInstanceResponse.set_requestid(TEST_REQUEST_ID); + testFuncAgentMgrActor_->SendRequestToAgentServiceActor(dstActor_->GetAID(), "DeployInstance", + std::move(deployInstanceReq->SerializeAsString())); + ASSERT_AWAIT_TRUE([&]() -> bool { return testFuncAgentMgrActor_->GetDeployInstanceResponse()->code() == 70052; }); +} } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.cpp b/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.cpp index ca2eadea71ba390aee3d9d9132dd1e9bed2a3c2b..f07e63dad07176e1866c55f3916dd49b2382d126 100644 --- a/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.cpp +++ b/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.cpp @@ -16,7 +16,7 @@ #include "agent_service_test_actor.h" -#include "logs/logging.h" +#include "common/logs/logging.h" namespace functionsystem::function_agent::test { void MockActor::SendRequestToAgentServiceActor(const litebus::AID &to, std::string &&name, std::string &&msg) @@ -153,6 +153,8 @@ void MockFunctionAgentMgrActor::Init() ActorBase::Receive("UpdateCredResponse", &MockFunctionAgentMgrActor::UpdateCredResponse); ActorBase::Receive("SetNetworkIsolationResponse", &MockFunctionAgentMgrActor::SetNetworkIsolationResponse); ActorBase::Receive("QueryDebugInstanceInfosResponse", &MockFunctionAgentMgrActor::QueryDebugInstanceInfosResponse); + ActorBase::Receive("StaticFunctionScheduleRequest", &MockFunctionAgentMgrActor::StaticFunctionScheduleRequest); + ActorBase::Receive("NotifyFunctionStatusChangeResp", &MockFunctionAgentMgrActor::NotifyFunctionStatusChangeResp); } void MockFunctionAgentMgrActor::DeployInstanceResponse(const litebus::AID &from, std::string &&, std::string &&msg) @@ -248,4 +250,24 @@ void MockFunctionAgentMgrActor::SetNetworkIsolationResponse(const litebus::AID & (void)setNetworkIsolationResponse_->ParseFromString(msg); YRLOG_DEBUG("received SetNetworkIsolationResponse(requestid:{}) from {}", setNetworkIsolationResponse_->requestid(), std::string(from)); } + +void MockFunctionAgentMgrActor::StaticFunctionScheduleRequest(const litebus::AID &from, std::string &&, std::string &&msg) +{ + YRLOG_INFO("set receivedScheduleRequest to true"); + receivedScheduleRequest_ = true; + scheduleRequest_->ParseFromString(msg); + YRLOG_DEBUG("{}|received static function schedule message from {}", scheduleRequest_->requestid(), std::string(from)); + Send(from, "StaticFunctionScheduleResponse", MockStaticFunctionScheduleResponse()); +} + +void MockFunctionAgentMgrActor::NotifyFunctionStatusChangeResp(const litebus::AID &from, std::string &&, std::string &&msg) +{ + messages::StaticFunctionChangeResponse resp; + if (msg.empty() || !resp.ParseFromString(msg)) { + YRLOG_WARN("invalid request body, failed to get response of notify instance healthy change from {}.", + from.HashString()); + return; + } + YRLOG_INFO("{}|notify instance status change instance({}) successfully ", resp.requestid(), resp.instanceid()); +} } // namespace functionsystem::function_agent::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.h b/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.h index aaf92c10f29b7dc5c8fa43a4abe941a718ed56e0..57a6629ba0b6f1728d7ea77055c1eae92228f431 100644 --- a/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.h +++ b/functionsystem/tests/unit/function_agent/agent_service/agent_service_test_actor.h @@ -22,8 +22,8 @@ #include #include #include -#include "logs/logging.h" -#include "proto/pb/message_pb.h" +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::function_agent::test { @@ -52,6 +52,7 @@ public: MOCK_METHOD(std::string, MockUpdateResourceResponse, ()); MOCK_METHOD(std::string, MockUpdateAgentStatusResponse, ()); MOCK_METHOD(std::string, MockRegisteredResponse, ()); + MOCK_METHOD(std::string, MockStaticFunctionScheduleResponse, ()); // Simulates the Function Agent Manager Actor to receive DeployInstanceResponse messages. void DeployInstanceResponse(const litebus::AID &from, std::string &&name, std::string &&msg); @@ -72,6 +73,8 @@ public: void SetNetworkIsolationResponse(const litebus::AID &, std::string &&, std::string &&msg); // Simulates the Function Agent Manager Actor to receive QueryDebugInstanceInfosResponse messages. void QueryDebugInstanceInfosResponse(const litebus::AID &from, std::string &&name, std::string &&msg); + void StaticFunctionScheduleRequest(const litebus::AID &from, std::string &&name, std::string &&msg); + void NotifyFunctionStatusChangeResp(const litebus::AID &from, std::string &&, std::string &&msg); [[maybe_unused]] [[nodiscard]] bool GetReceivedScheduleRequest() const { diff --git a/functionsystem/tests/unit/function_agent/common/s3_bucket_opt.h b/functionsystem/tests/unit/function_agent/common/s3_bucket_opt.h new file mode 100644 index 0000000000000000000000000000000000000000..81cb55b1a1b05aaafb55d1b647c6acbfa7a373c4 --- /dev/null +++ b/functionsystem/tests/unit/function_agent/common/s3_bucket_opt.h @@ -0,0 +1,93 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef FUNCTION_AGENT_COMMON_S3_BUCKET_OPT_H +#define FUNCTION_AGENT_COMMON_S3_BUCKET_OPT_H + +#include + +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" + +namespace functionsystem::test::function_agent { +class S3BucketOpt { +public: + static void InitObsOptions(obs_options *options, const S3Config &s3Config) + { + init_obs_options(options); + + options->bucket_options.host_name = const_cast(s3Config.endpoint.c_str()); + options->bucket_options.access_key = const_cast(s3Config.accessKey.c_str()); + options->bucket_options.secret_access_key = const_cast(s3Config.secretKey.GetData()); + + options->bucket_options.protocol = obs_protocol::OBS_PROTOCOL_HTTP; + options->bucket_options.uri_style = obs_uri_style::OBS_URI_STYLE_PATH; + } + + static void OnHandleComplete(obs_status status, const obs_error_details *error, void *data) + { + auto *retStatus = (obs_status *)data; + *retStatus = status; + } + + static Status CreateBucket(const std::string &bucketName, const S3Config &s3Config) + { + obs_options options; + InitObsOptions(&options, s3Config); + options.bucket_options.bucket_name = const_cast(bucketName.c_str()); + + obs_response_handler handler = { nullptr, &OnHandleComplete }; + + obs_status retStatus = OBS_STATUS_BUTT; + obs_canned_acl cannedAcl = OBS_CANNED_ACL_BUCKET_OWNER_FULL_CONTROL; + create_bucket(&options, cannedAcl, nullptr, &handler, &retStatus); + if (retStatus == OBS_STATUS_OK || retStatus == OBS_STATUS_BucketAlreadyExists || + retStatus == OBS_STATUS_BucketAlreadyOwnedByYou) { + YRLOG_DEBUG("success to create bucket({}).", bucketName); + return Status::OK(); + } else { + YRLOG_DEBUG("failed({}) to create bucket({}).", obs_get_status_name(retStatus), bucketName); + return Status(StatusCode::FUNC_AGENT_OBS_ADD_BUCKET_ERROR, + "failed({" + std::string(obs_get_status_name(retStatus)) + "}) to create bucket({" + + bucketName + "})."); + } + } + + static Status DeleteBucket(const std::string &bucketName, const S3Config &s3Config) + { + obs_options options; + InitObsOptions(&options, s3Config); + options.bucket_options.bucket_name = const_cast(bucketName.c_str()); + + obs_response_handler handler = { nullptr, &OnHandleComplete }; + + obs_status retStatus = OBS_STATUS_BUTT; + delete_bucket(&options, &handler, &retStatus); + + if (retStatus == OBS_STATUS_OK) { + YRLOG_DEBUG("success to delete bucket({}).", bucketName); + return Status::OK(); + } else { + YRLOG_DEBUG("failed({}) to delete bucket({}).", obs_get_status_name(retStatus), bucketName); + return Status( + StatusCode::FUNC_AGENT_OBS_DEL_BUCKET_ERROR, + "failed(" + std::string(obs_get_status_name(retStatus)) + ") to delete bucket(" + bucketName + ")."); + } + } +}; +} // namespace functionsystem::test::function_agent + +#endif // FUNCTION_AGENT_COMMON_S3_BUCKET_OPT_H diff --git a/functionsystem/tests/unit/function_agent/common/s3_deploy_test.cpp b/functionsystem/tests/unit/function_agent/common/s3_deploy_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4d09ed2e8472ce3abf29b3e8cedde3902264955c --- /dev/null +++ b/functionsystem/tests/unit/function_agent/common/s3_deploy_test.cpp @@ -0,0 +1,696 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include + +#include + +#include "common/logs/logging.h" +#include "common/metrics/metrics_adapter.h" +#include "common/proto/pb/message_pb.h" +#include "common/utils/exec_utils.h" +#include "common/utils/files.h" +#include "function_agent/code_deployer/s3_deployer.h" +#include "mocks/mock_obs_wrapper.h" +#include "s3_bucket_opt.h" +#include "s3_object_del.h" +#include "s3_object_put.h" +#include "utils/os_utils.hpp" + +namespace functionsystem::test::function_agent { +using namespace functionsystem::function_agent; + +const std::string AK = "root"; // NOLINT +const std::string SK = "MFDe8ZR7zqSk"; // NOLINT + +const std::string BUCKET_ID = "yr-cxx-refactor-bucket-test"; // NOLINT + +const std::string OBJECT_ID = "yr-cxx-refactor-object-test"; // NOLINT +const std::string OBJECT_BUFFER = "YuanRong C++ refactor test"; // NOLINT + +class S3DeployerTest : public ::testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + codePackageThresholds_.set_filecountsmax(3); + codePackageThresholds_.set_zipfilesizemaxmb(1); + codePackageThresholds_.set_unzipfilesizemaxmb(1); + codePackageThresholds_.set_dirdepthmax(3); + + s3Config_ = std::make_shared(); + s3Config_->endpoint = "10.243.31.23:30110"; + s3Config_->protocol = "http"; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; // pwd + + obs_initialize(OBS_INIT_ALL); + + if (S3BucketOpt::CreateBucket(BUCKET_ID, *s3Config_).IsOk() && + S3ObjectPut::PutObjectFromBuffer(OBJECT_BUFFER, OBJECT_ID, BUCKET_ID, *s3Config_).IsOk()) { + YRLOG_DEBUG("s3 service is available."); + } else { + YRLOG_ERROR("s3 service is not available."); + s3ServiceAvailable = false; + } + + obs_deinitialize(); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + if (s3ServiceAvailable) { + obs_initialize(OBS_INIT_ALL); + + S3ObjectDel::DeleteObject(BUCKET_ID, OBJECT_ID, *s3Config_); + (void)S3BucketOpt::DeleteBucket(BUCKET_ID, *s3Config_); + + obs_deinitialize(); + } + + s3Config_ = nullptr; + } + +protected: + void SetUp() override + { + } + + void TearDown() override + { + } + +protected: + inline static bool s3ServiceAvailable = true; + + inline static std::shared_ptr s3Config_; + + inline static messages::CodePackageThresholds codePackageThresholds_; +}; + +TEST_F(S3DeployerTest, DirectoryTest) // NOLINT +{ + std::string layerDir = litebus::os::Join("/home/s3", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, "bucket"); + + EXPECT_TRUE(litebus::os::Mkdir(bucketDir).IsNone()); + EXPECT_TRUE(litebus::os::Mkdir(bucketDir).IsNone()); + EXPECT_TRUE(litebus::os::ExistPath(bucketDir)); + + std::string objectFile = litebus::os::Join(bucketDir, "xxx.zip"); + FILE *file = fopen(objectFile.c_str(), "wb"); + EXPECT_TRUE(file != nullptr); + EXPECT_TRUE(litebus::os::ExistPath(objectFile)); + EXPECT_TRUE(fclose(file) == 0); + + const std::string bucketDirTmp = bucketDir + "_tmp"; + EXPECT_EQ(rename(bucketDir.c_str(), bucketDirTmp.c_str()), 0); + EXPECT_TRUE(litebus::os::ExistPath(bucketDirTmp)); + + EXPECT_TRUE(litebus::os::Rmdir(bucketDirTmp).IsNone()); + EXPECT_FALSE(litebus::os::ExistPath(bucketDirTmp)); + EXPECT_FALSE(litebus::os::ExistPath(bucketDir)); + + EXPECT_TRUE(litebus::os::ExistPath(funcDir)); + EXPECT_TRUE(litebus::os::Rmdir(funcDir).IsNone()); + + EXPECT_TRUE(litebus::os::ExistPath(layerDir)); + EXPECT_TRUE(litebus::os::Rmdir(layerDir).IsNone()); +} + +TEST_F(S3DeployerTest, IsDeployedTest) // NOLINT +{ + S3Deployer deployer(s3Config_, codePackageThresholds_); + std::string destination = "/home/s3/layer/func/bucketid/objectid"; + litebus::os::Mkdir(destination); + EXPECT_TRUE(deployer.IsDeployed(destination, false)); + litebus::os::Rmdir(destination); + EXPECT_FALSE(deployer.IsDeployed(destination, false)); + litebus::os::Mkdir(destination); + EXPECT_FALSE(deployer.IsDeployed(destination, true)); + EXPECT_TRUE(deployer.IsDeployed("/home/s3/layer/func/bucketid", true)); + litebus::os::Rmdir(destination); +} + +TEST_F(S3DeployerTest, GetDestinationTest) // NOLINT +{ + S3Deployer deployer(s3Config_, codePackageThresholds_); + { + std::string destination = "/home/s3/layer/func/bucketid/objectid"; + EXPECT_EQ(deployer.GetDestination("/home/s3", "bucketid", "objectid"), destination); + } + { + std::string destination = "/home/s3/layer/func/bucketid/a-b-c-objectid"; + EXPECT_EQ(deployer.GetDestination("/home/s3", "bucketid", "a/b/c/objectid"), destination); + } + +} + +class UnzipMockS3Deployer : public S3Deployer { +public: + explicit UnzipMockS3Deployer(std::shared_ptr config, messages::CodePackageThresholds msg) + : S3Deployer(std::move(config), std::move(msg)) + { + } + + explicit UnzipMockS3Deployer(std::shared_ptr config, const std::shared_ptr &obsWrapper, + messages::CodePackageThresholds msg) + : S3Deployer(std::move(config), std::move(obsWrapper), std::move(msg)) + { + } + +public: + MOCK_METHOD(Status, UnzipFile, (const std::string &, const std::string &), (override)); + MOCK_METHOD2(CollectFunctionInfo, void(const std::string &, const std::shared_ptr<::messages::DeployRequest> &)); +}; + +/** + * Feature: S3DeployerTest--S3DeployDownloadFailed + * Description: Download user code with error param + * Steps: + * 1. Create DeploymentConfig, set hostname + * 2. Download with just hostname + * 3. Download with just hostname and security token + * 4. Download without secret access key + * Expectation: + * 1. First download, failed when obs init options + * 2. Second download, failed when obs init options + * 4. Third download, failed when obs init options + */ +TEST_F(S3DeployerTest, S3DeployDownloadFailed) // NOLINT +{ + messages::DeploymentConfig deploymentConfig; + deploymentConfig.set_deploydir("/home/s3/"); + deploymentConfig.set_bucketid(BUCKET_ID); + deploymentConfig.set_objectid(OBJECT_ID); + deploymentConfig.set_storagetype("s3"); + + deploymentConfig.set_hostname("10.175.127.111:9000"); + + UnzipMockS3Deployer deployer(s3Config_, codePackageThresholds_); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig.bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, deploymentConfig.objectid()); + + EXPECT_EQ(deployer.DownloadCode(objectDir, deploymentConfig), StatusCode::FUNC_AGENT_OBS_INIT_OPTIONS_ERROR); + deploymentConfig.set_securitytoken("xxx"); + EXPECT_EQ(deployer.DownloadCode(objectDir, deploymentConfig), StatusCode::FUNC_AGENT_OBS_INIT_OPTIONS_ERROR); + deploymentConfig.set_temporaryaccesskey("xxx"); + EXPECT_EQ(deployer.DownloadCode(objectDir, deploymentConfig), StatusCode::FUNC_AGENT_OBS_INIT_OPTIONS_ERROR); + deploymentConfig.set_temporarysecretkey("xxx"); + EXPECT_NE(deployer.DownloadCode(objectDir, deploymentConfig), StatusCode::FUNC_AGENT_OBS_INIT_OPTIONS_ERROR); +} + +class DownloadMockS3Deployer : public S3Deployer { +public: + explicit DownloadMockS3Deployer(std::shared_ptr config, messages::CodePackageThresholds msg) + : S3Deployer(std::move(config), std::move(msg)) + { + } + + explicit DownloadMockS3Deployer(std::shared_ptr config, const std::shared_ptr &obsWrapper, + messages::CodePackageThresholds msg) + : S3Deployer(std::move(config), std::move(obsWrapper), std::move(msg)) + { + } + +public: + MOCK_METHOD(Status, DownloadCode, (const std::string &, const ::messages::DeploymentConfig &), (override)); + MOCK_METHOD(Status, PackageValidation, (const std::string &, const std::string &, const std::string &), (override)); + MOCK_METHOD(Status, UnzipFile, (const std::string &, const std::string &), (override)); +}; + + +TEST_F(S3DeployerTest, S3DeployProcess) // NOLINT +{ + DownloadMockS3Deployer deployer(s3Config_, codePackageThresholds_); + EXPECT_CALL(deployer, DownloadCode) + .WillRepeatedly(testing::Invoke([](const std::string &destFile, const messages::DeploymentConfig &config) { + litebus::os::Mkdir(destFile); + return Status::OK(); + })); + EXPECT_CALL(deployer, PackageValidation) + .WillRepeatedly(testing::Return(Status::OK())); + EXPECT_CALL(deployer, UnzipFile) + .WillOnce(testing::Return(Status::OK())) + .WillOnce(testing::Return(Status(StatusCode::FUNC_AGENT_OBS_OPEN_FILE_ERROR, "failed to unzip file"))); + + auto request = std::make_shared(); + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(BUCKET_ID); + deploymentConfig->set_objectid(OBJECT_ID); + deploymentConfig->set_storagetype("s3"); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig->bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, deploymentConfig->objectid()); + std::string objectFile = litebus::os::Join(objectDir, deploymentConfig->objectid()); + std::string objectInnerDir = litebus::os::Join(objectDir, "tmp"); + + // EXPECT true with already created tmp directory + EXPECT_TRUE(deployer.Deploy(request).status.IsOk()); + + EXPECT_TRUE(deployer.Clear(objectDir, deploymentConfig->objectid())); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + EXPECT_TRUE(litebus::os::Rmdir(layerDir).IsNone()); + EXPECT_FALSE(litebus::os::ExistPath(layerDir)); + + // EXPECT true with UnzipFile return error status + EXPECT_TRUE(deployer.Deploy(request).status.IsError()); + EXPECT_TRUE(deployer.Clear(objectDir, deploymentConfig->objectid())); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + EXPECT_TRUE(litebus::os::Rmdir(layerDir).IsNone()); + EXPECT_FALSE(litebus::os::ExistPath(layerDir)); +} + +TEST_F(S3DeployerTest, S3DeployProcessWithMultiDir) +{ + DownloadMockS3Deployer deployer(s3Config_, codePackageThresholds_); + std::string expected; + EXPECT_CALL(deployer, DownloadCode) + .WillOnce(testing::Invoke([&expected](const std::string &destFile, const messages::DeploymentConfig &config) { + litebus::os::Mkdir(destFile); + expected = destFile; + return Status::OK(); + })); + EXPECT_CALL(deployer, PackageValidation) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(deployer, UnzipFile) + .WillOnce(testing::Return(Status::OK())); + + auto request = std::make_shared(); + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(BUCKET_ID); + deploymentConfig->set_objectid("a/b/c/" + OBJECT_ID); + deploymentConfig->set_storagetype("s3"); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig->bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, "a-b-c-" + OBJECT_ID); + std::string objectFile = litebus::os::Join(objectDir, OBJECT_ID); + std::string objectInnerDir = objectDir + "-tmp"; + std::string objectFileTmp = litebus::os::Join(objectInnerDir, OBJECT_ID); + + // EXPECT true with already created tmp directory + EXPECT_TRUE(deployer.Deploy(request).status.IsOk()); + EXPECT_TRUE(deployer.Clear(objectDir, deploymentConfig->objectid())); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + EXPECT_TRUE(litebus::os::Rmdir(layerDir).IsNone()); + EXPECT_FALSE(litebus::os::ExistPath(layerDir)); + EXPECT_STREQ(expected.c_str(), objectFileTmp.c_str()); +} + +TEST_F(S3DeployerTest, S3DeployWithEmptyDownload) +{ + auto request = std::make_shared(); + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(""); + deploymentConfig->set_objectid(""); + deploymentConfig->set_storagetype(""); + auto deployer = std::make_shared(s3Config_, codePackageThresholds_); + EXPECT_TRUE(deployer->Deploy(request).status.IsOk()); +} + +TEST_F(S3DeployerTest, S3RetryDownloadCodeSuccess) // NOLINT +{ + auto request = std::make_shared(); + + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(BUCKET_ID); + deploymentConfig->set_objectid(OBJECT_ID); + deploymentConfig->set_storagetype("s3"); + + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillRepeatedly(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()) + .WillOnce(testing::Return(OBS_STATUS_ConnectionFailed)) + .WillOnce(testing::Return(OBS_STATUS_ConnectionFailed)) + .WillOnce(testing::Return(OBS_STATUS_OK)) + .WillOnce(testing::Return(OBS_STATUS_OK)); + + DownloadMockS3Deployer deployer(s3Config_, std::move(mockObsWrapper), codePackageThresholds_); + EXPECT_CALL(deployer, DownloadCode).WillOnce(testing::Return(Status::OK())); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig->bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, deploymentConfig->objectid()); + + const ::messages::DeploymentConfig &config = request->deploymentconfig(); + std::string objectFile = litebus::os::Join(objectDir, deploymentConfig->objectid()); + EXPECT_TRUE(deployer.RetryDownloadCode(objectFile, config).IsOk()); + + EXPECT_TRUE(deployer.Clear(objectDir, deploymentConfig->objectid())); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + EXPECT_FALSE(litebus::os::Rmdir(layerDir).IsNone()); + EXPECT_FALSE(litebus::os::ExistPath(layerDir)); +} + +TEST_F(S3DeployerTest, S3RetryDownloadCodeFailedWhenConnect) // NOLINT +{ + auto request = std::make_shared(); + + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(BUCKET_ID); + deploymentConfig->set_objectid(OBJECT_ID); + deploymentConfig->set_storagetype("s3"); + + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillRepeatedly(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()) + .WillOnce(testing::Return(OBS_STATUS_OK)) + .WillOnce(testing::Return(OBS_STATUS_ConnectionFailed)); + + DownloadMockS3Deployer deployer(s3Config_, std::move(mockObsWrapper), codePackageThresholds_); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig->bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, deploymentConfig->objectid()); + + const ::messages::DeploymentConfig &config = request->deploymentconfig(); + std::string objectFile = litebus::os::Join(objectDir, deploymentConfig->objectid()); + auto status = deployer.RetryDownloadCode(objectFile, config); + EXPECT_EQ(status.StatusCode(), StatusCode::FUNC_AGENT_OBS_CONNECTION_ERROR); +} + +TEST_F(S3DeployerTest, S3RetryDownloadCodeFailedWhenDownload) // NOLINT +{ + auto request = std::make_shared(); + + auto *deploymentConfig = request->mutable_deploymentconfig(); + deploymentConfig->set_deploydir("/home/s3/"); + deploymentConfig->set_bucketid(BUCKET_ID); + deploymentConfig->set_objectid(OBJECT_ID); + deploymentConfig->set_storagetype("s3"); + + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillRepeatedly(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()).WillRepeatedly(testing::Return(OBS_STATUS_OK)); + + DownloadMockS3Deployer deployer(s3Config_, std::move(mockObsWrapper), codePackageThresholds_); + EXPECT_CALL(deployer, DownloadCode) + .WillOnce(testing::Return(Status(StatusCode::FUNC_AGENT_OBS_OPEN_FILE_ERROR, "failed to open file"))); + + std::string layerDir = litebus::os::Join("/home/s3/", "layer"); + std::string funcDir = litebus::os::Join(layerDir, "func"); + std::string bucketDir = litebus::os::Join(funcDir, deploymentConfig->bucketid()); + std::string objectDir = litebus::os::Join(bucketDir, deploymentConfig->objectid()); + + const ::messages::DeploymentConfig &config = request->deploymentconfig(); + std::string objectFile = litebus::os::Join(objectDir, deploymentConfig->objectid()); + auto status = deployer.RetryDownloadCode(objectFile, config); + EXPECT_EQ(status.StatusCode(), StatusCode::FUNC_AGENT_OBS_OPEN_FILE_ERROR); +} + +TEST_F(S3DeployerTest, S3InitFailedTest) // NOLINT +{ + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillRepeatedly(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()).Times(3).WillRepeatedly(testing::Return(OBS_STATUS_ConnectionFailed)); + + std::shared_ptr s3Config = std::make_shared(); + s3Config_->endpoint = "10.247.5.237:19002"; + s3Config_->protocol = "http"; + s3Config_->credentialType = CREDENTIAL_TYPE_PERMANENT_CREDENTIALS; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; + auto s3Deployer = std::make_shared(s3Config, std::move(mockObsWrapper), + codePackageThresholds_); +} + +TEST_F(S3DeployerTest, S3InitSuccessAfterRetryTest) // NOLINT +{ + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillRepeatedly(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()) + .WillOnce(testing::Return(OBS_STATUS_ConnectionFailed)) + .WillOnce(testing::Return(OBS_STATUS_ConnectionFailed)) + .WillOnce(testing::Return(OBS_STATUS_OK)); + + std::shared_ptr s3Config = std::make_shared(); + s3Config_->endpoint = "10.247.5.237:19002"; + s3Config_->protocol = "http"; + s3Config_->credentialType = CREDENTIAL_TYPE_PERMANENT_CREDENTIALS; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; + auto s3Deployer = std::make_shared(s3Config, std::move(mockObsWrapper), + codePackageThresholds_); +} + +TEST_F(S3DeployerTest, S3InitSuccessTest) // NOLINT +{ + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillOnce(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()).WillOnce(testing::Return(OBS_STATUS_OK)); + + std::shared_ptr s3Config = std::make_shared(); + s3Config_->endpoint = "10.247.5.237:19002"; + s3Config_->protocol = "https"; + s3Config_->credentialType = CREDENTIAL_TYPE_PERMANENT_CREDENTIALS; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; + auto s3Deployer = std::make_shared(s3Config, std::move(mockObsWrapper), + codePackageThresholds_); +} + +TEST_F(S3DeployerTest, S3CredentialRotationInitSuccessTest) +{ + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillOnce(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()).WillOnce(testing::Return(OBS_STATUS_OK)); + + std::shared_ptr s3Config = std::make_shared(); + s3Config_->credentialType = CREDENTIAL_TYPE_ROTATING_CREDENTIALS; + s3Config_->endpoint = "10.247.5.237:19002"; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; + auto s3Deployer = std::make_shared(s3Config, std::move(mockObsWrapper), + codePackageThresholds_); +} + +TEST_F(S3DeployerTest, UnzipFileTest) // NOLINT +{ + std::shared_ptr mockObsWrapper = std::make_shared(); + EXPECT_CALL(*mockObsWrapper, DeinitializeObs()).WillOnce(testing::Return()); + EXPECT_CALL(*mockObsWrapper, InitializeObs()).WillOnce(testing::Return(OBS_STATUS_OK)); + + std::shared_ptr s3Config = std::make_shared(); + s3Config_->endpoint = "10.247.5.237:19002"; + s3Config_->protocol = "https"; + s3Config_->credentialType = CREDENTIAL_TYPE_PERMANENT_CREDENTIALS; + + s3Config_->accessKey = AK; // user + s3Config_->secretKey = SK; + auto s3Deployer = std::make_shared(s3Config, std::move(mockObsWrapper), + codePackageThresholds_); + const std::string DEST_DIR = "object_id-tmp"; + const std::string DEST_FILE = "object_id-tmp"; + auto status = s3Deployer->UnzipFile(DEST_DIR, DEST_FILE); + EXPECT_TRUE(status.IsError()); +} + +// test for interface Clear +TEST_F(S3DeployerTest, ClearTest) +{ + std::shared_ptr s3Config = std::make_shared(); + S3Deployer deployer(s3Config, codePackageThresholds_); + std::string bucketDir = litebus::os::Join("/home/s3/", BUCKET_ID); + std::string objectDir = litebus::os::Join(bucketDir, OBJECT_ID); + std::string objectTmpDir = objectDir + "_tmp"; + std::string objectTmpInnerDir = litebus::os::Join(objectTmpDir, "tmp"); + + litebus::os::Mkdir(objectDir); + EXPECT_TRUE(deployer.Clear(objectDir, OBJECT_ID)); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + + // rename failed + litebus::os::Mkdir(objectDir); + litebus::os::Mkdir(objectTmpInnerDir); + EXPECT_TRUE(deployer.Clear(objectDir, OBJECT_ID)); + EXPECT_FALSE(litebus::os::ExistPath(objectDir)); + EXPECT_TRUE(litebus::os::ExistPath(objectTmpInnerDir)); + + litebus::os::Rmdir(bucketDir); +} + +TEST_F(S3DeployerTest, CheckPackageContentTest) +{ + std::shared_ptr s3Config = std::make_shared(); + S3Deployer deployer(s3Config, codePackageThresholds_); + + std::string objDir = "/home/s3/layer/func/bucket/files"; + litebus::os::Mkdir(objDir); + auto file1 = objDir + "/a.txt"; + EXPECT_TRUE(Write(file1, "a")); + auto file2 = objDir + "/b.txt"; + EXPECT_TRUE(Write(file2, "b")); + auto file3 = objDir + "/c.txt"; + std::string largeStr; + for (int i = 0; i < 1000000; i++) { + largeStr += "cccccccccccccccccccccccccccccc"; + } + EXPECT_TRUE(Write(file3, largeStr)); + auto file4 = objDir + "/d.txt"; + EXPECT_TRUE(Write(file4, "d")); + auto file5 = objDir + "/e.txt"; + EXPECT_TRUE(Write(file5, "e")); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test1.zip " + objDir + "/a.txt").error.empty(), true); + auto status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "", ""); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("the depth of dir exceeds maximum limit")); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test2.zip " + objDir + "/c.txt").error.empty(), true); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test2.zip", "", ""); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("check zip file failed, error: file(home/s3/layer/func/bucket/files/c.txt) is bigger than 1048576")); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test3.zip " + objDir + "/a.txt " + objDir + "/b.txt " + objDir + "/d.txt " + objDir + "/e.txt").error.empty(), true); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test3.zip", "", ""); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("the number of files exceeds maximum limit")); + EXPECT_TRUE(litebus::os::Rmdir("/home/s3/layer/func/bucket").IsNone()); +} + +TEST_F(S3DeployerTest, CheckPackageContentWithMaxThresholds) +{ + messages::CodePackageThresholds codePackageThresholds; + codePackageThresholds.set_filecountsmax(30000); + codePackageThresholds.set_zipfilesizemaxmb(1024 * 1024 * 10); + codePackageThresholds.set_unzipfilesizemaxmb(1024 * 1024 * 10); + codePackageThresholds.set_dirdepthmax(50); + std::shared_ptr s3Config = std::make_shared(); + S3Deployer deployer(s3Config, codePackageThresholds); + + std::string objDir = "/home/s3/layer/func/bucket/files"; + litebus::os::Mkdir(objDir); + auto file1 = objDir + "/a.txt"; + EXPECT_TRUE(Write(file1, "a")); + auto file2 = objDir + "/b.txt"; + EXPECT_TRUE(Write(file2, "b")); + auto file3 = objDir + "/c.txt"; + std::string largeStr; + for (int i = 0; i < 1000000; i++) { + largeStr += "cccccccccccccccccccccccccccccc"; + } + EXPECT_TRUE(Write(file3, largeStr)); + auto file4 = objDir + "/d.txt"; + EXPECT_TRUE(Write(file4, "d")); + auto file5 = objDir + "/e.txt"; + EXPECT_TRUE(Write(file5, "e")); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test1.zip " + objDir + "/a.txt").error.empty(), true); + auto status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "", ""); + EXPECT_TRUE(status.IsOk()); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test2.zip " + objDir + "/c.txt").error.empty(), true); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test2.zip", "", ""); + EXPECT_TRUE(status.IsOk()); + + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test3.zip " + objDir + "/a.txt " + objDir + "/b.txt " + objDir + "/d.txt " + objDir + "/e.txt").error.empty(), true); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test3.zip", "", ""); + EXPECT_TRUE(status.IsOk()); + EXPECT_TRUE(litebus::os::Rmdir("/home/s3/layer/func/bucket").IsNone()); +} + +TEST_F(S3DeployerTest, CheckPackageSignatureTest) +{ + std::shared_ptr s3Config = std::make_shared(); + messages::CodePackageThresholds codePackageThresholds; + codePackageThresholds.set_filecountsmax(30); + codePackageThresholds.set_zipfilesizemaxmb(10); + codePackageThresholds.set_unzipfilesizemaxmb(10); + codePackageThresholds.set_dirdepthmax(10); + S3Deployer deployer(s3Config, codePackageThresholds, true); + std::string objDir = "/home/s3/layer/func/bucket/files"; + litebus::os::Mkdir(objDir); + auto file1 = objDir + "/a.txt"; + std::string largeStr; + for (int i = 0; i < 100; i++) { + largeStr += "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + } + EXPECT_TRUE(Write(file1, largeStr)); + EXPECT_EQ(ExecuteCommand("zip -r /home/s3/layer/func/bucket/test1.zip " + objDir + "/a.txt").error.empty(), true); + auto status =deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "", ""); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("package signature doesn't match")); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "aaaaaaaaaaaa", ""); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("package signature doesn't match")); + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "", "aaaaaaaaaaaa"); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("package signature doesn't match")); + std::stringstream ss(ExecuteCommand("sha256sum /home/s3/layer/func/bucket/test1.zip").output); + std::string cmdResult; + ss >> cmdResult; + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", "", cmdResult); + EXPECT_TRUE(status.IsOk()); + std::stringstream ss1(ExecuteCommand("sha512sum /home/s3/layer/func/bucket/test1.zip").output); + std::string cmdResult1; + ss1 >> cmdResult1; + status = deployer.PackageValidation("/home/s3/layer/func/bucket/test1.zip", cmdResult1, ""); + EXPECT_TRUE(status.IsOk()); + EXPECT_TRUE(litebus::os::Rmdir("/home/s3/layer/func/bucket").IsNone()); +} +} // namespace functionsystem::test::function_agent + + + +namespace functionsystem::function_agent { +class S3DeployerPrivateTest : public ::testing::Test { +protected: + inline static messages::CodePackageThresholds codePackageThresholds_; +}; + +TEST_F(S3DeployerPrivateTest, InitObsOptions) +{ + auto s3Config = std::make_shared(); + s3Config->protocol = "https"; + + S3Deployer deployer(s3Config, codePackageThresholds_); + messages::DeploymentConfig deploymentConfig; + obs_options options; + init_obs_options(&options); + + // domain + s3Config->endpoint = "obs.cn-north-7.ulanqab.huawei.com"; + deployer.InitObsOptions(&options, deploymentConfig, s3Config); + const int obsType = 1; + EXPECT_EQ(options.request_options.auth_switch, obsType); + const int obsUriStyleVirtualhost = 0; + EXPECT_EQ(options.bucket_options.uri_style, obsUriStyleVirtualhost); + + // ip:port + s3Config->endpoint = "10.243.31.23:30110"; + deployer.InitObsOptions(&options, deploymentConfig, s3Config); + const int obsUriStylePath = 1; + EXPECT_EQ(options.bucket_options.uri_style, obsUriStylePath); +} +} // namespace functionsystem::function_agent \ No newline at end of file diff --git a/functionsystem/tests/unit/function_agent/common/s3_object_del.h b/functionsystem/tests/unit/function_agent/common/s3_object_del.h new file mode 100644 index 0000000000000000000000000000000000000000..e94ccfab226e8909020585e2ec61b70bb12282a4 --- /dev/null +++ b/functionsystem/tests/unit/function_agent/common/s3_object_del.h @@ -0,0 +1,72 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef FUNCTIONSYSTEM_FUNCTION_AGENT_COMMON_S3_OBJECT_DEL_H +#define FUNCTIONSYSTEM_FUNCTION_AGENT_COMMON_S3_OBJECT_DEL_H + +#include +#include + +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" + +namespace functionsystem::test::function_agent { +class S3ObjectDel { +public: + static void OnHandleComplete(obs_status status, const obs_error_details *error, void *data) + { + YRLOG_DEBUG("complete({}) to delete object.", obs_get_status_name(status)); + auto *retStatus = (obs_status *)data; + *retStatus = status; + } + + static void DeleteObject(const std::string &bucketID, const std::string &objectID, const S3Config &s3Config) + { + obs_object_info objectInfo; + memset_s(&objectInfo, sizeof(objectInfo), 0, sizeof(obs_object_info)); + objectInfo.key = const_cast(objectID.c_str()); + objectInfo.version_id = nullptr; + + obs_options options; + InitObsOptions(&options, s3Config); + options.bucket_options.bucket_name = const_cast(bucketID.c_str()); + + obs_status retStatus = OBS_STATUS_BUTT; + obs_response_handler handler = { nullptr, &OnHandleComplete }; + delete_object(&options, &objectInfo, &handler, &retStatus); + if (OBS_STATUS_OK == retStatus) { + YRLOG_DEBUG("success to delete object({}).", objectID); + } else { + YRLOG_DEBUG("failed({}) to delete object({}).", obs_get_status_name(retStatus), objectID); + } + } + +private: + static void InitObsOptions(obs_options *options, const S3Config &s3Config) + { + init_obs_options(options); + + options->bucket_options.host_name = const_cast(s3Config.endpoint.c_str()); + options->bucket_options.access_key = const_cast(s3Config.accessKey.c_str()); + options->bucket_options.secret_access_key = const_cast(s3Config.secretKey.GetData()); + + options->bucket_options.protocol = obs_protocol::OBS_PROTOCOL_HTTP; + options->bucket_options.uri_style = obs_uri_style::OBS_URI_STYLE_PATH; + } +}; +} // namespace functionsystem::test::function_agent + +#endif // FUNCTIONSYSTEM_FUNCTION_AGENT_COMMON_S3_OBJECT_DEL_H diff --git a/functionsystem/tests/unit/function_agent/common/s3_object_put.h b/functionsystem/tests/unit/function_agent/common/s3_object_put.h new file mode 100644 index 0000000000000000000000000000000000000000..b1f2c68758925d7ff259ce0dfbbbf4c22b709d69 --- /dev/null +++ b/functionsystem/tests/unit/function_agent/common/s3_object_put.h @@ -0,0 +1,109 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef FUNCTION_AGENT_S3_PUT_OBJECT_H +#define FUNCTION_AGENT_S3_PUT_OBJECT_H + +#include +#include +#include + +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" + +namespace functionsystem::test::function_agent { +struct PutObjectData { + char *buffer; + + uint64_t bufferSize; + + uint64_t offset; + + obs_status status; +}; + +class S3ObjectPut { +public: + static void InitObsOptions(obs_options *options, const S3Config &s3Config) + { + init_obs_options(options); + + options->bucket_options.host_name = const_cast(s3Config.endpoint.c_str()); + options->bucket_options.access_key = const_cast(s3Config.accessKey.c_str()); + options->bucket_options.secret_access_key = const_cast(s3Config.secretKey.GetData()); + + options->bucket_options.protocol = obs_protocol::OBS_PROTOCOL_HTTP; + options->bucket_options.uri_style = obs_uri_style::OBS_URI_STYLE_PATH; + } + + static obs_status OnPutObjectStart(const obs_response_properties *properties, void *_data) + { + return OBS_STATUS_OK; + } + + static int OnPutObjectProgress(int bufferSize, char *buffer, void *_data) + { + auto *data = (PutObjectData *)_data; + + int toRead = 0; + if (data->bufferSize) { + toRead = ((data->bufferSize > (unsigned)bufferSize) ? (unsigned)bufferSize : data->bufferSize); + memcpy(buffer, data->buffer + data->offset, toRead); + } + + data->bufferSize -= toRead; + data->offset += toRead; + + return toRead; + } + + static void OnPutObjectComplete(obs_status status, const obs_error_details *error, void *_data) + { + auto *data = (PutObjectData *)_data; + data->status = status; + } + + static Status PutObjectFromBuffer(const std::string &buffer, const std::string &objectID, + const std::string &bucketID, const S3Config &s3Config) + { + obs_options options; + InitObsOptions(&options, s3Config); + options.bucket_options.bucket_name = const_cast(bucketID.c_str()); + + obs_put_properties properties; + init_put_properties(&properties); + + PutObjectData data = { nullptr }; + // Initialize the structure that stores the uploaded data + data.buffer = const_cast(buffer.c_str()); + data.bufferSize = strlen(data.buffer); + + YRLOG_DEBUG("put object({}) from buffer after init obs.", objectID); + obs_put_object_handler handler = { { &OnPutObjectStart, &OnPutObjectComplete }, &OnPutObjectProgress }; + put_object(&options, const_cast(objectID.c_str()), data.bufferSize, &properties, nullptr, &handler, + &data); + if (OBS_STATUS_OK == data.status) { + YRLOG_DEBUG("success to put object({}) from buffer", objectID); + return Status::OK(); + } else { + YRLOG_ERROR("failed({}) to put object({}) from buffer", obs_get_status_name(data.status), objectID); + return Status(StatusCode::FUNC_AGENT_OBS_PUT_OBJECT_ERROR, + "failed(" + std::string(obs_get_status_name(data.status)) + ") to put object(*) from buffer"); + } + } +}; +} // namespace functionsystem::test::function_agent +#endif // FUNCTION_AGENT_S3_PUT_OBJECT_H \ No newline at end of file diff --git a/functionsystem/tests/unit/function_agent/common/shared_dir_deployer_test.cpp b/functionsystem/tests/unit/function_agent/common/shared_dir_deployer_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..29b4653c6de200add3e8e43eedf2c54313283aa6 --- /dev/null +++ b/functionsystem/tests/unit/function_agent/common/shared_dir_deployer_test.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "function_agent/code_deployer/shared_dir_deployer.h" + +#include +#include + +#include "common/metadata/metadata.h" +#include "common/utils/path.h" +#include "function_agent/common/constants.h" + +namespace functionsystem::test { + +class SharedDirDeployerTest : public testing::Test { + void TearDown() override + { + auto defaultFilePath = litebus::os::Join(functionsystem::GetDeployDir(), "shared"); + litebus::os::Rmdir(defaultFilePath); + std::string filePath = litebus::os::Join("/tmp/DeploySharedDir", "shared"); + litebus::os::Rmdir(filePath); + } +}; + +TEST_F(SharedDirDeployerTest, DeploySharedDir) +{ + auto deployer = std::make_shared(); + + auto defaultFilePath = "/tmp/shared"; + litebus::os::Rmdir(defaultFilePath); + auto des = deployer->GetDestination("", "", "shared1"); + ASSERT_EQ(des, litebus::os::Join(defaultFilePath, "shared1")); + ASSERT_FALSE(deployer->IsDeployed(des, false)); + auto request = std::make_shared(); + request->mutable_deploymentconfig()->set_objectid("shared1"); + auto res = deployer->Deploy(request); + EXPECT_FALSE(res.status.IsError()) << res.status.ToString(); + ASSERT_TRUE(deployer->IsDeployed(des, false)); + struct stat fileStat; + ASSERT_TRUE(stat(des.c_str(), &fileStat) == 0); + mode_t actualMode = fileStat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); + mode_t expectedMode = S_IRWXU | S_IRWXG; + ASSERT_EQ(actualMode, expectedMode) << actualMode; + deployer->Clear(des, ""); + ASSERT_FALSE(deployer->IsDeployed(des, false)); + + std::string filePath = "/tmp/DeploySharedDir"; + litebus::os::Rmdir(filePath); + des = deployer->GetDestination(filePath, "", "shared1"); + ASSERT_EQ(des, litebus::os::Join(filePath, "shared/shared1")); + ASSERT_FALSE(deployer->IsDeployed(des, false)); + request = std::make_shared(); + request->mutable_deploymentconfig()->set_deploydir(filePath); + request->mutable_deploymentconfig()->set_objectid("shared1"); + res = deployer->Deploy(request); + EXPECT_FALSE(res.status.IsError()) << res.status.ToString(); + ASSERT_TRUE(deployer->IsDeployed(des, false)) << des; + deployer->Clear(des, ""); + ASSERT_FALSE(deployer->IsDeployed(des, false)); +} + +TEST_F(SharedDirDeployerTest, DeploySharedDirFailed) +{ + auto deployer = std::make_shared(); + + auto defaultFilePath = litebus::os::Join(functionsystem::GetDeployDir(), "shared"); + auto request = std::make_shared(); + request->mutable_deploymentconfig()->set_objectid("shared1;"); + auto res = deployer->Deploy(request); + EXPECT_TRUE(res.status.IsError()) << res.status.ToString(); + + request->mutable_deploymentconfig()->set_objectid(""); + res = deployer->Deploy(request); + EXPECT_TRUE(res.status.IsError()) << res.status.ToString(); + + request->mutable_deploymentconfig()->set_objectid(std::string(65, 'A')); + res = deployer->Deploy(request); + EXPECT_TRUE(res.status.IsError()) << res.status.ToString(); +} + +TEST_F(SharedDirDeployerTest, SetTTL) +{ + auto deployer = std::make_shared(); + auto dest = "/tmp/shared/abc"; + EXPECT_EQ(deployer->GetTTL(dest), -1); + int ttl = 5; + deployer->SetTTL(dest, ttl); + EXPECT_EQ(deployer->GetTTL(dest), ttl); + deployer->Clear(dest, ""); + EXPECT_EQ(deployer->GetTTL(dest), -1); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_agent/common/utils_test.cpp b/functionsystem/tests/unit/function_agent/common/utils_test.cpp index fd00d2f423447218456b9313edd211a7d90ce403..a30a18fa710b5add2753afce6e08908fccda585b 100644 --- a/functionsystem/tests/unit/function_agent/common/utils_test.cpp +++ b/functionsystem/tests/unit/function_agent/common/utils_test.cpp @@ -22,9 +22,9 @@ #include #include #include - -#include "logs/logging.h" -#include "hex/hex.h" +#include "common/utils/sensitive_value.h" +#include "common/logs/logging.h" +#include "common/hex/hex.h" #include "function_agent/common/constants.h" namespace functionsystem::test { @@ -74,6 +74,8 @@ const std::string PARSED_JSON = "usr/lib/python3/dist-packages\",\"func-FAAS_FUNCTION_REGION\":\"cn\",\"func-FAAS_FUNCTION_TIMEZONE\":\"Asia/" "Shanghai\",\"func-adminFuncLoad\":\"true\",\"func-stream\":\"true\"}"; +const std::string RUNTIME_ENV_PREFIX = "func-"; + class FunctionAgentUtilsTest : public ::testing::Test { public: void SetUp() override @@ -216,6 +218,11 @@ TEST_F(FunctionAgentUtilsTest, SetRuntimeConfigSuccess) EXPECT_EQ(runtimeConfig.subdirectoryconfig().parentdirectory(), "/tmp"); EXPECT_EQ(runtimeConfig.subdirectoryconfig().quota(), 355); + // set disk mount point + auto diskEnvKey = RUNTIME_ENV_PREFIX + resource_view::DISK_MOUNT_POINT; + (*deployInstanceRequest->mutable_createoptions())[diskEnvKey] = "/tmp/abc/"; + runtimeConfig = function_agent::SetRuntimeConfig(deployInstanceRequest); + EXPECT_TRUE(runtimeConfig.userenvs().find(diskEnvKey) != runtimeConfig.userenvs().end()); (void)litebus::os::Rmdir(resourcePath); } @@ -495,6 +502,22 @@ TEST_F(FunctionAgentUtilsTest, AddDefaultEnvWithDELEGATE_ENV_VAR) EXPECT_EQ((*runtimeConf2.mutable_posixenvs()).size(), 1); } +TEST_F(FunctionAgentUtilsTest, SensitiveValueHashTest) +{ + const std::string plaint = "secret-key"; + + const auto value = SensitiveValue(plaint); + + ::messages::TenantCredentials credentials; + credentials.set_secretkey(value.GetData()); + + ::messages::TenantCredentials credentials2; + credentials2.set_secretkey(value.GetData(), value.GetSize()); + + const auto value_hash = std::hash()(value.GetData()); + EXPECT_EQ(value_hash, std::hash()(credentials.secretkey())); + EXPECT_EQ(value_hash, std::hash()(credentials2.secretkey())); +} TEST_F(FunctionAgentUtilsTest, DecryptDelegateDataTest) { std::string delegateData = "{\"accessKey\":\"\",\"authToken\":\"\",\"cryptoAlgorithm\":\"NO_CRYPTO\",\"encrypted_user_data\":\"\",\"envKey\":\"\",\"environment\":\"{\\\"key1\\\":\\\"val111\\\",\\\"key2\\\":\\\"val222\\\"}\",\"secretKey\":\"\",\"securityAk\":\"\",\"securitySk\":\"\",\"securityToken\":\"\"}"; diff --git a/functionsystem/tests/unit/function_agent/network/network_tool_test.cpp b/functionsystem/tests/unit/function_agent/network/network_tool_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..39321a297495f49e0d3d40f90aa90373bd417085 --- /dev/null +++ b/functionsystem/tests/unit/function_agent/network/network_tool_test.cpp @@ -0,0 +1,492 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "function_agent/network/network_tool.h" +#include "common/utils/exec_utils.h" +#include "common/utils/path.h" + +namespace functionsystem::test { + +class NetworkToolTest : public testing::Test { + +}; + +/** + * Feature: SetFirewallTest + * Description: Test Network Tool Set Firewall + * Steps: + * 1. iptables add command with lacking field + * 2. iptables delete command with lacking field + * Expectation: + * 1. return false + * 2. return false + * 3. return false + * 4. return false + * 5. return false + * 6. return false + */ +TEST_F(NetworkToolTest, SetFirewallTest) +{ + // given + function_agent::FirewallConfig configs[6] = { + {"", "filter", "add", "1.1.1.1", "-j ACCEPT"}, + {"", "filter", "delete", "", ""}, + {"", "", "add", "1.1.1.1", "-j ACCEPT"}, + {"INPUT", " ", "add", "1.1.1.1", "-j ACCEPT"}, + {"INPUT", "filter", "delete", "1.1.1.1", "-j ACCEPT"}, + {"INPUT", "filter", "get", "1.1.1.1", "-j ACCEPT"}, + }; + + // want + bool wants[6] = {false, false, false, false, false, false}; + + // got + for (size_t i = 0; i < sizeof(configs) / sizeof(function_agent::FirewallConfig); i++) { + EXPECT_TRUE(function_agent::NetworkTool::SetFirewall(configs[i]) == wants[i]); + } +} + +/** + * Feature: SetRouteTest + * Description: Test Network Tool Set Route + * Steps: + * 1. route config with lacking field + * Expectation: + * 1. return false + */ +TEST_F(NetworkToolTest, SetRouteTest) +{ + auto res = ExecuteCommand("route -n"); + if (!res.error.empty()) { + std::cout << "execute route failed, error: " << res.error << std::endl; + return; + } + + if (res.output.find("eth0") == res.output.npos) { + std::cout << "does not contain interface eth0" << std::endl; + return; + } + + // given + function_agent::RouteConfig configs[1] = { + {"", "0.0.0.0/20", "eth0"}, + }; + + // want + bool wants[1] = {false}; + + // got + for (size_t i = 0; i < sizeof(configs) / sizeof(function_agent::RouteConfig); i++) { + EXPECT_TRUE(function_agent::NetworkTool::SetRoute(configs[i]) == wants[i]); + } + +} + +/** + * Feature: SetTunnelTest + * Description: Test Network Tool Set Tunnel + * Steps: + * 1. tunnel config with lacking field + * Expectation: + * 1. return false + * 2. return false + * 3. return false + * 4. return false + * 5. return false + */ +TEST_F(NetworkToolTest, SetTunnelTest) +{ + // given + function_agent::TunnelConfig configs[5] = { + {"", "127.0.0.1", "ipip", "0.0.0.0"}, + {"tun1", "127.0.0.1", "", "0.0.0.0"}, + {"tun1", "", "ipip", "0.0.0.0"}, + {"tun1", "localip", "ipip", "0.0.0.0"}, + {"tun1", "127.0.0.1", " ", "0.0.0.0"}, + }; + + // want + bool wants[5] = {false, false, false, false, false}; + + // got + for (size_t i = 0; i < sizeof(configs) / sizeof(function_agent::TunnelConfig); i++) { + EXPECT_TRUE(function_agent::NetworkTool::SetTunnel(configs[i]) == wants[i]); + } + +} + +/** + * Feature: GetAddrTest + * Description: Test Network Tool Get Addr Info By ip + * Steps: + * 1. get address information by 127.0.0.1 + * Expectation: + * 1. get interface lo + */ +TEST_F(NetworkToolTest, GetAddrTest) +{ + auto addr = function_agent::NetworkTool::GetAddr("127.0.0.1"); + EXPECT_TRUE(addr.IsSome()); + EXPECT_TRUE(addr.Get().interface == "lo"); +} + +/** + * Feature: ParseNetworkConfigTest + * Description: Test Network Tool Parse NetworkConfig + * Steps: + * 1. give network config string + * Expectation: + * get correct value + */ +TEST_F(NetworkToolTest, ParseNetworkConfigTest) +{ + std::string str = R"([{"routeConfig":{"gateway":"1.1.1.1","cidr":"1.1.1.0/24"}, "tunnelConfig":{"tunnelName":"test","remoteIP":"1.1.1.1","mode":"ipip"},"firewallConfig":{"chain":"OUTPUT","table":"filter","operation":"add","target":"1.1.1.1","args":"-j ACCEPT"}}])"; + auto config = function_agent::NetworkTool::ParseNetworkConfig(str); + EXPECT_TRUE(config[0].routeConfig.IsSome()); + EXPECT_TRUE(config[0].routeConfig.Get().gateway == "1.1.1.1"); + EXPECT_TRUE(config[0].routeConfig.Get().cidr == "1.1.1.0/24"); + + EXPECT_TRUE(config[0].tunnelConfig.IsSome()); + EXPECT_TRUE(config[0].tunnelConfig.Get().tunnelName == "test"); + EXPECT_TRUE(config[0].tunnelConfig.Get().remoteIP == "1.1.1.1"); + EXPECT_TRUE(config[0].tunnelConfig.Get().mode == "ipip"); + + EXPECT_TRUE(config[0].firewallConfig.IsSome()); + EXPECT_TRUE(config[0].firewallConfig.Get().chain == "OUTPUT"); + EXPECT_TRUE(config[0].firewallConfig.Get().table == "filter"); + EXPECT_TRUE(config[0].firewallConfig.Get().operation == "add"); + EXPECT_TRUE(config[0].firewallConfig.Get().target == "1.1.1.1"); + EXPECT_TRUE(config[0].firewallConfig.Get().args == "-j ACCEPT"); + + std::string str1 = R"([{"routeConfig":{"gateway":1,"cidr":1}, "tunnelConfig":{"tunnelName":12,"remoteIP":1,"mode":1},"firewallConfig":{"chain":1,"table":1,"operation":1,"target":1,"args":1}}])"; + auto config1 = function_agent::NetworkTool::ParseNetworkConfig(str1); + EXPECT_TRUE(config1[0].routeConfig.IsSome()); + EXPECT_TRUE(config1[0].routeConfig.Get().gateway == ""); + EXPECT_TRUE(config1[0].routeConfig.Get().cidr == "default"); + + EXPECT_TRUE(config1[0].tunnelConfig.IsSome()); + EXPECT_TRUE(config1[0].tunnelConfig.Get().tunnelName == ""); + EXPECT_TRUE(config1[0].tunnelConfig.Get().remoteIP == ""); + EXPECT_TRUE(config1[0].tunnelConfig.Get().mode == ""); + + EXPECT_TRUE(config1[0].firewallConfig.IsSome()); + EXPECT_TRUE(config1[0].firewallConfig.Get().chain == ""); + EXPECT_TRUE(config1[0].firewallConfig.Get().table == ""); + EXPECT_TRUE(config1[0].firewallConfig.Get().operation == ""); + EXPECT_TRUE(config1[0].firewallConfig.Get().target == ""); + EXPECT_TRUE(config1[0].firewallConfig.Get().args == ""); + + str1 = "fake_json"; + config1 = function_agent::NetworkTool::ParseNetworkConfig(str1); + EXPECT_EQ(config1.size(), 0); +} + +/** + * Feature: ParseNetworkConfigWithIncorrectStrTest + * Description: Test Network Tool Parse NetworkConfig + * Steps: + * 1. give Incorrect network config string + * Expectation: + * return configs with zero size + */ +TEST_F(NetworkToolTest, ParseNetworkConfigWithErrorStrTest) +{ + std::string str = R"(["routeConfig":{"gateway":"1.1.1.1","cidr":"1.1.1.0/24"}, "tunnelConfig":{"tunnelName":"test","remoteIP":"1.1.1.1","mode":"ipip"},"firewallConfig":{"chain":"OUTPUT","table":"filter","operation":"add","target":"1.1.1.1","args":"-j ACCEPT"}}])"; + auto config = function_agent::NetworkTool::ParseNetworkConfig(str); + EXPECT_TRUE(config.size() == 0); + + str = R"({"routeConfig":{"gateway":"1.1.1.1","cidr":"1.1.1.0/24"}, "tunnelConfig":{"tunnelName":"test","remoteIP":"1.1.1.1","mode":"ipip"},"firewallConfig":{"chain":"OUTPUT","table":"filter","operation":"add","target":"1.1.1.1","args":"-j ACCEPT"}}])"; + config = function_agent::NetworkTool::ParseNetworkConfig(str); + EXPECT_TRUE(config.size() == 0); +} + +/** + * Feature: ParseProberConfigTest + * Description: Test Network Tool Parse Probe Config + * Steps: + * 1. give Probe config string + * Expectation: + * get correct value + */ +TEST_F(NetworkToolTest, ParseProberConfigTest) +{ + std::string str = R"([{"protocol":"ICMP","address":"1.1.1.1","interval":10,"timeout":10,"failureThreshold":1}])"; + auto config = function_agent::NetworkTool::ParseProberConfig(str); + EXPECT_TRUE(config[0].protocol == "ICMP"); + EXPECT_TRUE(config[0].address == "1.1.1.1"); + EXPECT_TRUE(config[0].interval == 10); + EXPECT_TRUE(config[0].timeout == 10); + EXPECT_TRUE(config[0].failureThreshold == 1); + + std::string str1 = R"([{"protocol":1,"address":1,"interval":"10","timeout":10,"failureThreshold":1}])"; + auto config1 = function_agent::NetworkTool::ParseProberConfig(str1); + EXPECT_TRUE(config1[0].protocol == ""); + EXPECT_TRUE(config1[0].address == ""); + EXPECT_TRUE(config1[0].interval == 0); + EXPECT_TRUE(config1[0].timeout == 10); + EXPECT_TRUE(config1[0].failureThreshold == 1); + + EXPECT_FALSE(function_agent::NetworkTool::Probe(config1)); +} + +/** + * Feature: ParseProberConfigWithIncorrectStrTest + * Description: Test Network Tool Parse Probe Config + * Steps: + * 1. give Incorrect Probe config string + * Expectation: + * return configs with zero size + */ +TEST_F(NetworkToolTest, ParseProberConfigWithIncorrectStrTest) +{ + std::string str = R"(["protocol":"ICMP","address":"1.1.1.1","interval":10,"timeout":10,"failureThreshold":1}])"; + auto config = function_agent::NetworkTool::ParseProberConfig(str); + EXPECT_TRUE(config.size() == 0); + + str = R"({"protocol":"ICMP","address":"1.1.1.1","interval":10,"timeout":10,"failureThreshold":1}])"; + config = function_agent::NetworkTool::ParseProberConfig(str); + EXPECT_TRUE(config.size() == 0); +} + +/** + * Feature: PingSuccessTest + * Description: Test Ping + * Steps: + * give correct Probe config string + * Expectation: + * ping success + */ +TEST_F(NetworkToolTest, PingSuccessTest) +{ + if (LookPath("ping").IsNone()) { + return; + } + std::string str = R"([{"protocol":"ICMP","address":"127.0.0.1","interval":1,"timeout":1,"failureThreshold":1}])"; + auto configs = function_agent::NetworkTool::ParseProberConfig(str); + for (const auto &config: configs) { + EXPECT_TRUE(function_agent::NetworkTool::Ping(config)); + } +} + +/** + * Feature: PingFailedTest + * Description: Test Ping + * Steps: + * give Incorrect Probe config string + * Expectation: + * ping failed + */ +TEST_F(NetworkToolTest, PingFailedTest) +{ + if (LookPath("ping").IsNone()) { + return; + } + std::string str = R"([{"protocol":"ICMP","address":"1.1.1.1","interval":1,"timeout":1,"failureThreshold":1}])"; + auto configs = function_agent::NetworkTool::ParseProberConfig(str); + for (const auto &config: configs) { + EXPECT_FALSE(function_agent::NetworkTool::Ping(config)); + } + + std::string str1 = R"([{"protocol":"ICMP","address":"1.1.1.1.","interval":1,"timeout":1,"failureThreshold":1}])"; + auto configs1 = function_agent::NetworkTool::ParseProberConfig(str); + for (const auto &config: configs1) { + EXPECT_FALSE(function_agent::NetworkTool::Ping(config)); + } +} + +/** + * Feature: GetRouteConfigTest + * Description: Test RouteConfig + * Steps: + * give Incorrect AND correct RouteConfig + * Expectation: + * get failed + */ + +TEST_F(NetworkToolTest, GetRouteConfigTest) +{ + std::string emptyStr = ""; + const int32_t DEFAULT_EMPTY_SIZE = 0; + EXPECT_TRUE(function_agent::NetworkTool::GetRouteConfig(emptyStr).size() == DEFAULT_EMPTY_SIZE); + + std::string wrongStr = "/home"; + EXPECT_TRUE(function_agent::NetworkTool::GetRouteConfig(wrongStr).size() == DEFAULT_EMPTY_SIZE); + + wrongStr = "192.168.123.2/234"; + EXPECT_TRUE(function_agent::NetworkTool::GetRouteConfig(wrongStr).size() == DEFAULT_EMPTY_SIZE); + + std::string rightStr = "dev eth0"; + auto result = function_agent::NetworkTool::GetRouteConfig(rightStr); + EXPECT_TRUE(function_agent::NetworkTool::GetRouteConfig(rightStr).size() >= DEFAULT_EMPTY_SIZE); + + rightStr = "192.168.1.1/24"; + EXPECT_TRUE(function_agent::NetworkTool::GetRouteConfig(rightStr).size() >= DEFAULT_EMPTY_SIZE); +} + +/** + * Feature: SetRouteParameterTest + * Description: Test set route parameter + * Steps: + * give Incorrect RouteConfig + * Expectation: + * set route failed + */ +TEST_F(NetworkToolTest, SetRouteParameterTest) +{ + functionsystem::function_agent::RouteConfig wrongCidr1; + wrongCidr1.cidr = ""; + wrongCidr1.gateway = "0.0.0.0"; + wrongCidr1.interface = "ens33"; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongCidr1)); + + functionsystem::function_agent::RouteConfig wrongCidr2; + wrongCidr2.cidr = "dev eth0"; + wrongCidr2.gateway = "0.0.0.0"; + wrongCidr2.interface = "ens33"; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongCidr2)); + + functionsystem::function_agent::RouteConfig wrongCidr3; + wrongCidr3.cidr = "192.168.1.1/24"; + wrongCidr3.gateway = "0.0.0.0"; + wrongCidr3.interface = "ens33"; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongCidr3)); + + functionsystem::function_agent::RouteConfig wrongCidr4; + wrongCidr4.cidr = "192.168.1.1/24"; + wrongCidr4.gateway = "0.0.0.0"; + wrongCidr4.interface = ""; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongCidr4)); + + functionsystem::function_agent::RouteConfig wrongInterface1; + wrongInterface1.cidr = "dev eth0"; + wrongInterface1.gateway = "0.0.0.0"; + wrongInterface1.interface = ""; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongInterface1)); + + functionsystem::function_agent::RouteConfig wrongGateway1; + wrongGateway1.cidr = "dev eth0"; + wrongGateway1.gateway = ""; + wrongGateway1.interface = "ens33"; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongGateway1)); + + functionsystem::function_agent::RouteConfig wrongGateway2; + wrongGateway2.cidr = "dev eth0"; + wrongGateway2.gateway = "gateway"; + wrongGateway2.interface = "ens33"; + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(wrongGateway2)); + + functionsystem::function_agent::RouteConfig right1; + right1.cidr = "dev eth0"; + right1.gateway = "0.0.0.0"; + right1.interface = "ens33"; + + EXPECT_FALSE(function_agent::NetworkTool::SetRoute(right1)); +} + +/** + * Feature: RestoreRouteTest + * Description: Test restore route + * Steps: + * give incorrect config + * Expectation: + * restore route failed + */ +TEST_F(NetworkToolTest, RestoreRouteTest) +{ + std::vector config = {"192.168.2.0/24 via 192.168.1.1", + "", + "192.168.2.1/24 via 192.168.1.2", + "192.168.2.2/24 via 192.168.1.3"}; + EXPECT_TRUE(function_agent::NetworkTool::RestoreRoute(config)); +} + +/** + * Feature: GetNameServerListTest + * Description: Test get name server list + * Steps: + * call function to get name server list + * Expectation: + * restore route failed + */ +TEST_F(NetworkToolTest, GetNameServerListTest) +{ + auto result = function_agent::NetworkTool::GetNameServerList(); + const int32_t DEFAULT_SIZE = 0; + EXPECT_TRUE(result.size() > DEFAULT_SIZE); +} + +/** + * Feature: PingTest + * Description: Test Ping parameters + * Steps: + * give correct Probe config string + * Expectation: + * ping success + */ +TEST_F(NetworkToolTest, PingTest) +{ + // given + function_agent::ProberConfig configs[6] = { + {"ICMP", "", 200, 2, 3}, + {"ICMP", "localip", 200, 2, 3}, + {"ICMP", "1.1.1.1", -1, 2, 3}, + {"ICMP", "1.1.1.1", 256, 2, 3}, + {"ICMP", "1.1.1.1", 123, -1, 3}, + {"ICMP", "1.1.1.1", 123, 2, 3}, + }; + + // want + bool wants[6] = {false, false, false, false, false, false}; + + // got + for (size_t i = 0; i < 6; i++) { + EXPECT_TRUE(function_agent::NetworkTool::Ping(configs[i]) == wants[i]); + } + + EXPECT_FALSE(function_agent::NetworkTool::IsIpsetExists("name")); +} + +TEST_F(NetworkToolTest, CheckIllegalCharsTest) +{ + // given + std::string command1 = "cmd1;cmd2"; + std::string command2 = "$(cmd)"; + std::string command3 = "`cmd`"; + std::string command4 = "cmd1||cmd2"; + std::string command5 = "cmd1&&cmd2"; + std::string command6 = ">(cmd1)|<(cmd2)"; + std::string command7 = "[cmd1]"; + std::string command8 = "{cmd1}"; + std::string command9 = "cmd1* cmd2"; + std::string command10 = "cmd1?cmd2"; + std::string command11 = "cmd1\ncmd2"; + std::string command12 = "cmd1\\cmd2"; + EXPECT_FALSE(CheckIllegalChars(command1)); + EXPECT_FALSE(CheckIllegalChars(command2)); + EXPECT_FALSE(CheckIllegalChars(command3)); + EXPECT_FALSE(CheckIllegalChars(command4)); + EXPECT_FALSE(CheckIllegalChars(command5)); + EXPECT_FALSE(CheckIllegalChars(command6)); + EXPECT_FALSE(CheckIllegalChars(command7)); + EXPECT_FALSE(CheckIllegalChars(command8)); + EXPECT_FALSE(CheckIllegalChars(command9)); + EXPECT_FALSE(CheckIllegalChars(command10)); + EXPECT_FALSE(CheckIllegalChars(command11)); + EXPECT_FALSE(CheckIllegalChars(command12)); +} +} diff --git a/functionsystem/tests/unit/function_master/CMakeLists.txt b/functionsystem/tests/unit/function_master/CMakeLists.txt index 8ccca502b7b49b60408e277827b871eab33dfa05..cc0f86da920667923a4f3b155cc6770ba1fcba32 100644 --- a/functionsystem/tests/unit/function_master/CMakeLists.txt +++ b/functionsystem/tests/unit/function_master/CMakeLists.txt @@ -15,4 +15,6 @@ add_subdirectory(global_scheduler) add_subdirectory(instance_manager) add_subdirectory(meta_store) +add_subdirectory(system_function_loader) +add_subdirectory(scaler) add_subdirectory(resource_group_manager) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/global_scheduler/global_sched_driver_test.cpp b/functionsystem/tests/unit/function_master/global_scheduler/global_sched_driver_test.cpp index 5423a20e7d914c8bf448aeeac7f070b380a63b43..c1645a7062f9b011ca2409d64eb4e6cc4af38f40 100644 --- a/functionsystem/tests/unit/function_master/global_scheduler/global_sched_driver_test.cpp +++ b/functionsystem/tests/unit/function_master/global_scheduler/global_sched_driver_test.cpp @@ -109,14 +109,290 @@ TEST_F(GlobalSchedDriverTest, QueryHealthyRouter) globalSchedDriver_->Await(); } +// test query agent info +// case1: invalid method +// case2: query successful (not empty) +TEST_F(GlobalSchedDriverTest, QueryAgentsRouter) +{ + auto globalSchedDriver_ = + std::make_shared(mockGlobalSched_, flags_, mockMetaStoreClient_); + EXPECT_CALL(*mockGlobalSched_, Start(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, Stop).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, InitManager).WillOnce(Return()); + auto status = globalSchedDriver_->Start(); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::http::URL urlQueryAgents("http", "127.0.0.1", port, GLOBAL_SCHEDULER + QUERY_AGENTS_URL); + // case1: invalid method + { + auto response = litebus::http::Post(urlQueryAgents, litebus::None(), litebus::None(), litebus::None()); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::METHOD_NOT_ALLOWED); + } + + // case2: query successful (empty) + { + auto resp = messages::QueryAgentInfoResponse(); + auto info = resp.add_agentinfos(); + info->set_agentid("agentID"); + info->set_alias("alias"); + info->set_localid("localID"); + EXPECT_CALL(*mockGlobalSched_, QueryAgentInfo(_)).WillOnce(Return(resp)); + auto response = litebus::http::Get(urlQueryAgents, litebus::None()); + response.Wait(); + EXPECT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto body = response.Get().body; + auto infos = messages::ExternalQueryAgentInfoResponse(); + EXPECT_EQ(google::protobuf::util::JsonStringToMessage(body, &infos).ok(), true); + ASSERT_EQ(infos.data().size(), 1); + EXPECT_EQ(infos.data().Get(0).id(), "localID/agentID"); + EXPECT_EQ(infos.data().Get(0).alias(), "alias"); + } + + globalSchedDriver_->Stop(); + globalSchedDriver_->Await(); +} + messages::FunctionSystemStatus ParseResponse(const std::string &body) { messages::FunctionSystemStatus status; YRLOG_INFO("body: {}", body); - google::protobuf::util::JsonStringToMessage(body, &status) ; + (void)google::protobuf::util::JsonStringToMessage(body, &status) ; return status; } +// test evcit agent info +// case1: invalid method +// case2: invalid body +// case3: invalid timemout +// case4: invalid agentID +// case5: query return failed +// case6: query return successful +// case7: default timeout +// case8: empty agentID +// case9: serialize success but without agentID +// case10: serialize fail but with agentID +// case11: corner case +TEST_F(GlobalSchedDriverTest, EvictAgentRouter) +{ + auto globalSchedDriver_ = + std::make_shared(mockGlobalSched_, flags_, mockMetaStoreClient_); + EXPECT_CALL(*mockGlobalSched_, Start(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, Stop).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, InitManager).WillOnce(Return()); + auto status = globalSchedDriver_->Start(); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::http::URL urlEvictAgent("http", "127.0.0.1", port, GLOBAL_SCHEDULER + EVICT_AGENT_URL); + // case1: invalid method + { + auto response = litebus::http::Get(urlEvictAgent, litebus::None()); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::METHOD_NOT_ALLOWED); + } + // case2: invalid body + { + std::string reqData = + "{\"agentid\": \"EuerOS-220-41-65280/function_agent_10.30.220.41-29847\"," + "\"timeoutsec\": \"10\"}"; + std::string contentType = "application/json"; + auto response = + litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + } + // case3: invalid timemout + { + std::string reqData = "{\"agentID\": \"EuerOS-220-41-65280/function_agent_10.30.220.41-29847\"," + "\"timeoutSec\": \"6001\"}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + } + // case4: invalid agentID + { + std::string reqData = + "{\"agentID\": \"xxxxxxx\"," + "\"timeoutSec\": \"10\"}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + } + // case5: query return failed + { + std::string reqData = + "{\"agentID\": \"localID/agentID\"," + "\"timeoutSec\": \"10\"}"; + std::string contentType = "application/json"; + EXPECT_CALL(*mockGlobalSched_, EvictAgent(Eq("localID"), _)) + .WillOnce(Return(Status(StatusCode::PARAMETER_ERROR))); + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + } + // case6: query return successful + { + std::string reqData = + "{\"agentID\": \"localID/agentID\"," + "\"timeoutSec\": \"10\"}"; + std::string contentType = "application/json"; + EXPECT_CALL(*mockGlobalSched_, EvictAgent(Eq("localID"), _)) + .WillOnce(Return(Status::OK())); + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_NONE); + } + + // case7: default timeout + { + std::string reqData = + "{\"agentID\": \"localID/agentID\"," + "\"timeoutSec\": \"0\"}"; + std::string contentType = "application/json"; + EXPECT_CALL(*mockGlobalSched_, EvictAgent(Eq("localID"), _)) + .WillOnce( + DoAll(Invoke([](const std::string &localID, + const std::shared_ptr &req) -> litebus::Future { + EXPECT_EQ(req->timeoutsec(), uint32_t(30)); + EXPECT_EQ(req->agentid(), "agentID"); + return Status::OK(); + }))); + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_NONE); + } + + // case8: empty agentID + { + std::string reqData = "{}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "Empty", status.message()); + } + // case9: serialize success but without agentID + { + std::string reqData = "{\"timeoutSec\": 10}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "Empty", status.message()); + } + // case10: serialize fail but with agentID + { + std::string reqData = "{\"agentID\": \"localID/agentID\"," + "\"timeoutsec\": \"0\"}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + EXPECT_PRED_FORMAT2(testing::IsNotSubstring, "Empty", status.message()); + } + // case11: corner case + { + std::string reqData = "{\"agentID\": \"localID/\"," + "\"timeoutSec\": \"10\"}"; + std::string contentType = "application/json"; + auto response = litebus::http::Post(urlEvictAgent, litebus::None(), reqData, contentType); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto status = ParseResponse(response.Get().body); + EXPECT_EQ(status.code(), common::ERR_PARAM_INVALID); + EXPECT_PRED_FORMAT2(testing::IsSubstring, "Invalid", status.message()); + } + globalSchedDriver_->Stop(); + globalSchedDriver_->Await(); +} + +// test query agent count +// case1: invalid method +// case2: query successful (not empty) +// case3: query fail (not ok) +// case4: query fail (multiple results) +TEST_F(GlobalSchedDriverTest, QueryAgentCountRouter) +{ + auto globalSchedDriver_ = + std::make_shared(mockGlobalSched_, flags_, mockMetaStoreClient_); + EXPECT_CALL(*mockGlobalSched_, Start(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, Stop).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, InitManager).WillOnce(Return()); + auto status = globalSchedDriver_->Start(); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::http::URL urlQueryAgentCount("http", "127.0.0.1", port, GLOBAL_SCHEDULER + QUERY_AGENT_COUNT_URL); + // case1: invalid method + { + auto response = litebus::http::Post(urlQueryAgentCount, litebus::None(), litebus::None(), litebus::None()); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::METHOD_NOT_ALLOWED); + } + + // case2: query successful + { + auto resp = std::make_shared(); + KeyValue kv; + kv.set_key(READY_AGENT_CNT_KEY); + kv.set_value("100"); + resp->kvs.emplace_back(kv); + EXPECT_CALL(*mockMetaStoreClient_, Get(_, _)).WillOnce(Return(resp)); + auto response = litebus::http::Get(urlQueryAgentCount, litebus::None()); + response.Wait(); + EXPECT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto body = response.Get().body; + EXPECT_TRUE(body == "100"); + } + + // case3: query fail (not ok) + { + auto resp = std::make_shared(); + KeyValue kv; + kv.set_key(READY_AGENT_CNT_KEY); + kv.set_value("100"); + resp->kvs.emplace_back(kv); + resp->kvs.emplace_back(kv); + EXPECT_CALL(*mockMetaStoreClient_, Get(_, _)).WillOnce(Return(resp)); + auto response = litebus::http::Get(urlQueryAgentCount, litebus::None()); + response.Wait(); + EXPECT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto body = response.Get().body; + EXPECT_TRUE(body == "-1"); + } + + // case4: query fail (multiple results) + { + auto resp = std::make_shared(); + resp->status = Status(StatusCode::FAILED, "get failed"); + EXPECT_CALL(*mockMetaStoreClient_, Get(_, _)).WillOnce(Return(resp)); + auto response = litebus::http::Get(urlQueryAgentCount, litebus::None()); + response.Wait(); + EXPECT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto body = response.Get().body; + EXPECT_TRUE(body == "-1"); + } + + globalSchedDriver_->Stop(); + globalSchedDriver_->Await(); +} + resource_view::InstanceInfo GetInstanceInfo(std::string instanceId) { Resources resources; @@ -134,6 +410,46 @@ resource_view::InstanceInfo GetInstanceInfo(std::string instanceId) return instanceInfo; } +TEST_F(GlobalSchedDriverTest, GetSchedulingQueue) +{ + auto globalSchedDriver_ = + std::make_shared(mockGlobalSched_, flags_, mockMetaStoreClient_); + EXPECT_CALL(*mockGlobalSched_, Start(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockGlobalSched_, InitManager).WillOnce(Return()); + auto status = globalSchedDriver_->Start(); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + litebus::http::URL urlGetSchedulingQueue("http", "127.0.0.1", port, GLOBAL_SCHEDULER + GET_SCHEDULING_QUEUE_URL); + + // case1: invalid method + { + auto response = litebus::http::Post(urlGetSchedulingQueue, litebus::None(), litebus::None(), litebus::None()); + response.Wait(); + ASSERT_EQ(response.Get().retCode, litebus::http::ResponseCode::METHOD_NOT_ALLOWED); + } + + // case2: query successful + { + auto resp = messages::QueryInstancesInfoResponse(); + resp.set_requestid("requestIdIdId"); + google::protobuf::RepeatedPtrField &instanceinfos = *resp.mutable_instanceinfos(); + instanceinfos.Add(std::move(GetInstanceInfo("app-script-1-instanceid"))); + instanceinfos.Add(std::move(GetInstanceInfo("app-script-2-instanceid"))); + EXPECT_CALL(*mockGlobalSched_, GetSchedulingQueue(_)).WillOnce(Return(resp)); + + auto response = litebus::http::Get(urlGetSchedulingQueue, litebus::None()); + response.Wait(); + EXPECT_EQ(response.Get().retCode, litebus::http::ResponseCode::OK); + auto body = response.Get().body; + auto infos = messages::QueryInstancesInfoResponse(); + + EXPECT_EQ(google::protobuf::util::JsonStringToMessage(body, &infos).ok(), true); + EXPECT_EQ(infos.instanceinfos_size(), 2); + } + + globalSchedDriver_->Stop(); + globalSchedDriver_->Await(); +} + // test query resource info // case1: invalid method // case2: query successful (not empty) diff --git a/functionsystem/tests/unit/function_master/global_scheduler/global_sched_test.cpp b/functionsystem/tests/unit/function_master/global_scheduler/global_sched_test.cpp index 356a1b0760ade05a916522b604601ace2b843d1e..b8873a7accd5855e0e39e9348684c99192b28d4f 100644 --- a/functionsystem/tests/unit/function_master/global_scheduler/global_sched_test.cpp +++ b/functionsystem/tests/unit/function_master/global_scheduler/global_sched_test.cpp @@ -37,12 +37,16 @@ using namespace global_scheduler; using namespace domain_scheduler; using namespace ::testing; -const std::string TEST_META_STORE_ADDRESS = "127.0.0.1:32279"; const std::string TEST_DOMAIN_ACTIVATOR_ACTOR_NAME = "TestDomainActivator"; const std::string TEST_GLOBAL_SCHEDULER_ACTOR_NAME = "TestGlobalSchedActor"; class GlobalSchedTest : public ::testing::Test { public: + [[maybe_unused]] static void SetUpTestSuite() + { + metaStoreAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + } + void SetUp() override { auto domainSchedMgr = std::make_unique(); @@ -61,9 +65,9 @@ public: EXPECT_CALL(*mockLocalSchedMgr_, DelLocalSchedCallback).WillOnce(Return(Status::OK())); globalSched_.InitManager(std::move(domainSchedMgr), std::move(localSchedMgr)); - mockMetaStoreClient_ = std::make_shared(TEST_META_STORE_ADDRESS); + mockMetaStoreClient_ = std::make_shared(metaStoreAddress_); - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); explorer::Explorer::NewStandAloneExplorerActorForMaster(explorer::ElectionInfo{}, GetLeaderInfo(litebus::AID("function_master", "127.0.0.1:" + std::to_string(port)))); @@ -97,6 +101,7 @@ public: protected: GlobalSched globalSched_; + inline static std::string metaStoreAddress_; std::shared_ptr globalSchedActor_; std::shared_ptr mockMetaStoreClient_; std::shared_ptr mockDomainSchedulerLauncher_; @@ -136,7 +141,7 @@ TEST_F(GlobalSchedTest, StartGlobalScheduler) EXPECT_CALL(*localSchedMgr, DelLocalSchedCallback).WillRepeatedly(Return(Status::OK())); globalSched.InitManager(std::move(domainSchedMgr), std::move(localSchedMgr)); - auto mockMetaStoreClient = std::make_shared(TEST_META_STORE_ADDRESS); + auto mockMetaStoreClient = std::make_shared(metaStoreAddress_); auto topologyTree = std::make_unique(2, 2); auto mockDomainSchedulerLauncher = std::make_shared(); diff --git a/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_actor_test.cpp b/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_actor_test.cpp index 91237d486bf7540b7b7f906d78b57e31d18d40c1..b2a3b898cd249c8b1930e4bed630219095c5aec0 100644 --- a/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_actor_test.cpp +++ b/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_actor_test.cpp @@ -17,7 +17,7 @@ #include "function_master/global_scheduler/scheduler_manager/domain_sched_mgr_actor.h" #include "common/constants/actor_name.h" -#include "heartbeat/ping_pong_driver.h" +#include "common/heartbeat/heartbeat_client.h" #include "common/utils/generate_message.h" #include "domain_scheduler/domain_scheduler_service/uplayer_stub.h" #include "gmock/gmock.h" @@ -27,6 +27,7 @@ #include "utils/generate_info.h" namespace functionsystem::test { +const long long HEARTBEAT_TIMEOUT = 30000; using namespace functionsystem::global_scheduler; @@ -43,7 +44,8 @@ protected: public: static void DomainSchedulerRegister(std::string &&name, std::string &&responseMsg, std::string &®isterMsg) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); litebus::Spawn(actor); litebus::Async(actor->GetAID(), &DomainSchedMgrActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); @@ -135,7 +137,8 @@ TEST_F(DomainSchedMgrActorTest, DomainSchedulerRegisterWithValidRequest) }; // got - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); auto scheduler = std::make_shared("MockDomainScheduler"); for (uint32_t i = 0; i < sizeof(givens) / sizeof(std::string); i++) { auto wantName = wants[i][0]; @@ -192,7 +195,7 @@ TEST_F(DomainSchedMgrActorTest, NotifySchedAbnormal) // got for (uint32_t i = 0; i < sizeof(wants) / sizeof(std::string); i++) { - auto actor = std::make_shared("TestDomainSchedActor"); + auto actor = std::make_shared("TestDomainSchedActor", HEARTBEAT_TIMEOUT); auto scheduler = std::make_shared("MockDomainScheduler"); litebus::Spawn(actor); litebus::Spawn(scheduler); @@ -226,7 +229,7 @@ TEST_F(DomainSchedMgrActorTest, NotifySchedAbnormal) // test Notify worker status from other scheduler TEST_F(DomainSchedMgrActorTest, NotifyWorkerStatus) { - auto actor = std::make_shared("TestDomainSchedActor"); + auto actor = std::make_shared("TestDomainSchedActor", HEARTBEAT_TIMEOUT); auto scheduler = std::make_shared("MockDomainScheduler"); litebus::Spawn(actor); litebus::Spawn(scheduler); @@ -258,7 +261,7 @@ TEST_F(DomainSchedMgrActorTest, NotifyWorkerStatus) */ TEST_F(DomainSchedMgrActorTest, QueryAgentInfo) { - auto actor = std::make_shared("TestDomainSchedActor"); + auto actor = std::make_shared("TestDomainSchedActor", HEARTBEAT_TIMEOUT); auto scheduler = std::make_shared("MockDomainScheduler" + DOMAIN_SCHEDULER_SRV_ACTOR_NAME_POSTFIX); litebus::Spawn(actor); @@ -284,7 +287,7 @@ TEST_F(DomainSchedMgrActorTest, QueryAgentInfo) TEST_F(DomainSchedMgrActorTest, GetSchedulingQueue) { - auto actor = std::make_shared("TestDomainSchedActor"); + auto actor = std::make_shared("TestDomainSchedActor", HEARTBEAT_TIMEOUT); auto scheduler = std::make_shared("MockDomainScheduler" + DOMAIN_SCHEDULER_SRV_ACTOR_NAME_POSTFIX); litebus::Spawn(actor); @@ -312,7 +315,8 @@ TEST_F(DomainSchedMgrActorTest, GetSchedulingQueue) TEST_F(DomainSchedMgrActorTest, QueryResourcesInfo) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); litebus::Spawn(actor); auto scheduler = @@ -345,7 +349,8 @@ TEST_F(DomainSchedMgrActorTest, QueryResourcesInfo) */ TEST_F(DomainSchedMgrActorTest, ResponseScheduleWithInvalidResponse) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); actor->Init(); actor->ResponseSchedule("domainSchedA", "ResponseSchedule", ""); messages::ScheduleResponse response; @@ -363,7 +368,8 @@ TEST_F(DomainSchedMgrActorTest, ResponseScheduleWithInvalidResponse) */ TEST_F(DomainSchedMgrActorTest, ConnectFail) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); actor->DelDomainSchedCallback([](const std::string &name, const std::string &ip) { EXPECT_TRUE(name == "test"); }); litebus::Spawn(actor); @@ -385,12 +391,14 @@ TEST_F(DomainSchedMgrActorTest, ConnectFail) */ TEST_F(DomainSchedMgrActorTest, ReConnect) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); actor->DelDomainSchedCallback([](const std::string &name, const std::string &ip) { EXPECT_TRUE(name == "test"); }); litebus::Spawn(actor); litebus::Async(actor->GetAID(), &DomainSchedMgrActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); - PingPongDriver pingpong("pinged", 1000, [](const litebus::AID &aid, HeartbeatConnection type) {}); + HeartbeatClientDriver pingpong("pinged", [](const litebus::AID &aid) {}); + pingpong.Start(actor->GetAID()); litebus::Async(actor->GetAID(), &DomainSchedMgrActor::Connect, "pinged", pingpong.GetActorAID().GetIp() + ":" + std::to_string(pingpong.GetActorAID().GetPort())) .Get(); @@ -400,4 +408,5 @@ TEST_F(DomainSchedMgrActorTest, ReConnect) litebus::Terminate(actor->GetAID()); litebus::Await(actor->GetAID()); } + } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_test.cpp b/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_test.cpp index fcc8cd9bac5da455a92538af323dea8e97fc99c0..3462b997d90a6e2cb056872ea04329d61ddab992 100644 --- a/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_test.cpp +++ b/functionsystem/tests/unit/function_master/global_scheduler/scheduler_manager/domain_sched_mgr_test.cpp @@ -25,6 +25,7 @@ #include "utils/future_test_helper.h" namespace functionsystem::test { +const long long HEARTBEAT_TIMEOUT = 30000; class DomainSchedMgrTest : public ::testing::Test { public: @@ -39,9 +40,11 @@ public: static void Registered(litebus::Option topology, std::string &name, std::string &responseMsg) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); auto schedMgr = std::make_shared(actor); schedMgr->Start(); + actor->curStatus_ = MASTER_BUSINESS; auto domainScheduler = std::make_shared("MockDomainScheduler"); litebus::Spawn(domainScheduler); @@ -77,7 +80,8 @@ public: */ TEST_F(DomainSchedMgrTest, AddCallback) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); auto schedMgr = std::make_shared(actor); schedMgr->Start(); @@ -112,7 +116,8 @@ TEST_F(DomainSchedMgrTest, AddCallback) */ TEST_F(DomainSchedMgrTest, DelCallback) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); auto schedMgr = std::make_shared(actor); schedMgr->Start(); @@ -145,7 +150,8 @@ TEST_F(DomainSchedMgrTest, DelCallback) */ TEST_F(DomainSchedMgrTest, UpdateSchedTopoView) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", + HEARTBEAT_TIMEOUT); auto schedMgr = std::make_shared(actor); schedMgr->Start(); @@ -235,7 +241,7 @@ TEST_F(DomainSchedMgrTest, Registered) */ TEST_F(DomainSchedMgrTest, ScheduleWithInvalidParameters) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", HEARTBEAT_TIMEOUT); auto client = std::make_shared(actor); client->Start(); @@ -267,9 +273,10 @@ TEST_F(DomainSchedMgrTest, ScheduleWithInvalidParameters) */ TEST_F(DomainSchedMgrTest, ScheduleWithValidParameters) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", HEARTBEAT_TIMEOUT); auto client = std::make_shared(actor); client->Start(); + actor->curStatus_ = MASTER_BUSINESS; std::string domainName = "test"; auto scheduler = std::make_shared(domainName + DOMAIN_SCHEDULER_SRV_ACTOR_NAME_POSTFIX); @@ -305,9 +312,10 @@ TEST_F(DomainSchedMgrTest, ScheduleWithValidParameters) */ TEST_F(DomainSchedMgrTest, ScheduleRepeat) { - auto actor = std::make_shared("TestDomainSchedMgrActor"); + auto actor = std::make_shared("TestDomainSchedMgrActor", HEARTBEAT_TIMEOUT); auto client = std::make_shared(actor); client->Start(); + actor->curStatus_ = MASTER_BUSINESS; std::string domainName = "test"; auto scheduler = std::make_shared(domainName + DOMAIN_SCHEDULER_SRV_ACTOR_NAME_POSTFIX); diff --git a/functionsystem/tests/unit/function_master/instance_manager/family_management_test.cpp b/functionsystem/tests/unit/function_master/instance_manager/family_management_test.cpp index 1850b472e320f1dd6185d5a0900007319354e8f6..d4eb2456474919cdc06edaf0cb90f34c2ae679e3 100644 --- a/functionsystem/tests/unit/function_master/instance_manager/family_management_test.cpp +++ b/functionsystem/tests/unit/function_master/instance_manager/family_management_test.cpp @@ -19,10 +19,10 @@ #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" -#include "metadata/metadata.h" -#include "resource_type.h" +#include "common/metadata/metadata.h" +#include "common/resource_view/resource_type.h" #include "common/types/instance_state.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "function_master/instance_manager/instance_family_caches.h" #include "function_master/instance_manager/instance_manager_actor.h" #include "function_master/instance_manager/instance_manager_driver.h" @@ -181,6 +181,28 @@ TEST_F(FamilyManagementTest, GetDescendants) // NOLINT caches.RemoveInstance("D"); descendantsOfAll = caches.GetAllDescendantsOf(""); ASSERT_EQ(descendantsOfAll.size(), 6u); + EXPECT_TRUE(caches.GetInstance("") == nullptr); + EXPECT_TRUE(caches.GetInstance("NotExist") == nullptr); + EXPECT_TRUE(caches.GetInstance("C") != nullptr); } -}; // namespace functionsystem::instance_manager::test \ No newline at end of file +TEST_F(FamilyManagementTest, AddAndRemoveDetachedInstance) // NOLINT +{ + InstanceFamilyCaches caches; + auto insA = MakeInstanceInfo("A", "", "", "node001", InstanceState::RUNNING); + auto insB = MakeInstanceInfo("B", "", "A", "node001", InstanceState::RUNNING); + auto insC = MakeInstanceInfo("C", "", "B", "node001", InstanceState::RUNNING); + insC->set_detached(true); + auto insD = MakeInstanceInfo("D", "", "C", "node001", InstanceState::RUNNING); + auto insE = MakeInstanceInfo("E", "", "D", "node001", InstanceState::RUNNING); + caches.AddInstance(insA); + caches.AddInstance(insB); + caches.AddInstance(insC); + caches.AddInstance(insD); + caches.AddInstance(insE); + EXPECT_EQ(caches.GetAllDescendantsOf("A").size(), 1); + EXPECT_EQ(caches.GetAllDescendantsOf("C").size(), 2); + caches.RemoveInstance("C"); + EXPECT_EQ(caches.GetAllDescendantsOf("D").size(), 1); +} +} // namespace functionsystem::instance_manager::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/instance_manager/group_manager_test.cpp b/functionsystem/tests/unit/function_master/instance_manager/group_manager_test.cpp index 009682d387ba0da91df0ac0536d6e63fdd18b9eb..2d8de7eddffdf6dd63c4b6abfb9deae9ca74a0bc 100644 --- a/functionsystem/tests/unit/function_master/instance_manager/group_manager_test.cpp +++ b/functionsystem/tests/unit/function_master/instance_manager/group_manager_test.cpp @@ -19,10 +19,10 @@ #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" -#include "metadata/metadata.h" -#include "resource_type.h" +#include "common/metadata/metadata.h" +#include "common/resource_view/resource_type.h" #include "common/types/instance_state.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "function_master/instance_manager/instance_manager_actor.h" #include "function_master/instance_manager/instance_manager_driver.h" #include "function_proxy/local_scheduler/instance_control/instance_ctrl_actor.h" @@ -95,6 +95,9 @@ const std::string INSTANCE_ID_4 = "004"; const std::string INSTANCE_ID_5 = "005"; const std::string INSTANCE_ID_6 = "006"; +const std::string DEFAULT_FUNCTION_NAME = "123/0-yr-yr/0"; +const std::string DEFAULT_REQUEST_ID = "123456"; + std::shared_ptr MakeGroupInfo(const std::string &groupID, const std::string &ownerProxyID, const GroupState &state, const std::string &parentID) { @@ -290,7 +293,7 @@ protected: instCtrlActor3{ nullptr }; std::shared_ptr localGroupctlActor1{ nullptr }; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -298,7 +301,7 @@ protected: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -365,12 +368,30 @@ protected: ASSERT_TRUE(client->Put(GROUP_PATH_PREFIX + "/" + groupID, jsonString, {}).Get()->status.IsOk()); } + void PutGroup(std::shared_ptr &groupInfo) + { + auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); + std::string jsonString; + ASSERT_TRUE(TransToJsonFromGroupInfo(jsonString, *groupInfo)); + ASSERT_TRUE(client->Put(GROUP_PATH_PREFIX + "/" + groupInfo->groupid(), jsonString, {}).Get()->status.IsOk()); + } + void DelGroup(const std::string &groupID) { auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); ASSERT_TRUE(client->Delete(GROUP_PATH_PREFIX + "/" + groupID, {}).Get()->status.IsOk()); } + void AddRequestsToGroup(std::shared_ptr &groupInfo, + const std::vector& instanceIds) { + for (const auto& id : instanceIds) { + auto request = groupInfo->add_requests(); + request->mutable_instance()->set_instanceid(id); + request->mutable_instance()->set_function(DEFAULT_FUNCTION_NAME); + request->mutable_instance()->set_requestid(DEFAULT_REQUEST_ID); + } + } + void PutDefaultGroupsAndInstances() { // +------------------------------------------------------------+ @@ -381,8 +402,14 @@ protected: // | node3 | | inst-5 | inst-6 | // +-------+------------------+--------------------+------------+ - PutGroup(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, ""); - PutGroup(GROUP_ID_2, NODE_ID_2, GroupState::RUNNING, ""); + + auto groupInfo1 = MakeGroupInfo(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, ""); + AddRequestsToGroup(groupInfo1, {INSTANCE_ID_1, INSTANCE_ID_3, INSTANCE_ID_4}); + PutGroup(groupInfo1); + + auto groupInfo2 = MakeGroupInfo(GROUP_ID_2, NODE_ID_2, GroupState::RUNNING, ""); + AddRequestsToGroup(groupInfo2, {INSTANCE_ID_2, INSTANCE_ID_5}); + PutGroup(groupInfo2); PutInstance(INSTANCE_ID_1, GROUP_ID_1, NODE_ID_1, InstanceState::RUNNING); PutInstance(INSTANCE_ID_2, GROUP_ID_2, NODE_ID_1, InstanceState::RUNNING); @@ -392,6 +419,18 @@ protected: PutInstance(INSTANCE_ID_6, "", NODE_ID_3, InstanceState::RUNNING); } + void CheckGroupState(const std::string groupID, const GroupState &state) + { + auto groupInfoInEtcdFuture = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }) + ->Get(GROUP_PATH_PREFIX + "/" + groupID, {}); + ASSERT_AWAIT_READY(groupInfoInEtcdFuture); + ASSERT_TRUE(groupInfoInEtcdFuture.Get()->status.IsOk()); + ASSERT_TRUE(groupInfoInEtcdFuture.Get()->kvs.size() == 1); + auto info = messages::GroupInfo{}; + ASSERT_TRUE(TransToGroupInfoFromJson(info, groupInfoInEtcdFuture.Get()->kvs[0].value())); + ASSERT_EQ(info.status(), static_cast(state)); + } + GroupManagerActor::GroupCaches AsyncGetGroupCaches(std::shared_ptr groupMgrActor) { litebus::Future f = @@ -438,8 +477,6 @@ public: TEST_F(GroupManagerTest, PutAndDelGroupOK) { DEFAULT_START_INSTANCE_MANAGER_DRIVER(false); - auto mockGroupCaches = std::make_shared(); - groupMgrActor->member_->groupCaches = mockGroupCaches; auto mockInstanceMgr = std::make_shared(); EXPECT_CALL(*mockInstanceMgr, GetInstanceInfoByInstanceID) @@ -456,48 +493,31 @@ TEST_F(GroupManagerTest, PutAndDelGroupOK) { litebus::Future argGroupKey; - EXPECT_CALL(*mockGroupCaches, AddGroup).WillOnce(FutureArg<0>(&argGroupKey)); // When: group is put into metastore PutGroup(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, INSTANCE_ID_1); - // Then: add group should be called - ASSERT_AWAIT_READY(argGroupKey); - ASSERT_EQ(argGroupKey.Get(), GROUP_PATH_PREFIX + "/" + GROUP_ID_1); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->GetGroups().find(GROUP_ID_1) != groupMgrActor->member_->groupCaches->GetGroups().end(); }); } { litebus::Future argGroupKey; - EXPECT_CALL(*mockGroupCaches, AddGroup).WillOnce(FutureArg<0>(&argGroupKey)); - // When: group 2 is put into metastore PutGroup(GROUP_ID_2, NODE_ID_2, GroupState::RUNNING, INSTANCE_ID_1); - // Then: add group should be called - ASSERT_AWAIT_READY(argGroupKey); - ASSERT_EQ(argGroupKey.Get(), GROUP_PATH_PREFIX + "/" + GROUP_ID_2); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->GetGroups().find(GROUP_ID_2) != groupMgrActor->member_->groupCaches->GetGroups().end(); }); } { litebus::Future argGroupID; - EXPECT_CALL(*mockGroupCaches, RemoveGroup).WillOnce(FutureArg<0>(&argGroupID)); - // When: group 2 is delete from metastore DelGroup(GROUP_ID_1); - - // Then: - ASSERT_AWAIT_READY(argGroupID); - ASSERT_EQ(argGroupID.Get(), GROUP_ID_1); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->GetGroups().find(GROUP_ID_1) == groupMgrActor->member_->groupCaches->GetGroups().end(); }); } { litebus::Future argGroupID; - EXPECT_CALL(*mockGroupCaches, RemoveGroup).WillOnce(FutureArg<0>(&argGroupID)); - // When: group 2 is delete from metastore DelGroup(GROUP_ID_2); - - // Then: - ASSERT_AWAIT_READY(argGroupID); - ASSERT_EQ(argGroupID.Get(), GROUP_ID_2); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->GetGroups().find(GROUP_ID_2) == groupMgrActor->member_->groupCaches->GetGroups().end(); }); } DEFAULT_STOP_INSTANCE_MANAGER_DRIVER; @@ -509,10 +529,6 @@ TEST_F(GroupManagerTest, PutAndDelGroupOK) TEST_F(GroupManagerTest, PutAndDelInstanceOK) { DEFAULT_START_INSTANCE_MANAGER_DRIVER(false); - - auto mockGroupCaches = std::make_shared(); - groupMgrActor->member_->groupCaches = mockGroupCaches; - auto mockInstanceMgr = std::make_shared(); EXPECT_CALL(*mockInstanceMgr, GetInstanceInfoByInstanceID) .WillRepeatedly(testing::Invoke([](const std::string &instanceID) { @@ -525,36 +541,21 @@ TEST_F(GroupManagerTest, PutAndDelInstanceOK) // Given: 2 groups already in auto groupInfo1 = MakeGroupInfo(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, "not-exist"); auto groupInfo2 = MakeGroupInfo(GROUP_ID_2, NODE_ID_2, GroupState::RUNNING, "not-exist"); - mockGroupCaches->groups_[GROUP_ID_1] = { GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo1 }; - mockGroupCaches->nodeName2Groups_[NODE_ID_1] = { { GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo1 } }; - mockGroupCaches->groups_[GROUP_ID_2] = { GROUP_PATH_PREFIX + "/" + GROUP_ID_2, groupInfo2 }; - mockGroupCaches->nodeName2Groups_[NODE_ID_2] = { { GROUP_PATH_PREFIX + "/" + GROUP_ID_2, groupInfo2 } }; + groupMgrActor->member_->groupCaches->groups_[GROUP_ID_1] = { GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo1 }; + groupMgrActor->member_->groupCaches->nodeName2Groups_[NODE_ID_1] = { { GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo1 } }; + groupMgrActor->member_->groupCaches->groups_[GROUP_ID_2] = { GROUP_PATH_PREFIX + "/" + GROUP_ID_2, groupInfo2 }; + groupMgrActor->member_->groupCaches->nodeName2Groups_[NODE_ID_2] = { { GROUP_PATH_PREFIX + "/" + GROUP_ID_2, groupInfo2 } }; { - litebus::Future faGroupID, faInstKey; - EXPECT_CALL(*mockGroupCaches, AddGroupInstance) - .WillOnce(testing::DoAll(FutureArg<0>(&faGroupID), FutureArg<1>(&faInstKey))); - // When: put an instance PutInstance(INSTANCE_ID_1, GROUP_ID_1, NODE_ID_1, InstanceState::RUNNING); - - // Then: AddGroupInstance is called - ASSERT_AWAIT_READY(faGroupID); - ASSERT_EQ(faGroupID.Get(), GROUP_ID_1); - ASSERT_AWAIT_READY(faInstKey); - ASSERT_EQ(faInstKey.Get(), INSTANCE_PATH_PREFIX + "/123/function/0-yr-yr/version/0/defaultaz/123456/" + INSTANCE_ID_1); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->groupID2Instances_.find(GROUP_ID_1) != groupMgrActor->member_->groupCaches->groupID2Instances_.end(); }); } { - litebus::Future faInstKey; - EXPECT_CALL(*mockGroupCaches, RemoveGroupInstance).WillOnce(FutureArg<0>(&faInstKey)); - // When: delete an instance DelInstance(INSTANCE_ID_1); - - // Then: RemoveGroupInstance is called - ASSERT_AWAIT_READY(faInstKey); - ASSERT_EQ(faInstKey.Get(), INSTANCE_PATH_PREFIX + "/123/function/0-yr-yr/version/0/defaultaz/123456/" + INSTANCE_ID_1); + ASSERT_AWAIT_TRUE([=]() { return groupMgrActor->member_->groupCaches->groupID2Instances_.find(GROUP_ID_1) == groupMgrActor->member_->groupCaches->groupID2Instances_.end(); }); } DEFAULT_STOP_INSTANCE_MANAGER_DRIVER; @@ -1052,6 +1053,7 @@ TEST_F(GroupManagerTest, GroupPutWithParentAbnormal) auto clearGroupFuture = localGroupctlActor1->ExpectCallMockClearGroupResponseReturnOK()->GetFuture(); // When : put the group + // PutGroup(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, INSTANCE_ID_1); auto groupInfo = MakeGroupInfo(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, INSTANCE_ID_1); litebus::Async(groupMgrActor->GetAID(), &GroupManagerActor::OnGroupPut, GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo); @@ -1090,6 +1092,7 @@ TEST_F(GroupManagerTest, GroupPutWithParentAbnormal) .WillOnce(testing::DoAll(FutureArg<1>(&faPutValue), testing::Return(std::make_shared()))); // When : put the group + // PutGroup(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, INSTANCE_ID_1); auto groupInfo = MakeGroupInfo(GROUP_ID_1, NODE_ID_1, GroupState::RUNNING, INSTANCE_ID_2); litebus::Async(groupMgrActor->GetAID(), &GroupManagerActor::OnGroupPut, GROUP_PATH_PREFIX + "/" + GROUP_ID_1, groupInfo); @@ -1139,53 +1142,44 @@ TEST_F(GroupManagerTest, GroupInfoSyncerTest) GetLeaderInfo(groupMgrActor->GetAID())); { // for get failed - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status(StatusCode::FAILED, ""); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = groupMgrActor->GroupInfoSyncer(); + auto future = groupMgrActor->GroupInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); } { // for get response is empty - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = groupMgrActor->GroupInfoSyncer(); + auto future = groupMgrActor->GroupInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } { // for get response is empty - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = groupMgrActor->GroupInfoSyncer(); + auto future = groupMgrActor->GroupInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } { // both in etcd and cache auto key1 = R"(/yr/group/ce052e60c86d76ee00/group-6c764080-aa61-4000-8000-000024957149)"; - auto value1 = R"({"requestID":"ce052e60c86d76ee00","traceID":"job-b4465ac5-trace-X","groupID":"group-6c764080-aa61-4000-8000-000024957149","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"3abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"d8ab6100-0000-4000-801a-f4f814674753","requestID":"ce052e60c86d76ee00-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b4cbac61-0000-4000-8000-b0076050a971","requestID":"ce052e60c86d76ee00-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3aad6100-0000-4000-8018-0c3b0e297ae0","requestID":"ce052e60c86d76ee00-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"ae610000-0000-4000-bb54-2c1e5cb40d27","requestID":"ce052e60c86d76ee00-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"6282c1dc-d5af-4100-8000-0000006740f0","requestID":"ce052e60c86d76ee00-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"07b20ff7-dcb0-4100-8000-000000551b0a","requestID":"ce052e60c86d76ee00-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"d4928db1-6100-4000-8000-0081a0de67af","requestID":"ce052e60c86d76ee00-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; + auto value1 = R"({"requestID":"ce052e60c86d76ee00","traceID":"job-b4465ac5-trace-X","groupID":"group-6c764080-aa61-4000-8000-000024957149","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"3abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"d8ab6100-0000-4000-801a-f4f814674753","requestID":"ce052e60c86d76ee00-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b4cbac61-0000-4000-8000-b0076050a971","requestID":"ce052e60c86d76ee00-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3aad6100-0000-4000-8018-0c3b0e297ae0","requestID":"ce052e60c86d76ee00-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"ae610000-0000-4000-bb54-2c1e5cb40d27","requestID":"ce052e60c86d76ee00-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"6282c1dc-d5af-4100-8000-0000006740f0","requestID":"ce052e60c86d76ee00-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"07b20ff7-dcb0-4100-8000-000000551b0a","requestID":"ce052e60c86d76ee00-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"d4928db1-6100-4000-8000-0081a0de67af","requestID":"ce052e60c86d76ee00-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-6c764080-aa61-4000-8000-000024957149"},"requestID":"ce052e60c86d76ee00-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; // in etcd and not in cache auto key2 = R"(/yr/group/d9e1da12636d45e400/group-cda5051a-d278-48b3-a100-00000000000d)"; - auto value2 = R"({"requestID":"d9e1da12636d45e400","traceID":"job-b4465ac5-trace-X","groupID":"group-cda5051a-d278-48b3-a100-00000000000d","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"9abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"4eb3b461-0000-4000-8000-d2434ffd0ae2","requestID":"d9e1da12636d45e400-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"288ee2b5-6100-4000-8000-0024482c6b4d","requestID":"d9e1da12636d45e400-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"adb66100-0000-4000-809f-0d1bd179ea08","requestID":"d9e1da12636d45e400-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"36d4a234-0c2e-4761-8000-0000000042c6","requestID":"d9e1da12636d45e400-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"1fabb861-0000-4000-8000-725edf9bd3a0","requestID":"d9e1da12636d45e400-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"43b906b9-6100-4000-8000-009cc54e1076","requestID":"d9e1da12636d45e400-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"8d02ba61-0000-4000-8000-7e3fb0844dfe","requestID":"d9e1da12636d45e400-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; + auto value2 = R"({"requestID":"d9e1da12636d45e400","traceID":"job-b4465ac5-trace-X","groupID":"group-cda5051a-d278-48b3-a100-00000000000d","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"9abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"4eb3b461-0000-4000-8000-d2434ffd0ae2","requestID":"d9e1da12636d45e400-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"288ee2b5-6100-4000-8000-0024482c6b4d","requestID":"d9e1da12636d45e400-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"adb66100-0000-4000-809f-0d1bd179ea08","requestID":"d9e1da12636d45e400-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"36d4a234-0c2e-4761-8000-0000000042c6","requestID":"d9e1da12636d45e400-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"1fabb861-0000-4000-8000-725edf9bd3a0","requestID":"d9e1da12636d45e400-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"43b906b9-6100-4000-8000-009cc54e1076","requestID":"d9e1da12636d45e400-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"8d02ba61-0000-4000-8000-7e3fb0844dfe","requestID":"d9e1da12636d45e400-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-cda5051a-d278-48b3-a100-00000000000d"},"requestID":"d9e1da12636d45e400-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; // in cache and not in etcd auto key3 = R"(/yr/group/d4b532ab08a7d4d000/group-5b9f3eba-404e-48a2-a100-0000000000a3)"; - auto value3 = R"({"requestID":"d4b532ab08a7d4d000","traceID":"job-b4465ac5-trace-X","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"6abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"a3610000-0000-4000-b581-7112ee42b43b","requestID":"d4b532ab08a7d4d000-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b40e9b7a-e614-4461-8000-000000007942","requestID":"d4b532ab08a7d4d000-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"791aa563-ff30-4561-8000-0000000026a6","requestID":"d4b532ab08a7d4d000-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3e37a9b4-894e-4661-8000-00000000e7ba","requestID":"d4b532ab08a7d4d000-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b6d05947-d2a7-4100-8000-0000007c27b9","requestID":"d4b532ab08a7d4d000-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"9674a861-0000-4000-8000-ecdcb9363dd8","requestID":"d4b532ab08a7d4d000-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3d9ca0a9-6100-4000-8000-00a0ad160bce","requestID":"d4b532ab08a7d4d000-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@127.0.0.1:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; + auto value3 = R"({"requestID":"d4b532ab08a7d4d000","traceID":"job-b4465ac5-trace-X","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3","parentID":"0d810043-06a6-4000-8000-00006ac6907d","ownerProxy":"siaphisprh00132","groupOpts":{"timeout":"300","groupName":"6abcdef0008","sameRunningLifecycle":true},"requests":[{"instance":{"instanceID":"a3610000-0000-4000-b581-7112ee42b43b","requestID":"d4b532ab08a7d4d000-0","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false","DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-0","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b40e9b7a-e614-4461-8000-000000007942","requestID":"d4b532ab08a7d4d000-1","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-1","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"791aa563-ff30-4561-8000-0000000026a6","requestID":"d4b532ab08a7d4d000-2","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","DELEGATE_DIRECTORY_QUOTA":"512"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-2","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3e37a9b4-894e-4661-8000-00000000e7ba","requestID":"d4b532ab08a7d4d000-3","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456","DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-3","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"b6d05947-d2a7-4100-8000-0000007c27b9","requestID":"d4b532ab08a7d4d000-4","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":128}},"CPU":{"name":"CPU","scalar":{"value":300}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0","tenantId":"12345678901234561234567890123456"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-4","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"9674a861-0000-4000-8000-ecdcb9363dd8","requestID":"d4b532ab08a7d4d000-5","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","DATA_AFFINITY_ENABLED":"false","RecoverRetryTimes":"0"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-5","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}},{"instance":{"instanceID":"3d9ca0a9-6100-4000-8000-00a0ad160bce","requestID":"d4b532ab08a7d4d000-6","function":"12345678901234561234567890123456/0-yr10882-yr-gangschedule/$latest","resources":{"resources":{"CPU":{"name":"CPU","scalar":{"value":300}},"Memory":{"name":"Memory","scalar":{"value":128}}}},"scheduleOption":{"affinity":{"instanceAffinity":{},"resource":{},"instance":{"topologyKey":"agent"}},"extension":{"DELEGATE_DIRECTORY_QUOTA":"512"},"range":{}},"createOptions":{"DELEGATE_DIRECTORY_QUOTA":"512","tenantId":"12345678901234561234567890123456","RecoverRetryTimes":"0","DATA_AFFINITY_ENABLED":"false"},"instanceStatus":{"code":1,"msg":"new instance"},"jobID":"job-b4465ac5","parentID":"0d810043-06a6-4000-8000-00006ac6907d","parentFunctionProxyAID":"siaphisprh00132-LocalSchedInstanceCtrlActor@7.189.31.67:22772","storageType":"s3","scheduleTimes":1,"deployTimes":1,"args":[{"value":"AAAA"},{"value":"AAAAAAAAAAAAAAAAAAAAAAE="}],"gracefulShutdownTime":"-1","tenantID":"12345678901234561234567890123456","groupID":"group-5b9f3eba-404e-48a2-a100-0000000000a3"},"requestID":"d4b532ab08a7d4d000-6","traceID":"job-b4465ac5-trace-X","contexts":{"LabelAffinityScorePlugin":{"preferredAffinityCtx":{}}}}],"status":2})"; auto group1 = std::make_shared(); ASSERT_TRUE(TransToGroupInfoFromJson(*group1, value1)); @@ -1213,15 +1207,12 @@ TEST_F(GroupManagerTest, GroupInfoSyncerTest) groupKv1.set_key(key2) ; groupKv1.set_value(value2); - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); rep->kvs.emplace_back(groupKv1); rep->kvs.emplace_back(groupKv2); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = groupMgrActor->GroupInfoSyncer(); + auto future = groupMgrActor->GroupInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); @@ -1234,4 +1225,158 @@ TEST_F(GroupManagerTest, GroupInfoSyncerTest) DEFAULT_STOP_INSTANCE_MANAGER_DRIVER; } + +TEST_F(GroupManagerTest, OnInstancePutTest) +{ + auto mockGlobalScheduler = std::make_shared(); + auto groupCaches = std::make_shared(); + auto member = std::make_shared(); + member->groupCaches = groupCaches; + member->globalScheduler = mockGlobalScheduler; + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + EXPECT_CALL(*mockGlobalScheduler, GetLocalAddress) + .WillOnce(testing::Return(litebus::Option("127.0.0.1:" + std::to_string(port)))); + YRLOG_INFO("port:{} instCtrlActor1:{}", std::to_string(port), std::string(instCtrlActor1->GetAID())); + auto mockMetaClient = std::make_shared(metaStoreServerHost_); + auto groupMgrActor = std::make_shared(mockMetaClient, mockGlobalScheduler); + litebus::Spawn(groupMgrActor); + auto masterBusiness = std::make_shared(member, groupMgrActor); + auto info = MakeInstanceInfo(INSTANCE_ID_1, GROUP_ID_1, NODE_ID_1, InstanceState::RUNNING); + auto instanceKey = INSTANCE_PATH_PREFIX + "/" + INSTANCE_ID_1; + masterBusiness->member_->groupCaches->AddGroup(GROUP_KEY_1, MakeGroupInfo(GROUP_ID_1, NODE_ID_1, + GroupState::FAILED, "--")); + masterBusiness->member_->groupCaches->AddGroupInstance(info->groupid(), instanceKey, info); + + auto future = masterBusiness->OnInstancePut(instanceKey, info); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().IsOk(), true); + litebus::Terminate(groupMgrActor->GetAID()); + litebus::Await(groupMgrActor->GetAID()); +} + +TEST_F(GroupManagerTest, OnChangeTest) +{ + auto mockGlobalScheduler = std::make_shared(); + auto groupCaches = std::make_shared(); + auto member = std::make_shared(); + member->groupCaches = groupCaches; + member->globalScheduler = mockGlobalScheduler; + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + EXPECT_CALL(*mockGlobalScheduler, GetLocalAddress) + .WillOnce(testing::Return(litebus::Option("127.0.0.1:" + std::to_string(port)))); + + auto mockForwardCustomSignalReceived = std::make_shared>(); + EXPECT_CALL(*instCtrlActor1, MockForwardCustomSignalResponse) + .WillRepeatedly(testing::Invoke([mockForwardCustomSignalReceived]( + const litebus::AID &from, const std::string &name, const std::string &msg) { + internal::ForwardKillRequest fkReq; + fkReq.ParseFromString(msg); + mockForwardCustomSignalReceived->Set(fkReq); + internal::ForwardKillResponse fkRsp; + fkRsp.set_requestid(fkReq.requestid()); + return std::make_pair(true, fkRsp); + })); + auto mockMetaClient = std::make_shared(metaStoreServerHost_); + auto groupMgrActor = std::make_shared(mockMetaClient, mockGlobalScheduler); + litebus::Spawn(groupMgrActor); + auto masterBusiness = std::make_shared(member, groupMgrActor); + auto info = MakeInstanceInfo(INSTANCE_ID_1, GROUP_ID_1, NODE_ID_1, InstanceState::RUNNING); + auto instanceKey = INSTANCE_PATH_PREFIX + "/" + INSTANCE_ID_1; + masterBusiness->member_->groupCaches->AddGroup(GROUP_KEY_1, MakeGroupInfo(GROUP_ID_1, NODE_ID_1, + GroupState::FAILED, "--")); + masterBusiness->member_->groupCaches->AddGroupInstance(info->groupid(), instanceKey, info); + + masterBusiness->OnChange(); + ASSERT_AWAIT_READY(mockForwardCustomSignalReceived->GetFuture()); + litebus::Terminate(groupMgrActor->GetAID()); + litebus::Await(groupMgrActor->GetAID()); +} + +// Checks if the master verifies group instance consistency after switching from slave to master upon instance deletion. +TEST_F(GroupManagerTest, VerifyGroupInstanceConsistencyAfterSlaveDeletionEvent) { + DEFAULT_START_INSTANCE_MANAGER_DRIVER(false); + + auto mockInstanceMgr = std::make_shared(); + EXPECT_CALL(*mockInstanceMgr, GetInstanceInfoByInstanceID) + .WillRepeatedly(testing::Invoke([](const std::string &instanceID) { + auto inst = std::make_shared(); + inst->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + return std::make_pair("", inst); + })); + groupMgrActor->BindInstanceManager(mockInstanceMgr); + + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + EXPECT_CALL(*scheduler, GetLocalAddress) + .WillRepeatedly(testing::Return(litebus::Option("127.0.0.1:" + std::to_string(port)))); + auto mockForwardCustomSignalReceived = std::make_shared>(); + EXPECT_CALL(*instCtrlActor2, MockForwardCustomSignalResponse) + .Times(2) + .WillRepeatedly(testing::Invoke([mockForwardCustomSignalReceived]( + const litebus::AID &from, const std::string &name, const std::string &msg) { + internal::ForwardKillRequest fkReq; + fkReq.ParseFromString(msg); + mockForwardCustomSignalReceived->Set(fkReq); + return std::make_pair(true, internal::ForwardKillResponse()); + })); + + PutDefaultGroupsAndInstances(); + DelInstance(INSTANCE_ID_1); + + litebus::Async(groupMgrActor->GetAID(), &GroupManagerActor::UpdateLeaderInfo, + GetLeaderInfo(groupMgrActor->GetAID())); + + ASSERT_AWAIT_READY(mockForwardCustomSignalReceived->GetFuture()); + CheckGroupState(GROUP_ID_1, GroupState::FAILED); + CheckGroupState(GROUP_ID_2, GroupState::RUNNING); + + DEFAULT_STOP_INSTANCE_MANAGER_DRIVER; +} + +// Validates whether the master ensures group instance consistency after crash recovery, +// especially for deleted instances. +TEST_F(GroupManagerTest, VerifyGroupInstanceConsistencyAfterCrashRecovery) { + PutDefaultGroupsAndInstances(); + DelInstance(INSTANCE_ID_1); + DEFAULT_START_INSTANCE_MANAGER_DRIVER(false); + + auto mockInstanceMgr = std::make_shared(); + EXPECT_CALL(*mockInstanceMgr, GetInstanceInfoByInstanceID) + .WillRepeatedly(testing::Invoke([](const std::string &instanceID) { + auto inst = std::make_shared(); + inst->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + return std::make_pair("", inst); + })); + groupMgrActor->BindInstanceManager(mockInstanceMgr); + + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + EXPECT_CALL(*scheduler, GetLocalAddress) + .WillRepeatedly(testing::Return(litebus::Option("127.0.0.1:" + std::to_string(port)))); + auto mockForwardCustomSignalReceived = std::make_shared>(); + EXPECT_CALL(*instCtrlActor2, MockForwardCustomSignalResponse) + .Times(2) + .WillRepeatedly(testing::Invoke([mockForwardCustomSignalReceived]( + const litebus::AID &from, const std::string &name, const std::string &msg) { + internal::ForwardKillRequest fkReq; + fkReq.ParseFromString(msg); + mockForwardCustomSignalReceived->Set(fkReq); + return std::make_pair(true, internal::ForwardKillResponse()); + })); + + auto start = std::chrono::steady_clock::now(); + while (groupMgrActor->member_->groupCaches->GetGroups().size() != 2) { + if (std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) { + EXPECT_TRUE(false) << "Timeout after 5 seconds"; + } + std::this_thread::yield(); + } + + litebus::Async(groupMgrActor->GetAID(), &GroupManagerActor::UpdateLeaderInfo, + GetLeaderInfo(groupMgrActor->GetAID())); + + ASSERT_AWAIT_READY(mockForwardCustomSignalReceived->GetFuture()); + CheckGroupState(GROUP_ID_1, GroupState::FAILED); + CheckGroupState(GROUP_ID_2, GroupState::RUNNING); + + DEFAULT_STOP_INSTANCE_MANAGER_DRIVER; +} }; // namespace functionsystem::instance_manager::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/instance_manager/instance_manager_test.cpp b/functionsystem/tests/unit/function_master/instance_manager/instance_manager_test.cpp index ea6e72c77168a7c5fb3be43cb7edbc9482ff38ed..c4367f4ca494685b8421fa5b1f6cfc592e38b81f 100644 --- a/functionsystem/tests/unit/function_master/instance_manager/instance_manager_test.cpp +++ b/functionsystem/tests/unit/function_master/instance_manager/instance_manager_test.cpp @@ -23,10 +23,10 @@ #define protected public #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" -#include "metadata/metadata.h" +#include "common/metadata/metadata.h" #include "common/types/instance_state.h" #include "common/utils/generate_message.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "common/utils/struct_transfer.h" #include "function_master/instance_manager/group_manager.h" #include "function_master/instance_manager/instance_manager_actor.h" @@ -127,7 +127,7 @@ protected: std::shared_ptr mockInstCtrlActorNode01_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -137,7 +137,7 @@ protected: localAddress_ = "127.0.0.1:" + std::to_string(port); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -271,6 +271,19 @@ protected: return instanceMgrActor->member_; } + static void WaitSyncFuncMeta(const std::shared_ptr mockMetaStoreClient, + const std::shared_ptr instanceMgrActor) + { + instanceMgrActor->SetInstancesReady(); + std::shared_ptr emptyResp = std::make_shared(); + emptyResp->status = Status::OK(); + bool isMetaSynced = false; + EXPECT_CALL(*mockMetaStoreClient, Get(FUNC_META_PATH_PREFIX, testing::_)) + .WillOnce(testing::DoAll(testing::Assign(&isMetaSynced, true), testing::Return(emptyResp))); + litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::UpdateLeaderInfo, + GetLeaderInfo(instanceMgrActor->GetAID())); + EXPECT_AWAIT_TRUE([&isMetaSynced]() -> bool { return isMetaSynced; }); + } }; TEST_F(InstanceManagerTest, SyncInstance) // NOLINT @@ -307,6 +320,33 @@ TEST_F(InstanceManagerTest, SyncInstance) // NOLINT instanceMgrDriver->Await(); } +TEST_F(InstanceManagerTest, StartSyncFail) +{ + auto scheduler = std::make_shared(); + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + auto groupMgrActor = std::make_shared(mockMetaStoreClient, scheduler); + groupMgrActor->isSuicide_ = true; + auto groupMgr = std::make_shared(groupMgrActor); + std::shared_ptr rep = std::make_shared(); + rep->status = Status(StatusCode::FAILED, ""); + std::shared_ptr rep1 = std::make_shared(); + rep1->status = Status::OK(); + rep1->header.revision = INT64_MAX; + EXPECT_CALL(*mockMetaStoreClient, + GetAndWatchWithHandler(testing::_, testing::_, testing::_, testing::_, testing::_)) + .WillRepeatedly(testing::Return(litebus::Future>())); + auto instanceMgrActor = std::make_shared( + mockMetaStoreClient, scheduler, groupMgr, + InstanceManagerStartParam{ .runtimeRecoverEnable = false }); + instanceMgrActor->isSuicide_ = true; + auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); + instanceMgrDriver->Start(); + EXPECT_TRUE(instanceMgrActor->CheckSyncResponse(rep).Get().IsError()); + EXPECT_TRUE(instanceMgrActor->CheckSyncResponse(rep1).Get().IsError()); + instanceMgrDriver->Stop(); + instanceMgrDriver->Await(); +} + TEST_F(InstanceManagerTest, SchedulerWatchTest) // NOLINT { auto scheduler = std::make_shared(); @@ -334,6 +374,42 @@ TEST_F(InstanceManagerTest, SchedulerWatchTest) // NOLINT litebus::Await(aid); } +TEST_F(InstanceManagerTest, ReplayFailedDeleteOperationTest) // NOLINT +{ + auto scheduler = std::make_shared(); + + auto client = MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }); + auto groupMgrActor = std::make_shared(client, scheduler); + auto groupMgr = std::make_shared(groupMgrActor); + + const auto actor = std::make_shared( + client, scheduler, groupMgr, InstanceManagerStartParam{ .runtimeRecoverEnable = false }); + auto instanceMgrDriver = std::make_shared(actor, groupMgrActor); + instanceMgrDriver->Start(); + litebus::Async(actor->GetAID(), &InstanceManagerActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); + + auto mockInstanceOpt = std::make_shared(); + actor->member_->instanceOpt = mockInstanceOpt; + + std::list> futures; + auto eraseDelKeys = std::make_shared>(); + actor->member_->operateCacher->AddDeleteEvent( + INSTANCE_PATH_PREFIX, + "/sn/instance/business/yrk/tenant/0/function/0-system-faasscheduler/version/$latest/defaultaz/" + "38bb25d0ccc79faa00/e19cf1ed-2455-43eb-a8a0-6c6083cdee2b"); + litebus::Future future; + EXPECT_CALL(*mockInstanceOpt, ForceDelete) + .WillOnce(testing::DoAll( + testing::Invoke([&future](std::shared_ptr, std::shared_ptr routeInfo, + std::shared_ptr, bool) { future.SetValue(*routeInfo); }), + testing::Return(OperateResult{ Status::OK(), "", 3 }))); + actor->ReplayFailedDeleteOperation(futures, eraseDelKeys); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().key, "/yr/route/business/yrk/e19cf1ed-2455-43eb-a8a0-6c6083cdee2b"); + instanceMgrDriver->Stop(); + instanceMgrDriver->Await(); +} + TEST_F(InstanceManagerTest, SyncAbnormalScheduler) // NOLINT { PutInstances(); @@ -412,7 +488,6 @@ TEST_F(InstanceManagerTest, PutAndDeleteInstance) // NOLINT client.Init(); // eg. /sn/instance/business/yrk/tenant/0/function/0-test-0/version/.. ASSERT_TRUE(client.Put(instance003.instanceid(), jsonString, {}).Get()->status.IsOk()); - functionsystem::instance_manager::InstanceManagerMap map; ASSERT_AWAIT_TRUE([&]() -> bool { map.clear(); // [notice] clear and then Get @@ -578,9 +653,11 @@ TEST_F(InstanceManagerTest, OnLocalSchedulerFaultRecover) // NOLINT return response->kvs.size() == 1; }); - map.clear(); // [notice] clear and then Get - litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::Get, INSTANCE_MANAGER_OWNER, &map).Get(); - EXPECT_EQ(map.size(), 2u); + EXPECT_AWAIT_TRUE([&]() -> bool { + map.clear(); // [notice] clear and then Get + litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::Get, INSTANCE_MANAGER_OWNER, &map).Get(); + return map.size() == 2; + }); for (const auto &iterator : map) { EXPECT_TRUE(iterator.second->instancestatus().code() == static_cast(InstanceState::SCHEDULING)); } @@ -825,8 +902,16 @@ TEST_F(InstanceManagerTest, OnChange) // NOLINT ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString003, instance003)); ASSERT_TRUE(client.Put(instance003.instanceid(), jsonString003, {}).Get()->status.IsOk()); + std::string jsonString004; + resource_view::InstanceInfo instance004 = CreateInstance(INSTANCE_PATH_PREFIX + "/004", true); + instance004.set_functionproxyid("fake_node"); + instance004.set_parentid("frontendParent"); + (*instance004.mutable_extensions())["source"] = "frontend"; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString004, instance004)); + ASSERT_TRUE(client.Put(instance004.instanceid(), jsonString004, {}).Get()->status.IsOk()); + ASSERT_AWAIT_TRUE( - [&]() -> bool { return client.Get(INSTANCE_PATH_PREFIX, { .prefix = true }).Get()->kvs.size() == 3; }); + [&]() -> bool { return client.Get(INSTANCE_PATH_PREFIX, { .prefix = true }).Get()->kvs.size() == 4; }); ASSERT_TRUE(client.Put(KEY_ABNORMAL_SCHEDULER_PREFIX + NODE_ID_1, NODE_ID_1, {}).Get()->status.IsOk()); ASSERT_AWAIT_TRUE([&]() -> bool { @@ -977,15 +1062,19 @@ TEST_F(InstanceManagerTest, FamilyManagement_OnParentMissingInstancePut) // NOL // When : put an instance with an non-existing instance auto instA = MakeInstanceInfo("A", "", "X", NODE_ID_1, InstanceState::RUNNING); + instA->set_detached(true); + auto instB = MakeInstanceInfo("B", "", "X", NODE_ID_1, InstanceState::RUNNING); litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::OnInstancePut, INSTANCE_PATH_PREFIX + "/" + instA->instanceid(), instA); + litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::OnInstancePut, + INSTANCE_PATH_PREFIX + "/" + instB->instanceid(), instB); // Then : expect instance C is killed ASSERT_AWAIT_READY(sigArg); internal::ForwardKillRequest killReq; ASSERT_TRUE(killReq.ParseFromString(sigArg.Get())); ASSERT_EQ(killReq.req().signal(), SHUT_DOWN_SIGNAL); - ASSERT_EQ(killReq.req().instanceid(), instA->instanceid()); + ASSERT_EQ(killReq.req().instanceid(), instB->instanceid()); // Then : expect group manager get the message ASSERT_AWAIT_READY(putGroupArg); @@ -1439,7 +1528,7 @@ TEST_F(InstanceManagerTest, JobKillTest) // NOLINT instanceMgrDriver->Await(); } -TEST_F(InstanceManagerTest, putProxyAbnormalFailed) // NOLINT +TEST_F(InstanceManagerTest, PutProxyAbnormalFailed) // NOLINT { auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, QueryNodes).WillOnce(::testing::Return(NODES)); @@ -1458,8 +1547,9 @@ TEST_F(InstanceManagerTest, putProxyAbnormalFailed) // NOLINT GetLeaderInfo(instanceMgrActor->GetAID())); std::shared_ptr rep = std::make_shared(); - rep->status = Status(StatusCode::FAILED, "");; - EXPECT_CALL(*mockMetaStoreClient, Put).WillRepeatedly(testing::Return(litebus::Future>(rep))); + rep->status = Status(StatusCode::FAILED, ""); + EXPECT_CALL(*mockMetaStoreClient, Put) + .WillRepeatedly(testing::Return(litebus::Future>(rep))); auto future = litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::OnLocalSchedFault, NODE_ID_1); ASSERT_AWAIT_READY(future); @@ -1488,25 +1578,19 @@ TEST_F(InstanceManagerTest, ProxyAbnormalSyncerTest) // NOLINT GetLeaderInfo(instanceMgrActor->GetAID())); { // for get failed - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status(StatusCode::FAILED, ""); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = instanceMgrActor->ProxyAbnormalSyncer(); + auto future = instanceMgrActor->ProxyAbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); } { // for get response is empty - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = instanceMgrActor->ProxyAbnormalSyncer(); + auto future = instanceMgrActor->ProxyAbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } @@ -1516,14 +1600,11 @@ TEST_F(InstanceManagerTest, ProxyAbnormalSyncerTest) // NOLINT getKeyValue.set_key(KEY_ABNORMAL_SCHEDULER_PREFIX+"Node1") ; getKeyValue.set_value("Node1"); - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); rep->kvs.emplace_back(getKeyValue); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = instanceMgrActor->ProxyAbnormalSyncer(); + auto future = instanceMgrActor->ProxyAbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } @@ -1548,8 +1629,7 @@ TEST_F(InstanceManagerTest, FunctionMetaSyncerTest) // NOLINT auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); instanceMgrDriver->Start(); - litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::UpdateLeaderInfo, - GetLeaderInfo(instanceMgrActor->GetAID())); + WaitSyncFuncMeta(mockMetaStoreClient, instanceMgrActor); { // for get failed litebus::Future> getResponseFuture; @@ -1601,21 +1681,27 @@ TEST_F(InstanceManagerTest, FunctionMetaSyncerTest) // NOLINT instanceInfoA.set_jobid("job-1"); auto keyA = GenInstanceKey("123/helloworldA/$latest", instanceIDA, instanceIDA).Get(); std::string instanceIDB = "instanceB"; - auto instanceInfoB = GenInstanceInfo(instanceIDB, "funcAgent", "12345678901234561234567890123456/0-defaultservice-default/$latest", instanceStatusA); + auto instanceInfoB = + GenInstanceInfo(instanceIDB, "funcAgent", + "12345678901234561234567890123456/0-defaultservice-default/$latest", instanceStatusA); instanceInfoB.set_functionproxyid(NODE_ID_1); instanceInfoB.set_jobid("job-1"); - auto keyB = GenInstanceKey("12345678901234561234567890123456/0-defaultservice-default/$latest", instanceIDA, instanceIDA).Get(); + auto keyB = GenInstanceKey("12345678901234561234567890123456/0-defaultservice-default/$latest", instanceIDA, + instanceIDA) + .Get(); instanceMgrActor->OnInstancePut(keyA, std::make_shared(instanceInfoA)); instanceMgrActor->OnInstancePut(keyB, std::make_shared(instanceInfoB)); ASSERT_AWAIT_TRUE([&]() { return instanceMgrActor->member_->jobID2InstanceIDs["job-1"].size() == 2; }); - ASSERT_AWAIT_TRUE([&]() { return instanceMgrActor->member_->funcMeta2InstanceIDs["123/helloworldA/$latest"].size() == 1; }); + ASSERT_AWAIT_TRUE( + [&]() { return instanceMgrActor->member_->funcMeta2InstanceIDs["123/helloworldA/$latest"].size() == 1; }); litebus::Future sigArg1; EXPECT_CALL(*mockInstCtrlActorNode01_, MockForwardCustomSignalRequest) - .WillOnce(testing::DoAll(FutureArg<2>(&sigArg1), - testing::Return(std::make_pair( - true, GenForwardKillResponse("requestID0", common::ErrorCode::ERR_NONE, "ok"))))); + .WillOnce( + testing::DoAll(FutureArg<2>(&sigArg1), + testing::Return(std::make_pair( + true, GenForwardKillResponse("requestID0", common::ErrorCode::ERR_NONE, "ok"))))); EXPECT_CALL(*scheduler, GetLocalAddress(NODE_ID_1)) .WillOnce(testing::Return(litebus::Option(localAddress_))); @@ -1651,8 +1737,7 @@ TEST_F(InstanceManagerTest, InstanceInfoSyncerTest) // NOLINT auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); instanceMgrDriver->Start(); - litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::UpdateLeaderInfo, - GetLeaderInfo(instanceMgrActor->GetAID())); + WaitSyncFuncMeta(mockMetaStoreClient, instanceMgrActor); { // for get failed litebus::Future> getResponseFuture; @@ -1680,16 +1765,16 @@ TEST_F(InstanceManagerTest, InstanceInfoSyncerTest) // NOLINT { auto instanceKey1 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/d4f050f90ee2b90b00/609d910b-f65d-4efc-8000-000000000046)"; - auto instanceInfoJson1 = R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"127.0.0.1:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + auto instanceInfoJson1 = R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"10.42.2.100:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; auto instance1 = std::make_shared(); ASSERT_TRUE(TransToInstanceInfoFromJson(*instance1, instanceInfoJson1)); auto instanceKey2 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/xxxxxxxx999/aaaaa88888)"; - auto instanceInfoJson2 = R"({"instanceID":"aaaaa88888","requestID":"xxxxxxxx999","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"127.0.0.1:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + auto instanceInfoJson2 = R"({"instanceID":"aaaaa88888","requestID":"xxxxxxxx999","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"10.42.2.100:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; InstanceInfo instance2; ASSERT_TRUE(TransToInstanceInfoFromJson(instance2, instanceInfoJson2)); - auto instanceInfoJson2forRunning = R"({"instanceID":"aaaaa88888","requestID":"xxxxxxxx999","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"127.0.0.1:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","instanceStatus":{"code":3,"msg":"running"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"3"})"; + auto instanceInfoJson2forRunning = R"({"instanceID":"aaaaa88888","requestID":"xxxxxxxx999","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"10.42.2.100:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","instanceStatus":{"code":3,"msg":"running"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"3"})"; auto instanceRunning = std::make_shared(); ASSERT_TRUE(TransToInstanceInfoFromJson(*instanceRunning, instanceInfoJson2forRunning)); @@ -1745,8 +1830,7 @@ TEST_F(InstanceManagerTest, InstanceInfoSyncerOperationReplayTest) // NOLINT auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); instanceMgrDriver->Start(); - litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::UpdateLeaderInfo, - GetLeaderInfo(instanceMgrActor->GetAID())); + WaitSyncFuncMeta(mockMetaStoreClient, instanceMgrActor); auto mockInstanceOpt = std::make_shared(); instanceMgrActor->member_->instanceOpt = mockInstanceOpt; @@ -1769,7 +1853,7 @@ TEST_F(InstanceManagerTest, InstanceInfoSyncerOperationReplayTest) // NOLINT auto instanceKey1 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/d4f050f90ee2b90b00/609d910b-f65d-4efc-8000-000000000046)"; auto instanceInfoJson1 = - R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"127.0.0.1:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"10.42.2.100:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; std::shared_ptr instance1 = std::make_shared(); ASSERT_TRUE(TransToInstanceInfoFromJson(*instance1, instanceInfoJson1)); @@ -2018,7 +2102,7 @@ TEST_F(InstanceManagerTest, CompleteKillInstance) auto instanceKey1 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/d4f050f90ee2b90b00/609d910b-f65d-4efc-8000-000000000046)"; auto instanceInfoJson1 = - R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"127.0.0.1:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + R"({"instanceID":"609d910b-f65d-4efc-8000-000000000046","requestID":"d4f050f90ee2b90b00","runtimeID":"runtime-6de59705-0000-4000-8000-00abf61502f6","runtimeAddress":"10.42.2.100:22771","functionAgentID":"functionagent-pool1-776c6db574-nnmrn","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; std::shared_ptr instance1 = std::make_shared(); ASSERT_TRUE(TransToInstanceInfoFromJson(*instance1, instanceInfoJson1)); @@ -2114,4 +2198,194 @@ TEST_F(InstanceManagerTest, NodesTest) instanceMgrDriver->Await(); } +TEST_F(InstanceManagerTest, ExecuteTest) +{ + auto scheduler = std::make_shared(); + auto groupMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler); + auto groupMgr = std::make_shared(groupMgrActor); + + auto instanceMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler, groupMgr, + InstanceManagerStartParam{ .runtimeRecoverEnable = true }); + litebus::Spawn(instanceMgrActor); + auto count = std::make_shared(); + *count = 0; + auto fn = [count]() { + (*count)++; + return Status::OK(); + }; + auto future1 = litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::Execute, fn); + auto future2 = litebus::Async(instanceMgrActor->GetAID(), &InstanceManagerActor::Execute, fn); + EXPECT_AWAIT_READY(future1); + EXPECT_AWAIT_READY(future2); + EXPECT_EQ(*count, 2); + litebus::Terminate(instanceMgrActor->GetAID()); + litebus::Await(instanceMgrActor->GetAID()); +} + +TEST_F(InstanceManagerTest, ForwardQueryDebugInstancesInfo) +{ + auto scheduler = std::make_shared(); + auto groupMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler); + auto groupMgr = std::make_shared(groupMgrActor); + + auto instanceMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler, groupMgr, + InstanceManagerStartParam{ .runtimeRecoverEnable = true }); + auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); + instanceMgrDriver->Start(); + { + std::string empty = ""; + std::string name = "ForwardQueryDebugInstancesInfoHandler"; + instanceMgrActor->ForwardQueryDebugInstancesInfoHandler(litebus::AID(), std::move(name), std::move(empty)); + } + + { + messages::QueryDebugInstanceInfosRequest req; + std::string name = "ForwardQueryDebugInstancesInfoHandler"; + instanceMgrActor->ForwardQueryDebugInstancesInfoHandler(litebus::AID(), std::move(name), req.SerializeAsString()); + } + { + auto future = litebus::Future(); + future.SetFailed(StatusCode::FAILED); + instanceMgrActor->OnQueryDebugInstancesInfoFinished(litebus::AID(), future); + } + + { + std::string empty = ""; + std::string name = "ForwardQueryDebugInstancesInfoHandler"; + instanceMgrActor->ForwardQueryDebugInstancesInfoResponseHandler(litebus::AID(), std::move(name), std::move(empty)); + } + { + auto promise = std::make_shared>(); + instanceMgrActor->member_->queryDebugInstancesPromise = promise; + messages::QueryDebugInstanceInfosResponse rsp; + std::string name = "ForwardQueryDebugInstancesInfoHandler"; + instanceMgrActor->ForwardQueryDebugInstancesInfoResponseHandler(litebus::AID(), std::move(name), rsp.SerializeAsString()); + EXPECT_AWAIT_READY(promise->GetFuture()); + EXPECT_EQ(instanceMgrActor->member_->queryDebugInstancesPromise, nullptr); + } + instanceMgrDriver->Stop(); + instanceMgrDriver->Await(); +} + +TEST_F(InstanceManagerTest, ForwardQueryInstancesInfo) +{ + auto scheduler = std::make_shared(); + auto groupMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler); + auto groupMgr = std::make_shared(groupMgrActor); + + auto instanceMgrActor = std::make_shared( + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }), scheduler, groupMgr, + InstanceManagerStartParam{ .runtimeRecoverEnable = true }); + auto instanceMgrDriver = std::make_shared(instanceMgrActor, groupMgrActor); + instanceMgrDriver->Start(); + { + std::string empty = ""; + std::string name = "ForwardQueryInstancesInfoHandler"; + instanceMgrActor->ForwardQueryInstancesInfoHandler(litebus::AID(), std::move(name), std::move(empty)); + } + + { + messages::QueryInstancesInfoRequest req; + std::string name = "ForwardQueryInstancesInfoHandler"; + instanceMgrActor->ForwardQueryInstancesInfoHandler(litebus::AID(), std::move(name), req.SerializeAsString()); + } + { + auto future = litebus::Future(); + future.SetFailed(StatusCode::FAILED); + instanceMgrActor->OnQueryInstancesInfoFinished(litebus::AID(), future); + } + + { + std::string empty = ""; + std::string name = "ForwardQueryDebugInstancesInfoHandler"; + instanceMgrActor->ForwardQueryInstancesInfoResponseHandler(litebus::AID(), std::move(name), std::move(empty)); + } + { + auto promise = std::make_shared>(); + instanceMgrActor->member_->queryInstancesPromise = promise; + messages::QueryInstancesInfoResponse rsp; + std::string name = "ForwardQueryInstancesInfoResponseHandler"; + instanceMgrActor->ForwardQueryInstancesInfoResponseHandler(litebus::AID(), std::move(name), rsp.SerializeAsString()); + EXPECT_AWAIT_READY(promise->GetFuture()); + EXPECT_EQ(instanceMgrActor->member_->queryInstancesPromise, nullptr); + } + instanceMgrDriver->Stop(); + instanceMgrDriver->Await(); +} + +TEST_F(InstanceManagerTest, IsInstanceShouldBeKilledTest) +{ + auto member = std::make_shared(); + auto instanceMgrActor = std::make_shared( + nullptr, nullptr, nullptr, InstanceManagerStartParam{ .runtimeRecoverEnable = true }); + auto masterBusiness = std::make_shared(member, instanceMgrActor); + masterBusiness->member_->family = std::make_shared(); + std::string instanceIDA = "instanceA"; + InstanceState instanceStatusA = InstanceState::RUNNING; + std::shared_ptr instanceInfoA = std::make_shared(); + instanceInfoA->set_instanceid("instanceA"); + instanceInfoA->mutable_instancestatus()->set_code(int32_t(InstanceState::RUNNING)); + instanceInfoA->set_lowreliability(true); + instanceInfoA->set_parentid("123"); + EXPECT_TRUE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + instanceInfoA->mutable_instancestatus()->set_code(int32_t(InstanceState::FATAL)); + EXPECT_TRUE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + instanceInfoA->set_lowreliability(false); + instanceInfoA->set_detached(true); + EXPECT_FALSE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + + instanceInfoA->mutable_instancestatus()->set_code(int32_t(InstanceState::RUNNING)); + instanceInfoA->set_detached(false); + EXPECT_TRUE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + (*instanceInfoA->mutable_extensions())["source"] = "frontend"; + EXPECT_FALSE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + (*instanceInfoA->mutable_extensions())["source"] = ""; + (*instanceInfoA->mutable_createoptions())["resource.owner"] = "static_function"; + EXPECT_FALSE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); + masterBusiness->member_->exitingInstances.insert("123"); + EXPECT_TRUE(masterBusiness->IsInstanceShouldBeKilled(instanceInfoA)); +} + +TEST_F(InstanceManagerTest, IsInstanceManagedByJobTest) +{ + auto member = std::make_shared(); + auto instanceMgrActor = std::make_shared( + nullptr, nullptr, nullptr, InstanceManagerStartParam{ .runtimeRecoverEnable = true }); + instanceMgrActor->member_->family = std::make_shared(); + std::string instanceIDA = "instanceA"; + InstanceState instanceStatusA = InstanceState::RUNNING; + std::shared_ptr instanceInfoA = std::make_shared(); + instanceInfoA->set_instanceid("instanceA"); + instanceInfoA->mutable_instancestatus()->set_code(int32_t(InstanceState::RUNNING)); + EXPECT_FALSE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + instanceInfoA->set_jobid("job-123"); + instanceInfoA->set_detached(true); + EXPECT_FALSE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + instanceInfoA->set_detached(false); + instanceInfoA->set_parentid("instanceB"); + EXPECT_TRUE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + std::shared_ptr instanceInfoB = std::make_shared(); + instanceInfoB->set_instanceid("instanceB"); + instanceInfoB->set_function("0/0-system-faasfrontend/$latest"); + instanceMgrActor->member_->family->AddInstance(instanceInfoB); + EXPECT_TRUE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + instanceInfoB->set_detached(true); + instanceInfoB->set_function("0/hello/$latest"); + EXPECT_FALSE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + std::shared_ptr instanceInfoC = std::make_shared(); + instanceInfoC->set_instanceid("instanceC"); + instanceInfoC->set_function("0/hello$latest"); + instanceInfoC->set_parentid("instanceB"); + instanceMgrActor->member_->family->AddInstance(instanceInfoC); + instanceInfoB->set_detached(false); + instanceInfoA->set_parentid("instanceC"); + EXPECT_TRUE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); + instanceMgrActor->member_->jobID2InstanceIDs["job-123"] = {"instanceC"}; + EXPECT_TRUE(instanceMgrActor->IsInstanceManagedByJob(instanceInfoA)); +} } // namespace functionsystem::instance_manager::test diff --git a/functionsystem/tests/unit/function_master/meta_store/meta_store_kv_test.cpp b/functionsystem/tests/unit/function_master/meta_store/meta_store_kv_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..70b0593c38033b9fdaa6461239a0ba6ebf99628c --- /dev/null +++ b/functionsystem/tests/unit/function_master/meta_store/meta_store_kv_test.cpp @@ -0,0 +1,381 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "async/async.hpp" +#include "common/etcd_service/etcd_service_driver.h" +#include "meta_store_client/key_value/etcd_kv_client_strategy.h" +#include "kv_service_actor.h" +#include "utils/future_test_helper.h" +#include "utils/generate_info.h" +#include "utils/port_helper.h" + +namespace functionsystem::meta_store::test { +using namespace functionsystem::test; +using namespace test; +using ::testing::Invoke; +using ::testing::Return; + +class MetaStoreKvTest : public ::testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_, "MetaStoreLeaseTest-"); + + MetaStoreTimeoutOption options; + options.operationRetryIntervalLowerBound = 10; + options.operationRetryIntervalUpperBound = 100; + options.operationRetryTimes = 3; + options.grpcTimeout = 1; + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }, GrpcSslConfig(), options); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + etcdSrvDriver_->StopServer(); + } + +public: + inline static std::string metaStoreServerHost_; + inline static std::unique_ptr etcdSrvDriver_; + inline static std::shared_ptr metaStoreClient_; +}; + +#define RECEIVE_HELPER(Operation) \ + void On##Operation##Event(const litebus::AID &aid, std::string &&name, std::string &&msg) \ + { \ + On##Operation(aid, std::move(name), std::move(msg)); \ + } \ + MOCK_METHOD(void, On##Operation, (const litebus::AID, std::string, std::string)); + +class KvTestActor : public litebus::ActorBase { +public: + KvTestActor() : litebus::ActorBase("KvTestActor") + { + } + + void Init() override + { + Receive("OnPut", &KvTestActor::OnPutEvent); + Receive("OnDelete", &KvTestActor::OnDeleteEvent); + Receive("OnGet", &KvTestActor::OnGetEvent); + Receive("OnTxn", &KvTestActor::OnTxnEvent); + Receive("OnWatch", &KvTestActor::OnWatchEvent); + Receive("OnGetAndWatch", &KvTestActor::OnGetAndWatchEvent); + } + + void SendKvMessage(const litebus::AID &to, std::string name, std::string msg) + { + Send(to, std::move(name), std::move(msg)); + } + + RECEIVE_HELPER(Put); + RECEIVE_HELPER(Delete); + RECEIVE_HELPER(Get); + RECEIVE_HELPER(Txn); + RECEIVE_HELPER(Watch); + RECEIVE_HELPER(GetAndWatch); +}; + +TEST_F(MetaStoreKvTest, KvInitExploreTest) // NOLINT +{ + auto actor1 = std::make_shared(); + actor1->InitExplorer(); + EXPECT_FALSE(actor1->needExplore_); + + litebus::AID fakeAID; + auto actor2 = std::make_shared(fakeAID, false); + actor2->InitExplorer(); + EXPECT_FALSE(actor2->needExplore_); + + // fake AID, explore is true + auto actor3 = std::make_shared(fakeAID, true); + actor3->InitExplorer(); + EXPECT_FALSE(actor3->needExplore_); + EXPECT_EQ(actor3->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor3->businesses_.size(), 2); + EXPECT_EQ(actor3->business_, actor3->businesses_[MASTER_BUSINESS]); + + // needExplore_ == false, ignore UpdateLeaderInfo + actor3->UpdateLeaderInfo(GetLeaderInfo(actor2->GetAID())); + EXPECT_EQ(actor3->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor3->business_, actor3->businesses_[MASTER_BUSINESS]); + + // real AID, but explore false + auto actor4 = std::make_shared(actor3->GetAID(), false); + actor4->InitExplorer(); + EXPECT_FALSE(actor4->needExplore_); + + // real AID, and explore is true + auto actor5 = std::make_shared(actor3->GetAID(), true); + actor5->InitExplorer(); + EXPECT_TRUE(actor5->needExplore_); + EXPECT_EQ(actor5->curStatus_, SLAVE_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[SLAVE_BUSINESS]); + + // needExplore_ == true, handle UpdateLeaderInfo + actor5->UpdateLeaderInfo(GetLeaderInfo(actor5->GetAID())); + EXPECT_EQ(actor5->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[MASTER_BUSINESS]); + actor5->UpdateLeaderInfo(GetLeaderInfo(fakeAID)); + EXPECT_EQ(actor5->curStatus_, SLAVE_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[SLAVE_BUSINESS]); + + // with prefix, only used in test + std::string prefix = "prefix"; + auto actor7 = std::make_shared(prefix); + actor7->InitExplorer(); + EXPECT_FALSE(actor7->needExplore_); + EXPECT_EQ(actor7->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor7->business_, actor7->businesses_[MASTER_BUSINESS]); +} + +TEST_F(MetaStoreKvTest, KvSwitchoverTest) // NOLINT +{ + auto persistActor = + std::make_shared("Persist", metaStoreServerHost_, MetaStoreTimeoutOption{}); + litebus::Spawn(persistActor); + auto backupActor = std::make_shared("BackupActor", persistActor->GetAID()); + litebus::Spawn(backupActor); + + litebus::AID serviceAID; + auto actor = std::make_shared(backupActor->GetAID(), true); + litebus::Spawn(actor); + EXPECT_TRUE(actor->needExplore_); + + auto accessorActor = std::make_shared(actor->GetAID()); + litebus::Spawn(accessorActor); + + // slave test + auto updated = litebus::Async(actor->GetAID(), &KvServiceActor::UpdateLeaderInfo, GetLeaderInfo(serviceAID)); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, SLAVE_BUSINESS); + + auto testActor = std::make_shared(); + litebus::Spawn(testActor); + + messages::MetaStore::PutRequest putRequest; + putRequest.set_requestid("1"); + putRequest.set_key("/123"); + putRequest.set_value("123"); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Put", + putRequest.SerializeAsString()); + + messages::MetaStoreRequest req1; + ::etcdserverpb::DeleteRangeRequest deleteReq; + deleteReq.set_key("/123"); + req1.set_requestid("2"); + req1.set_requestmsg(deleteReq.SerializeAsString()); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Delete", + req1.SerializeAsString()); + + messages::MetaStoreRequest req2; + ::etcdserverpb::RangeRequest getReq; + getReq.set_key("/123"); + req2.set_requestid("3"); + req2.set_requestmsg(getReq.SerializeAsString()); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Get", + req2.SerializeAsString()); + + messages::MetaStoreRequest req3; + ::etcdserverpb::TxnRequest txnReq; + req3.set_requestid("4"); + req3.set_requestmsg(txnReq.SerializeAsString()); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Txn", + req3.SerializeAsString()); + + messages::MetaStoreRequest req4; + etcdserverpb::WatchRequest watchReq; + req4.set_requestid("5"); + req4.set_requestmsg(watchReq.SerializeAsString()); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Watch", + req4.SerializeAsString()); + + messages::MetaStoreRequest req5; + req5.set_requestid("6"); + req5.set_requestmsg(watchReq.SerializeAsString()); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "GetAndWatch", + req5.SerializeAsString()); + + sleep(1); + + // master test + updated = litebus::Async(actor->GetAID(), &KvServiceActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, MASTER_BUSINESS); + + litebus::Future msg; + EXPECT_CALL(*testActor, OnPut).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Put", + putRequest.SerializeAsString()); + messages::MetaStore::PutResponse resp; + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "1"); + + msg = {}; + EXPECT_CALL(*testActor, OnDelete).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Delete", + req1.SerializeAsString()); + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "2"); + + msg = {}; + EXPECT_CALL(*testActor, OnGet).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Get", + req2.SerializeAsString()); + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "3"); + + msg = {}; + EXPECT_CALL(*testActor, OnTxn).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Txn", + req3.SerializeAsString()); + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "4"); + + msg = {}; + EXPECT_CALL(*testActor, OnWatch).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "Watch", + req4.SerializeAsString()); + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "5"); + + msg = {}; + EXPECT_CALL(*testActor, OnGetAndWatch).WillOnce(testing::DoAll(test::FutureArg<2>(&msg), Return())); + litebus::Async(testActor->GetAID(), &KvTestActor::SendKvMessage, accessorActor->GetAID(), "GetAndWatch", + req5.SerializeAsString()); + ASSERT_AWAIT_READY(msg); + EXPECT_TRUE(resp.ParseFromString(msg.Get())); + EXPECT_EQ(resp.requestid(), "6"); + + litebus::Terminate(accessorActor->GetAID()); + litebus::Await(accessorActor->GetAID()); + litebus::Terminate(testActor->GetAID()); + litebus::Await(testActor->GetAID()); + litebus::Terminate(actor->GetAID()); + litebus::Await(actor->GetAID()); + litebus::Terminate(backupActor->GetAID()); + litebus::Await(backupActor->GetAID()); + litebus::Terminate(persistActor->GetAID()); + litebus::Await(persistActor->GetAID()); +} + +TEST_F(MetaStoreKvTest, KvWatchTest) // NOLINT +{ + DeleteOption opt; + opt.prefix = true; + metaStoreClient_->Delete("/", opt); + + auto persistActor = + std::make_shared("Persist", metaStoreServerHost_, MetaStoreTimeoutOption{}); + litebus::Spawn(persistActor); + auto backupActor = std::make_shared("BackupActor", persistActor->GetAID()); + litebus::Spawn(backupActor); + + // sync key + ::mvccpb::KeyValue kv; + kv.set_key("/1234"); + kv.set_value("1234"); + kv.set_mod_revision(9); + metaStoreClient_->Put("/metastore/kv//1234", kv.SerializeAsString(), {}); + ASSERT_AWAIT_TRUE([&]() { return !metaStoreClient_->Get("/metastore/kv//1234", {}).Get()->kvs.empty(); }); + + litebus::AID serviceAID; + auto actor = std::make_shared(backupActor->GetAID(), true); + actor->modRevision_ = 15; + + litebus::Spawn(actor); + EXPECT_TRUE(actor->needExplore_); + auto accessorActor = std::make_shared(actor->GetAID()); + litebus::Spawn(accessorActor); + sleep(1); + + auto updated = litebus::Async(actor->GetAID(), &KvServiceActor::UpdateLeaderInfo, GetLeaderInfo(serviceAID)); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, SLAVE_BUSINESS); + + // early revision ignore + kv.set_mod_revision(11); + metaStoreClient_->Put("/metastore/kv//1234", kv.SerializeAsString(), {}); + + kv.set_mod_revision(16); + metaStoreClient_->Put("/metastore/kv//1234", kv.SerializeAsString(), {}); + ASSERT_AWAIT_TRUE([&]() { return actor->cache_.size() == 1; }); + ASSERT_AWAIT_TRUE([&]() { return actor->cache_.find("/1234") != actor->cache_.end(); }); + EXPECT_EQ(actor->cache_.at("/1234").key(), "/1234"); + EXPECT_EQ(actor->cache_.at("/1234").value(), "1234"); + ASSERT_AWAIT_TRUE([&]() { return actor->cache_.at("/1234").mod_revision() == 16; }); + ASSERT_AWAIT_TRUE([&]() { return actor->modRevision_ >= 16; }); + + metaStoreClient_->Delete("/metastore/kv//1234", {}); + ASSERT_AWAIT_TRUE([&]() { return actor->cache_.size() == 0; }); + + litebus::Terminate(accessorActor->GetAID()); + litebus::Await(accessorActor->GetAID()); + litebus::Terminate(actor->GetAID()); + litebus::Await(actor->GetAID()); + litebus::Terminate(backupActor->GetAID()); + litebus::Await(backupActor->GetAID()); + litebus::Terminate(persistActor->GetAID()); + litebus::Await(persistActor->GetAID()); +} + +TEST_F(MetaStoreKvTest, KvSlaveTest) // NOLINT +{ + auto actor1 = std::make_shared(); + EXPECT_FALSE(actor1->needExplore_); + + auto member = std::make_shared(); + auto slaveBusiness = std::make_shared(member, actor1); + slaveBusiness->OnChange(); + + auto request = std::make_shared(); + auto status = slaveBusiness->AsyncTxn(litebus::AID(), request); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); + + status = slaveBusiness->AsyncGet(litebus::AID(), request); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); + + status = slaveBusiness->AsyncDelete(litebus::AID(), request); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); + + auto putRequest = std::make_shared(); + status = slaveBusiness->AsyncPut(litebus::AID(), putRequest); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); + + status = slaveBusiness->AsyncGetAndWatch(litebus::AID(), request); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); + + status = slaveBusiness->AsyncWatch(litebus::AID(), request); + ASSERT_AWAIT_READY(status); + EXPECT_TRUE(status.Get().IsError()); +} +} // namespace functionsystem::meta_store::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/meta_store/meta_store_lease_test.cpp b/functionsystem/tests/unit/function_master/meta_store/meta_store_lease_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dc6834400dff7f685fb80f0dd80e7c477858baec --- /dev/null +++ b/functionsystem/tests/unit/function_master/meta_store/meta_store_lease_test.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "async/async.hpp" +#include "common/etcd_service/etcd_service_driver.h" +#include "meta_store_client/key_value/etcd_kv_client_strategy.h" +#include "lease_service_actor.h" +#include "utils/future_test_helper.h" +#include "utils/generate_info.h" +#include "utils/port_helper.h" + +namespace functionsystem::meta_store::test { +using namespace functionsystem::test; +using namespace test; +using ::testing::Invoke; +using ::testing::Return; + +class MetaStoreLeaseTest : public ::testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_, "MetaStoreLeaseTest-"); + + MetaStoreTimeoutOption options; + options.operationRetryIntervalLowerBound = 10; + options.operationRetryIntervalUpperBound = 100; + options.operationRetryTimes = 3; + options.grpcTimeout = 1; + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }, GrpcSslConfig(), options); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + etcdSrvDriver_->StopServer(); + } + +public: + inline static std::string metaStoreServerHost_; + inline static std::unique_ptr etcdSrvDriver_; + inline static std::shared_ptr metaStoreClient_; +}; + +class LeaseTestActor : public litebus::ActorBase { +public: + LeaseTestActor() : litebus::ActorBase("LeaseTestActor") + { + } + + void Init() override + { + Receive("GrantCallback", &LeaseTestActor::GrantCallback); + Receive("RevokeCallback", &LeaseTestActor::RevokeCallback); + Receive("KeepAliveCallback", &LeaseTestActor::KeepAliveCallback); + } + + void SendLeaseMessage(const litebus::AID &to, std::string name, std::string msg) + { + Send(to, std::move(name), std::move(msg)); + } + + MOCK_METHOD(void, GrantCallback, (const litebus::AID &, std::string &&, std::string &&)); + MOCK_METHOD(void, RevokeCallback, (const litebus::AID &, std::string &&, std::string &&)); + MOCK_METHOD(void, KeepAliveCallback, (const litebus::AID &, std::string &&, std::string &&)); +}; + +TEST_F(MetaStoreLeaseTest, LeaseInitExploreTest) // NOLINT +{ + litebus::AID serviceAID; + auto actor1 = std::make_shared(serviceAID); + actor1->InitExplorer(); + EXPECT_FALSE(actor1->needExplore_); + + litebus::AID fakeAID; + auto actor2 = std::make_shared(serviceAID, false, fakeAID); + actor2->InitExplorer(); + EXPECT_FALSE(actor2->needExplore_); + + // fake AID, explore is true + auto actor3 = std::make_shared(serviceAID, true, fakeAID); + actor3->InitExplorer(); + EXPECT_FALSE(actor3->needExplore_); + EXPECT_EQ(actor3->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor3->businesses_.size(), 2); + EXPECT_EQ(actor3->business_, actor3->businesses_[MASTER_BUSINESS]); + + // needExplore_ == false, ignore UpdateLeaderInfo + actor3->UpdateLeaderInfo(GetLeaderInfo(actor2->GetAID())); + EXPECT_EQ(actor3->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor3->business_, actor3->businesses_[MASTER_BUSINESS]); + + // real AID, but explore false + auto actor4 = std::make_shared(serviceAID, false, actor3->GetAID()); + actor4->InitExplorer(); + EXPECT_FALSE(actor4->needExplore_); + + // real AID, and explore is true + auto actor5 = std::make_shared(serviceAID, true, actor3->GetAID()); + actor5->InitExplorer(); + EXPECT_TRUE(actor5->needExplore_); + EXPECT_EQ(actor5->curStatus_, SLAVE_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[SLAVE_BUSINESS]); + + // needExplore_ == true, handle UpdateLeaderInfo + actor5->UpdateLeaderInfo(GetLeaderInfo(actor5->GetAID())); + EXPECT_EQ(actor5->curStatus_, MASTER_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[MASTER_BUSINESS]); + actor5->UpdateLeaderInfo(GetLeaderInfo(serviceAID)); + EXPECT_EQ(actor5->curStatus_, SLAVE_BUSINESS); + EXPECT_EQ(actor5->business_, actor5->businesses_[SLAVE_BUSINESS]); +} + +TEST_F(MetaStoreLeaseTest, LeaseSwitchoverTest) // NOLINT +{ + auto persistActor = + std::make_shared("Persist", metaStoreServerHost_, MetaStoreTimeoutOption{}); + litebus::Spawn(persistActor); + auto backupActor = std::make_shared("BackupActor", persistActor->GetAID()); + litebus::Spawn(backupActor); + + litebus::AID serviceAID; + auto actor = std::make_shared(serviceAID, true, backupActor->GetAID()); + litebus::Spawn(actor); + EXPECT_TRUE(actor->needExplore_); + + // slave test + auto updated = litebus::Async(actor->GetAID(), &LeaseServiceActor::UpdateLeaderInfo, GetLeaderInfo(serviceAID)); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, SLAVE_BUSINESS); + + auto status = litebus::Async(actor->GetAID(), &LeaseServiceActor::Start); + ASSERT_AWAIT_READY(status); + ASSERT_AWAIT_TRUE([&]() { return actor->running_; }); + + auto testActor = std::make_shared(); + litebus::Spawn(testActor); + + { + EXPECT_CALL(*testActor, GrantCallback).Times(0); + EXPECT_CALL(*testActor, RevokeCallback).Times(0); + EXPECT_CALL(*testActor, KeepAliveCallback).Times(0); + + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveGrant", ""); + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveRevoke", ""); + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveKeepAliveOnce", + ""); + } + sleep(1); + + // master test + updated = litebus::Async(actor->GetAID(), &LeaseServiceActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, MASTER_BUSINESS); + + ::etcdserverpb::LeaseGrantRequest grantRequest; + messages::MetaStoreRequest req; + req.set_requestid("1"); + req.set_requestmsg(grantRequest.SerializeAsString()); + EXPECT_CALL(*testActor, GrantCallback).WillOnce(Return()); + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveGrant", + req.SerializeAsString()); + + ::etcdserverpb::LeaseRevokeRequest revokeRequest; + req.set_requestid("2"); + req.set_requestmsg(revokeRequest.SerializeAsString()); + EXPECT_CALL(*testActor, RevokeCallback).WillOnce(Return()); + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveRevoke", + req.SerializeAsString()); + + ::etcdserverpb::LeaseKeepAliveRequest keepAliveRequest; + req.set_requestid("3"); + req.set_requestmsg(keepAliveRequest.SerializeAsString()); + bool finished = false; + EXPECT_CALL(*testActor, KeepAliveCallback).WillOnce(testing::Assign(&finished, true)); + litebus::Async(testActor->GetAID(), &LeaseTestActor::SendLeaseMessage, actor->GetAID(), "ReceiveKeepAliveOnce", + req.SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() { return finished; }); + + litebus::Terminate(testActor->GetAID()); + litebus::Await(testActor->GetAID()); + litebus::Terminate(actor->GetAID()); + litebus::Await(actor->GetAID()); + litebus::Terminate(backupActor->GetAID()); + litebus::Await(backupActor->GetAID()); + litebus::Terminate(persistActor->GetAID()); + litebus::Await(persistActor->GetAID()); +} + +TEST_F(MetaStoreLeaseTest, LeaseWatchTest) // NOLINT +{ + DeleteOption opt; + opt.prefix = true; + metaStoreClient_->Delete("/", opt); + + auto persistActor = + std::make_shared("Persist", metaStoreServerHost_, MetaStoreTimeoutOption{}); + litebus::Spawn(persistActor); + auto backupActor = std::make_shared("BackupActor", persistActor->GetAID()); + litebus::Spawn(backupActor); + + litebus::AID serviceAID; + auto actor = std::make_shared(serviceAID, true, backupActor->GetAID()); + litebus::Spawn(actor); + EXPECT_TRUE(actor->needExplore_); + + // slave test + auto updated = litebus::Async(actor->GetAID(), &LeaseServiceActor::UpdateLeaderInfo, GetLeaderInfo(serviceAID)); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, SLAVE_BUSINESS); + + auto status = litebus::Async(actor->GetAID(), &LeaseServiceActor::Start); + ASSERT_AWAIT_READY(status); + ASSERT_AWAIT_TRUE([&]() { return actor->running_; }); + + ::messages::Lease lease; + lease.set_id(1234); + lease.set_ttl(123); + metaStoreClient_->Put("/metastore/lease/1234", lease.SerializeAsString(), {}); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.size() == 1; }); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.find(1234) != actor->leases_.end(); }); + EXPECT_EQ(actor->leases_.at(1234).id(), 1234); + EXPECT_EQ(actor->leases_.at(1234).ttl(), 123); + EXPECT_NE(actor->leases_.at(1234).expiry(), 0); + + metaStoreClient_->Delete("/metastore/lease/1234", {}); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.size() == 0; }); + + metaStoreClient_->Put("/metastore/lease/1234", lease.SerializeAsString(), {}); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.size() == 1; }); + + // master test + updated = litebus::Async(actor->GetAID(), &LeaseServiceActor::UpdateLeaderInfo, GetLeaderInfo(actor->GetAID())); + ASSERT_AWAIT_READY(updated); + EXPECT_EQ(actor->curStatus_, MASTER_BUSINESS); + + metaStoreClient_->Delete("/metastore/lease/1234", {}); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.size() == 1; }); + + metaStoreClient_->Put("/metastore/lease/1234", lease.SerializeAsString(), {}); + ASSERT_AWAIT_TRUE([&]() { return actor->leases_.size() == 1; }); + + litebus::Terminate(actor->GetAID()); + litebus::Await(actor->GetAID()); + litebus::Terminate(backupActor->GetAID()); + litebus::Await(backupActor->GetAID()); + litebus::Terminate(persistActor->GetAID()); + litebus::Await(persistActor->GetAID()); +} + +TEST_F(MetaStoreLeaseTest, LeaseSlaveTest) // NOLINT +{ + litebus::AID serviceAID; + auto actor1 = std::make_shared(serviceAID); + EXPECT_FALSE(actor1->needExplore_); + + auto member = std::make_shared(); + auto slaveBusiness = std::make_shared(member, actor1); + slaveBusiness->OnChange(); + + messages::MetaStoreRequest request; + slaveBusiness->ReceiveKeepAlive(litebus::AID(), request); + slaveBusiness->ReceiveRevoke(litebus::AID(), request); + slaveBusiness->ReceiveGrant(litebus::AID(), request); +} +} // namespace functionsystem::meta_store::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/meta_store/meta_store_test.cpp b/functionsystem/tests/unit/function_master/meta_store/meta_store_test.cpp index f907e59801dc7004f5b2eba5e14b9d1145a32787..6d8c6c149804233b36e0331bab19b5646eaba89e 100644 --- a/functionsystem/tests/unit/function_master/meta_store/meta_store_test.cpp +++ b/functionsystem/tests/unit/function_master/meta_store/meta_store_test.cpp @@ -16,18 +16,19 @@ #include #include "async/async.hpp" -#include "meta_store_client/key_value/etcd_kv_client_strategy.h" -#include "meta_store_client/meta_store_client.h" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" #include "kv_service_accessor_actor.h" #include "kv_service_actor.h" #include "lease_service_actor.h" +#include "meta_store_client/key_value/etcd_kv_client_strategy.h" +#include "meta_store_client/meta_store_client.h" #include "meta_store_driver.h" -#include "watch_service_actor.h" +#include "meta_store_monitor/meta_store_monitor_factory.h" #include "mock_store_client.h" #include "mocks/mock_etcd_kv_service.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" +#include "watch_service_actor.h" namespace functionsystem::meta_store::test { using namespace functionsystem::test; @@ -37,7 +38,7 @@ using ::testing::Return; class MetaStoreTest : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdKvService_ = std::make_shared(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -62,10 +63,10 @@ public: localAddress_ = "127.0.0.1:" + std::to_string(port); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { if (etcdServer_ != nullptr) { - etcdServer_->Shutdown(); + etcdServer_->Shutdown(std::chrono::system_clock::now()); etcdServer_ = nullptr; } } @@ -94,8 +95,8 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDPutTest) // NOLINT { auto persistAID = litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); - auto backupAID = litebus::Spawn(std::make_shared("backupActor", persistAID)); - litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); + auto backAID = litebus::Spawn(std::make_shared("backupActor", persistAID)); + litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backAID)); litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, .metaStoreAddress = kvServerAccessorAID.Url(), @@ -141,21 +142,21 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDPutTest) // NOLINT EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { - EXPECT_TRUE(request->compare_size() == 0); EXPECT_TRUE(request->success_size() == 1); + EXPECT_TRUE(request->compare_size() == 0); const auto &cmp = request->success(0); EXPECT_TRUE(cmp.request_case() == etcdserverpb::RequestOp::kRequestPut); - const auto &putRequest = cmp.request_put(); - EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + key); + const auto &putReq = cmp.request_put(); + EXPECT_TRUE(putReq.key() == std::string("/metastore/kv/") + key); - const auto &val = putRequest.value(); + const auto &val = putReq.value(); ::mvccpb::KeyValue kv; EXPECT_TRUE(kv.ParseFromString(val)); - EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv.key()); + EXPECT_TRUE(putReq.key() == std::string("/metastore/kv/") + kv.key()); - EXPECT_TRUE(kv.key() == key); EXPECT_TRUE(kv.value() == "mock-value-x"); + EXPECT_TRUE(kv.key() == key); *response = ::etcdserverpb::TxnResponse{}; response->mutable_header()->set_revision(1); finished = true; @@ -174,8 +175,8 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDPutTest) // NOLINT litebus::Terminate(kvServerAID); litebus::Await(kvServerAID); - litebus::Terminate(backupAID); - litebus::Await(backupAID); + litebus::Terminate(backAID); + litebus::Await(backAID); litebus::Terminate(persistAID); litebus::Await(persistAID); } @@ -199,7 +200,7 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDDeleteTest) // NOLINT bool finished = false; EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, - ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { + ::etcdserverpb::TxnResponse *resp) -> ::grpc::Status { EXPECT_TRUE(request->compare_size() == 0); EXPECT_TRUE(request->success_size() == 1); const auto &cmp = request->success(0); @@ -214,8 +215,8 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDDeleteTest) // NOLINT EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv1.key()); EXPECT_TRUE(kv1.key() == key); EXPECT_TRUE(kv1.value() == value); - *response = ::etcdserverpb::TxnResponse{}; - response->mutable_header()->set_revision(1); + *resp = ::etcdserverpb::TxnResponse{}; + resp->mutable_header()->set_revision(1); return ::grpc::Status::OK; })) .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, @@ -234,7 +235,7 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDDeleteTest) // NOLINT return ::grpc::Status::OK; })); - auto fut = client.Put(key, value, PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false }); + auto fut = client.Put(key, value, PutOption{ .asyncBackup = false }); EXPECT_AWAIT_READY(fut); auto future = client.Delete(key, DeleteOption{ .prevKv = true, .prefix = true, .asyncBackup = false }); @@ -246,22 +247,22 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDDeleteTest) // NOLINT EXPECT_AWAIT_TRUE([&]() { return finished; }); - litebus::Terminate(kvServerAccessorAID); - litebus::Await(kvServerAccessorAID); litebus::Terminate(kvServerAID); litebus::Await(kvServerAID); + litebus::Terminate(kvServerAccessorAID); + litebus::Await(kvServerAccessorAID); - litebus::Terminate(backupAID); - litebus::Await(backupAID); litebus::Terminate(persistAID); litebus::Await(persistAID); + litebus::Terminate(backupAID); + litebus::Await(backupAID); } TEST_F(MetaStoreTest, MetaStoreWithETCDTxnTest) // NOLINT { - auto persistAID = + auto persist = litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); - auto backupAID = litebus::Spawn(std::make_shared("BackupActor3", persistAID)); + auto backupAID = litebus::Spawn(std::make_shared("BackupActor3", persist)); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, @@ -291,6 +292,8 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDTxnTest) // NOLINT EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv.key()); EXPECT_TRUE(kv.value() == value); EXPECT_TRUE(kv.key() == key); + *response = ::etcdserverpb::TxnResponse{}; + response->mutable_header()->set_revision(1); return ::grpc::Status::OK; })) .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, @@ -308,7 +311,7 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDTxnTest) // NOLINT return ::grpc::Status::OK; })); - auto fut = client.Put(key, value, PutOption{.leaseId = 0, .prevKv = false, .asyncBackup = false }); + auto fut = client.Put(key, value, PutOption{ .asyncBackup = false }); EXPECT_AWAIT_READY(fut); auto transaction = client.BeginTransaction(); @@ -328,10 +331,202 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDTxnTest) // NOLINT litebus::Terminate(backupAID); litebus::Await(backupAID); + litebus::Terminate(persist); + litebus::Await(persist); +} + +TEST_F(MetaStoreTest, MetaStoreWithETCDPutErrorTest) // NOLINT +{ + auto persistAID = + litebus::Spawn(std::make_shared("Persist", etcdAddress_, + MetaStoreTimeoutOption{ + .operationRetryIntervalLowerBound = 100, + .operationRetryIntervalUpperBound = 200, + .operationRetryTimes = 1, + .grpcTimeout = GRPC_TIMEOUT_SECONDS})); + auto backAID = litebus::Spawn(std::make_shared("backupActor5", persistAID)); + litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backAID)); + litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, + .metaStoreAddress = kvServerAccessorAID.Url(), + .enableMetaStore = true },{}, MetaStoreTimeoutOption{}); + client.Init(); + + const std::string value = "mock-value5"; + const std::string key = "mock-key5"; + + EXPECT_CALL(*etcdKvService_, Txn) // test persistence + .WillRepeatedly(Return(::grpc::Status(::grpc::StatusCode::UNAVAILABLE, "etcd is unavailable ..."))); + + // failed to put key: + auto future = client.Put(key, value, PutOption{ .leaseId = 0, .prevKv = true, .asyncBackup = false }); + EXPECT_AWAIT_READY(future); + EXPECT_TRUE(future.Get()->status.IsError()); + + // get key: , expected: get nothing + auto getFut = client.Get(key, GetOption{ .prefix = true }); + EXPECT_AWAIT_READY(getFut); + EXPECT_TRUE(getFut.Get()->status.IsOk()); + EXPECT_EQ(getFut.Get()->count, 0); + EXPECT_EQ(getFut.Get()->kvs.size(), 0); + + litebus::Terminate(kvServerAccessorAID); + litebus::Await(kvServerAccessorAID); + + litebus::Terminate(kvServerAID); + litebus::Await(kvServerAID); + litebus::Terminate(backAID); + litebus::Await(backAID); litebus::Terminate(persistAID); litebus::Await(persistAID); } +TEST_F(MetaStoreTest, MetaStoreWithETCDDeleteErrorTest) // NOLINT +{ + auto persistAID = + litebus::Spawn(std::make_shared("Persist", etcdAddress_, + MetaStoreTimeoutOption{ + .operationRetryIntervalLowerBound = 100, + .operationRetryIntervalUpperBound = 200, + .operationRetryTimes = 1, + .grpcTimeout = GRPC_TIMEOUT_SECONDS})); + auto backupAID = litebus::Spawn(std::make_shared("BackupActor6", persistAID)); + litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); + litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, + .metaStoreAddress = kvServerAccessorAID.Url(), + .enableMetaStore = true },{}, MetaStoreTimeoutOption{}); + client.Init(); + + const std::string key = "mock-key6"; + const std::string value = "mock-value6"; + + EXPECT_CALL(*etcdKvService_, Txn) // test persistence + .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, + ::etcdserverpb::TxnResponse *resp) -> ::grpc::Status { + EXPECT_TRUE(request->compare_size() == 0); + EXPECT_TRUE(request->success_size() == 1); + const auto &cmp = request->success(0); + + EXPECT_TRUE(cmp.request_case() == etcdserverpb::RequestOp::kRequestPut); + const auto &putRequest = cmp.request_put(); + EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + key); + const auto &val = putRequest.value(); + + ::mvccpb::KeyValue kv1; + EXPECT_TRUE(kv1.ParseFromString(val)); + EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv1.key()); + EXPECT_TRUE(kv1.key() == key); + EXPECT_TRUE(kv1.value() == value); + *resp = ::etcdserverpb::TxnResponse{}; + resp->mutable_header()->set_revision(1); + return ::grpc::Status::OK; + })) + .WillRepeatedly(Return(::grpc::Status(::grpc::StatusCode::UNAVAILABLE, "etcd is unavailable ..."))); + + // put key: + auto fut = client.Put(key, value, PutOption{ .asyncBackup = false }); + EXPECT_AWAIT_READY(fut); + + // failed to delete key: + auto future = client.Delete(key, DeleteOption{ .prevKv = true, .prefix = true, .asyncBackup = false }); + EXPECT_AWAIT_READY(future); + EXPECT_TRUE(future.Get()->status.IsError()); + EXPECT_EQ(future.Get()->deleted, 1); + EXPECT_EQ(future.Get()->prevKvs.at(0).key(), key); + EXPECT_EQ(future.Get()->prevKvs.at(0).value(), value); + + // get key: + auto getFut = client.Get(key, GetOption{ .prefix = true }); + EXPECT_AWAIT_READY(getFut); + EXPECT_TRUE(getFut.Get()->status.IsOk()); + EXPECT_EQ(getFut.Get()->count, 1); + EXPECT_EQ(getFut.Get()->kvs.at(0).key(), key); + EXPECT_EQ(getFut.Get()->kvs.at(0).value(), value); + + litebus::Terminate(kvServerAID); + litebus::Await(kvServerAID); + litebus::Terminate(kvServerAccessorAID); + litebus::Await(kvServerAccessorAID); + + litebus::Terminate(persistAID); + litebus::Await(persistAID); + litebus::Terminate(backupAID); + litebus::Await(backupAID); +} + +TEST_F(MetaStoreTest, MetaStoreWithETCDTxnErrorTest) // NOLINT +{ + auto persist = + litebus::Spawn(std::make_shared("Persist", etcdAddress_, + MetaStoreTimeoutOption{ + .operationRetryIntervalLowerBound = 100, + .operationRetryIntervalUpperBound = 200, + .operationRetryTimes = 1, + .grpcTimeout = GRPC_TIMEOUT_SECONDS})); + auto backupAID = litebus::Spawn(std::make_shared("BackupActor7", persist)); + litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); + litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); + MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, + .metaStoreAddress = kvServerAccessorAID.Url(), + .enableMetaStore = true },{}, MetaStoreTimeoutOption{}); + client.Init(); + + const std::string key = "mock-key7"; + const std::string value = "mock-value7"; + + EXPECT_CALL(*etcdKvService_, Txn) // test persistence + .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, + ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { + EXPECT_TRUE(request->compare_size() == 0); + EXPECT_TRUE(request->success_size() == 1); + const auto &cmp = request->success(0); + EXPECT_TRUE(cmp.request_case() == etcdserverpb::RequestOp::kRequestPut); + const auto &putRequest = cmp.request_put(); + EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + key); + const auto &val = putRequest.value(); + + ::mvccpb::KeyValue kv; + EXPECT_TRUE(kv.ParseFromString(val)); + EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv.key()); + EXPECT_TRUE(kv.value() == value); + EXPECT_TRUE(kv.key() == key); + *response = ::etcdserverpb::TxnResponse{}; + response->mutable_header()->set_revision(1); + return ::grpc::Status::OK; + })) + .WillRepeatedly(Return(::grpc::Status(::grpc::StatusCode::UNAVAILABLE, "etcd is unavailable ..."))); + + // put key: + auto fut = client.Put(key, value, PutOption{ .asyncBackup = false }); + EXPECT_AWAIT_READY(fut); + + // txn key: + auto transaction = client.BeginTransaction(); + transaction->If(TxnCompare::OfValue(key, CompareOperator::EQUAL, value)); + transaction->Then(TxnOperation::Create(key, DeleteOption{ true, false, false })); + std::shared_ptr txnResponse = transaction->Commit().Get(); + EXPECT_TRUE(txnResponse->status.IsError()); + + // get key: + auto getFut = client.Get(key, GetOption{ .prefix = true }); + EXPECT_AWAIT_READY(getFut); + EXPECT_TRUE(getFut.Get()->status.IsOk()); + EXPECT_EQ(getFut.Get()->count, 1); + EXPECT_EQ(getFut.Get()->kvs.at(0).key(), key); + EXPECT_EQ(getFut.Get()->kvs.at(0).value(), value); + + litebus::Terminate(kvServerAccessorAID); + litebus::Await(kvServerAccessorAID); + litebus::Terminate(kvServerAID); + litebus::Await(kvServerAID); + litebus::Terminate(backupAID); + litebus::Await(backupAID); + + litebus::Terminate(persist); + litebus::Await(persist); +} + TEST_F(MetaStoreTest, MetaStoreWithETCDGetTest) // NOLINT { auto persistAID = @@ -353,15 +548,15 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDGetTest) // NOLINT EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillOnce(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { - EXPECT_TRUE(request->compare_size() == 0); EXPECT_TRUE(request->success_size() == 1); + EXPECT_TRUE(request->compare_size() == 0); const auto &cmp = request->success(0); EXPECT_TRUE(cmp.request_case() == etcdserverpb::RequestOp::kRequestPut); const auto &putRequest = cmp.request_put(); EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + key); - const auto &val = putRequest.value(); ::mvccpb::KeyValue kv; + const auto &val = putRequest.value(); EXPECT_TRUE(kv.ParseFromString(val)); EXPECT_TRUE(putRequest.key() == std::string("/metastore/kv/") + kv.key()); EXPECT_TRUE(kv.key() == key); @@ -372,7 +567,7 @@ TEST_F(MetaStoreTest, MetaStoreWithETCDGetTest) // NOLINT return ::grpc::Status::OK; })); - auto fut = client.Put(key, value, PutOption{.leaseId = 0, .prevKv = false, .asyncBackup = false }); + auto fut = client.Put(key, value, PutOption{ .asyncBackup = false }); EXPECT_AWAIT_READY(fut); auto future = client.Get(key, GetOption{ .prefix = true }); @@ -587,7 +782,9 @@ TEST_F(MetaStoreTest, MetaStoreClientAndMetaStoreServiceTest) // NOLINT return true; }; - auto syncer = []() -> litebus::Future { return SyncResult{ Status::OK(), 0 }; }; + auto syncer = [](const std::shared_ptr &) -> litebus::Future { + return SyncResult{ Status::OK() }; + }; functionsystem::WatchOption watchOption{}; watchOption.prevKv = true; auto watcher1 = metaStoreClient->Watch("key", watchOption, func, syncer); @@ -964,8 +1161,8 @@ TEST_F(MetaStoreTest, RangeObserverCacheTest) messages::MetaStoreRequest req; req.set_requestid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); auto request = std::make_shared(); - request->set_key(INSTANCE_ROUTE_PATH_PREFIX); request->set_range_end(StringPlusOne(INSTANCE_ROUTE_PATH_PREFIX)); + request->set_key(INSTANCE_ROUTE_PATH_PREFIX); req.set_requestmsg(request->SerializeAsString()); kvAccessorActor->AsyncDelete(client->GetAID(), "Delete", req.SerializeAsString()); } @@ -1034,31 +1231,33 @@ TEST_F(MetaStoreTest, GetAndWatchTest) put = true; })); + bool isPut = false; EXPECT_CALL(*client, MockOnWatch) - .WillOnce(Invoke([](const litebus::AID &from, std::string name, std::string msg) { // OnPut - etcdserverpb::WatchResponse response; - EXPECT_TRUE(ParseWatchResponse(response, msg)); - EXPECT_EQ(response.events_size(), 1); - auto &event = response.events(0); + .WillOnce(Invoke([&isPut](const litebus::AID &from, std::string name, std::string msg) { // OnPut + etcdserverpb::WatchResponse resp; + EXPECT_TRUE(ParseWatchResponse(resp, msg)); + EXPECT_EQ(resp.events_size(), 1); + auto &event = resp.events(0); EXPECT_EQ(event.type(), ::mvccpb::Event_EventType::Event_EventType_PUT); - EXPECT_EQ(event.kv().key(), "key"); EXPECT_EQ(event.kv().value(), "2.0"); + EXPECT_EQ(event.kv().key(), "key"); + isPut = true; })) .WillOnce(Invoke([&deleted](const litebus::AID &from, std::string name, std::string msg) { // OnDelete etcdserverpb::WatchResponse response; EXPECT_TRUE(ParseWatchResponse(response, msg)); EXPECT_EQ(response.events_size(), 1); auto &event = response.events(0); - EXPECT_EQ(event.type(), ::mvccpb::Event_EventType::Event_EventType_DELETE); EXPECT_EQ(event.prev_kv().value(), "2.0"); + EXPECT_EQ(event.type(), ::mvccpb::Event_EventType::Event_EventType_DELETE); deleted = true; })); { messages::MetaStore::PutRequest request; request.set_requestid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); - request.set_key("key"); request.set_value("1.0"); + request.set_key("key"); kvAccessorActor->AsyncPut(client->GetAID(), "Put", request.SerializeAsString()); } @@ -1067,18 +1266,17 @@ TEST_F(MetaStoreTest, GetAndWatchTest) auto *args = request->mutable_create_request(); args->set_key("key"); args->set_prev_kv(true); // prefix - args->set_range_end(StringPlusOne("key")); args->set_start_revision(0); - req.set_requestid(uuid); + args->set_range_end(StringPlusOne("key")); req.set_requestmsg(request->SerializeAsString()); + req.set_requestid(uuid); kvAccessorActor->AsyncGetAndWatch(client->GetAID(), "GetAndWatch", req.SerializeAsString()); ASSERT_AWAIT_TRUE([&]() -> bool { return put; }); // Put { - bool isPut = false; - EXPECT_CALL(*client, MockOnPut).WillOnce(DoAll(testing::Assign(&isPut, true), Return())); + EXPECT_CALL(*client, MockOnPut).WillOnce(Return()); messages::MetaStore::PutRequest request; request.set_requestid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); request.set_key("key"); @@ -1092,8 +1290,8 @@ TEST_F(MetaStoreTest, GetAndWatchTest) messages::MetaStoreRequest req; req.set_requestid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); auto request = std::make_shared(); - request->set_key("key"); request->set_range_end("kez"); // delete [key, kez) + request->set_key("key"); req.set_requestmsg(request->SerializeAsString()); kvAccessorActor->AsyncDelete(client->GetAID(), "Delete", req.SerializeAsString()); @@ -1339,8 +1537,7 @@ TEST_F(MetaStoreTest, BackupTest) { auto transaction = client.BeginTransaction(); - transaction->Then( - TxnOperation::Create("key1", "value1", PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); + transaction->Then(TxnOperation::Create("key1", "value1", PutOption{ .asyncBackup = false })); transaction->Then(TxnOperation::Create("key2", "value2", PutOption{})); transaction->Then(TxnOperation::Create("key3", "value3", PutOption{})); transaction->Then(TxnOperation::Create(frontendKey, frontendVal, PutOption{})); @@ -1351,20 +1548,21 @@ TEST_F(MetaStoreTest, BackupTest) } { auto transaction = client.BeginTransaction(); - transaction->Then( - TxnOperation::Create("key", DeleteOption{ .prevKv = false, .prefix = true, .asyncBackup = false })); - transaction->Then(TxnOperation::Create("/sn/instance", - DeleteOption{ .prevKv = false, .prefix = true, .asyncBackup = false })); + transaction->Then(TxnOperation::Create("key", DeleteOption{ .prefix = true, .asyncBackup = false })); + transaction->Then(TxnOperation::Create("/sn/instance", DeleteOption{ .prefix = true, .asyncBackup = false })); transaction->Commit().Get(); } - litebus::Terminate(kvServerAccessorAID); - litebus::Await(kvServerAccessorAID); + litebus::Terminate(kvServerAID); litebus::Await(kvServerAID); + litebus::Terminate(kvServerAccessorAID); + litebus::Await(kvServerAccessorAID); + litebus::Terminate(backupAID); litebus::Await(backupAID); + litebus::Terminate(persistAID); litebus::Await(persistAID); } @@ -1374,39 +1572,39 @@ TEST_F(MetaStoreTest, SlowBackupTest) auto persistAID = litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); auto backupAID = litebus::Spawn(std::make_shared("BackupActor", persistAID)); - litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); - - litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); + litebus::AID kvAID = litebus::Spawn(std::make_shared(backupAID)); + litebus::AID kvAccessorAID = litebus::Spawn(std::make_shared(kvAID)); MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, - .metaStoreAddress = kvServerAccessorAID.Url(), + .metaStoreAddress = kvAccessorAID.Url(), .enableMetaStore = true }, {}, MetaStoreTimeoutOption{}); client.Init(); EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillRepeatedly(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, - ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { + ::etcdserverpb::TxnResponse *resp) -> ::grpc::Status { TimeLoop(1); - *response = ::etcdserverpb::TxnResponse{}; - response->mutable_header()->set_revision(1); + *resp = ::etcdserverpb::TxnResponse{}; + resp->mutable_header()->set_revision(1); return ::grpc::Status::OK; })); std::vector>> futures; for (int i = 0; i < 10; i++) { - auto transaction = client.BeginTransaction(); - transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); - futures.emplace_back(transaction->Commit()); + auto txn = client.BeginTransaction(); + txn->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), + PutOption{ .asyncBackup = false })); + futures.emplace_back(txn->Commit()); } + for (auto &fut : futures) { ASSERT_AWAIT_READY(fut); } - litebus::Terminate(kvServerAccessorAID); - litebus::Await(kvServerAccessorAID); - litebus::Terminate(kvServerAID); - litebus::Await(kvServerAID); + litebus::Terminate(kvAccessorAID); + litebus::Await(kvAccessorAID); + litebus::Terminate(kvAID); + litebus::Await(kvAID); litebus::Terminate(backupAID); litebus::Await(backupAID); @@ -1420,8 +1618,7 @@ TEST_F(MetaStoreTest, BackupFlushBelowMaxConcurrency) litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); auto backupAID = litebus::Spawn(std::make_shared( "BackupActor", persistAID, - MetaStoreBackupOption{ .enableSyncSysFunc = false, .metaStoreMaxFlushConcurrency = 10, - .metaStoreMaxFlushBatchSize = 1 })); + MetaStoreBackupOption{ .metaStoreMaxFlushConcurrency = 10, .metaStoreMaxFlushBatchSize = 1 })); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); @@ -1433,18 +1630,17 @@ TEST_F(MetaStoreTest, BackupFlushBelowMaxConcurrency) EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillRepeatedly(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, - ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { + ::etcdserverpb::TxnResponse *txnResponse) -> ::grpc::Status { TimeLoop(1); - *response = ::etcdserverpb::TxnResponse{}; - response->mutable_header()->set_revision(1); + *txnResponse = ::etcdserverpb::TxnResponse{}; + txnResponse->mutable_header()->set_revision(1); return ::grpc::Status::OK; })); auto start = std::chrono::steady_clock::now(); std::vector>> futures; for (int i = 0; i < 10; i++) { auto transaction = client.BeginTransaction(); - transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); + transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value1" + std::to_string(i), PutOption{ .asyncBackup = false })); futures.emplace_back(transaction->Commit()); } for (auto &fut : futures) { @@ -1453,13 +1649,13 @@ TEST_F(MetaStoreTest, BackupFlushBelowMaxConcurrency) auto end = std::chrono::steady_clock::now(); EXPECT_TRUE(std::chrono::duration_cast(end - start).count() <= 10); litebus::Terminate(kvServerAccessorAID); - litebus::Await(kvServerAccessorAID); litebus::Terminate(kvServerAID); - litebus::Await(kvServerAID); - litebus::Terminate(backupAID); - litebus::Await(backupAID); litebus::Terminate(persistAID); + + litebus::Await(kvServerAccessorAID); + litebus::Await(kvServerAID); + litebus::Await(backupAID); litebus::Await(persistAID); } @@ -1469,22 +1665,22 @@ TEST_F(MetaStoreTest, BackupFlushAboveMaxConcurrency) litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); auto backupAID = litebus::Spawn(std::make_shared( "BackupActor", persistAID, - MetaStoreBackupOption{ .enableSyncSysFunc = false, .metaStoreMaxFlushConcurrency = 2, .metaStoreMaxFlushBatchSize = 1 })); + MetaStoreBackupOption{ .metaStoreMaxFlushConcurrency = 2, .metaStoreMaxFlushBatchSize = 1 })); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); - litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); + litebus::AID accessorAID = litebus::Spawn(std::make_shared(kvServerAID)); MetaStoreClient client(MetaStoreConfig{ .etcdAddress = etcdAddress_, - .metaStoreAddress = kvServerAccessorAID.Url(), + .metaStoreAddress = accessorAID.Url(), .enableMetaStore = true }, {}, MetaStoreTimeoutOption{}); client.Init(); EXPECT_CALL(*etcdKvService_, Txn) // test persistence .WillRepeatedly(Invoke([&](::grpc::ServerContext *, const ::etcdserverpb::TxnRequest *request, - ::etcdserverpb::TxnResponse *response) -> ::grpc::Status { + ::etcdserverpb::TxnResponse *rsp) -> ::grpc::Status { TimeLoop(1); - *response = ::etcdserverpb::TxnResponse{}; - response->mutable_header()->set_revision(1); + *rsp = ::etcdserverpb::TxnResponse{}; + rsp->mutable_header()->set_revision(1); return ::grpc::Status::OK; })); auto start = std::chrono::steady_clock::now(); @@ -1492,7 +1688,7 @@ TEST_F(MetaStoreTest, BackupFlushAboveMaxConcurrency) for (int i = 0; i < 10; i++) { auto transaction = client.BeginTransaction(); transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); + PutOption{ .asyncBackup = false })); futures.emplace_back(transaction->Commit()); } for (auto &fut : futures) { @@ -1500,8 +1696,8 @@ TEST_F(MetaStoreTest, BackupFlushAboveMaxConcurrency) } auto end = std::chrono::steady_clock::now(); EXPECT_TRUE(std::chrono::duration_cast(end - start).count() >= 4); - litebus::Terminate(kvServerAccessorAID); - litebus::Await(kvServerAccessorAID); + litebus::Terminate(accessorAID); + litebus::Await(accessorAID); litebus::Terminate(kvServerAID); litebus::Await(kvServerAID); @@ -1517,7 +1713,7 @@ TEST_F(MetaStoreTest, BackupFlushAsyncBack) litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); auto backupAID = litebus::Spawn(std::make_shared( "BackupActor", persistAID, - MetaStoreBackupOption{ .enableSyncSysFunc = false, .metaStoreMaxFlushConcurrency = 5, .metaStoreMaxFlushBatchSize = 2 })); + MetaStoreBackupOption{ .metaStoreMaxFlushConcurrency = 5, .metaStoreMaxFlushBatchSize = 2 })); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); litebus::AID kvServerAccessorAID = litebus::Spawn(std::make_shared(kvServerAID)); @@ -1539,11 +1735,11 @@ TEST_F(MetaStoreTest, BackupFlushAsyncBack) for (int i = 0; i < 10; i++) { auto transaction = client.BeginTransaction(); transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = true })); + PutOption{ .asyncBackup = true })); transaction->Then(TxnOperation::Create("key1" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = true })); + PutOption{ .asyncBackup = true })); transaction->Then(TxnOperation::Create("key2" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = true })); + PutOption{ .asyncBackup = true })); futures.emplace_back(transaction->Commit()); auto transaction1 = client.BeginTransaction(); transaction1->Then( @@ -1562,10 +1758,10 @@ TEST_F(MetaStoreTest, BackupFlushAsyncBack) litebus::Terminate(kvServerAID); litebus::Await(kvServerAID); - litebus::Terminate(backupAID); - litebus::Await(backupAID); litebus::Terminate(persistAID); litebus::Await(persistAID); + litebus::Terminate(backupAID); + litebus::Await(backupAID); } TEST_F(MetaStoreTest, BackupFlushRequestWithError) @@ -1574,7 +1770,7 @@ TEST_F(MetaStoreTest, BackupFlushRequestWithError) litebus::Spawn(std::make_shared("Persist", etcdAddress_, MetaStoreTimeoutOption{})); auto backupActor = std::make_shared( "BackupActor", persistAID, - MetaStoreBackupOption{ .enableSyncSysFunc = false, .metaStoreMaxFlushConcurrency = 10, .metaStoreMaxFlushBatchSize = 1 }); + MetaStoreBackupOption{ .metaStoreMaxFlushConcurrency = 10, .metaStoreMaxFlushBatchSize = 1 }); auto backupAID = litebus::Spawn(backupActor); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); @@ -1592,7 +1788,7 @@ TEST_F(MetaStoreTest, BackupFlushRequestWithError) for (int i = 0; i < 10; i++) { auto transaction = client.BeginTransaction(); transaction->Then(TxnOperation::Create("key" + std::to_string(i), "value" + std::to_string(i), - PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); + PutOption{ .asyncBackup = false })); futures.emplace_back(transaction->Commit()); } for (auto &fut : futures) { @@ -1612,12 +1808,8 @@ TEST_F(MetaStoreTest, BackupFlushRequestWithError) TEST_F(MetaStoreTest, BackupFailTest) { - auto persistAID = litebus::Spawn( - std::make_shared("Persist", etcdAddress_, - MetaStoreTimeoutOption{ .operationRetryIntervalLowerBound = 100, - .operationRetryIntervalUpperBound = 500, - .operationRetryTimes = 1, - .grpcTimeout = 0 })); + auto persistAID = litebus::Spawn(std::make_shared( + "Persist", etcdAddress_, MetaStoreTimeoutOption{ .operationRetryTimes = 1, .grpcTimeout = 0 })); auto backupAID = litebus::Spawn(std::make_shared("BackupActor", persistAID)); litebus::AID kvServerAID = litebus::Spawn(std::make_shared(backupAID)); @@ -1636,8 +1828,7 @@ TEST_F(MetaStoreTest, BackupFailTest) { auto transaction = client.BeginTransaction(); - transaction->Then( - TxnOperation::Create("key1", "value1", PutOption{ .leaseId = 0, .prevKv = false, .asyncBackup = false })); + transaction->Then(TxnOperation::Create("key1", "value1", PutOption{ .asyncBackup = false })); transaction->Commit().Get(); } @@ -1800,9 +1991,76 @@ TEST_F(MetaStoreTest, MetaStoreDriverest) // NOLINT // start with persist metaStoreDriver = std::make_shared(); - metaStoreDriver->Start(localAddress_); + metaStoreDriver->Start({localAddress_}); + + EXPECT_TRUE(metaStoreDriver->Stop().IsOk()); + metaStoreDriver->Await(); + auto monitor = MetaStoreMonitorFactory::GetInstance().GetMonitor(localAddress_); + litebus::Future fut = monitor->ClearObservers(); + ASSERT_AWAIT_READY(fut); + + // start with passthrough + metaStoreDriver = std::make_shared(); EXPECT_TRUE(metaStoreDriver->Stop().IsOk()); metaStoreDriver->Await(); } + +TEST_F(MetaStoreTest, KvServiceActorDuplicateOngoingTest) +{ + auto kvActor = std::make_shared(); + + // Put + { + etcdserverpb::PutRequest request; + request.set_key("key"); + request.set_value("1"); + etcdserverpb::PutResponse response; + + // check + EXPECT_TRUE(kvActor->CheckUniqueOngoingValid(request.key())); + // try run + auto puts = kvActor->TryPut(&request, &response); + auto kv = puts.first; + auto preKv = puts.second; + EXPECT_EQ(kv.key(), request.key()); + EXPECT_EQ(kv.value(), request.value()); + // insert + kvActor->InsertUniqueOngoing(etcdserverpb::RequestOp::RequestCase::kRequestPut, kv, preKv); + // check failed: duplicate key + EXPECT_FALSE(kvActor->CheckUniqueOngoingValid(request.key())); + // callback + kvActor->OnTryPutCache(request.key(), Status::OK()); + // check + EXPECT_TRUE(kvActor->CheckUniqueOngoingValid(request.key())); + } + + // Delete + { + etcdserverpb::PutRequest putRequest; + putRequest.set_key("key2"); + putRequest.set_value("2"); + etcdserverpb::PutResponse getResponse; + kvActor->Put(&putRequest, &getResponse); + + etcdserverpb::DeleteRangeRequest request; + request.set_key("key2"); + etcdserverpb::DeleteRangeResponse response; + // try run + auto deletes = kvActor->TryDeleteRange(&request, &response); + EXPECT_EQ(deletes->size(), static_cast(1)); + EXPECT_EQ(deletes->front().key(), request.key()); + + // check + EXPECT_TRUE(kvActor->CheckDeletesOngoingValid(deletes)); + // insert + kvActor->InsertDeleteOngoing(deletes); + // check failed: duplicate ongoing + EXPECT_FALSE(kvActor->CheckUniqueOngoingValid(deletes->front().key())); + // callbak + kvActor->OnTryDeleteRange(deletes, Status::OK()); + // check + EXPECT_TRUE(kvActor->CheckDeletesOngoingValid(deletes)); + } +} } // namespace functionsystem::meta_store::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/resource_group_manager/resource_group_manager_actor_test.cpp b/functionsystem/tests/unit/function_master/resource_group_manager/resource_group_manager_actor_test.cpp index 4049b8b708f4a96a8066c9c04e5de9b48b0462b6..b9e907bdfe8d78a1d2e08aacff6b2f2c40679761 100644 --- a/functionsystem/tests/unit/function_master/resource_group_manager/resource_group_manager_actor_test.cpp +++ b/functionsystem/tests/unit/function_master/resource_group_manager/resource_group_manager_actor_test.cpp @@ -1,6 +1,18 @@ /* -* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -*/ + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include @@ -147,17 +159,17 @@ public: class ResourceGroupManagerActorTest : public ::testing::Test { protected: - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); etcdSrvDriver_->StartServer(metaStoreServerHost_); - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); localAddress_ = "127.0.0.1:" + std::to_string(port); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } diff --git a/functionsystem/tests/unit/function_master/scaler/CMakeLists.txt b/functionsystem/tests/unit/function_master/scaler/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..3233b5f7044725c84bb73b9997250d245cc7c39a --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} SCALER_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${SCALER_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/scaler/parse_helper_test.cpp b/functionsystem/tests/unit/function_master/scaler/parse_helper_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..88d0344bb8a530363cb91f67593ddfe82776890a --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/parse_helper_test.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "function_master/scaler/utils/parse_helper.h" + +namespace functionsystem::test { + +class ParseHelperTest : public ::testing::Test {}; + +TEST_F(ParseHelperTest, TruncateIllegalLabel) +{ + std::string input(63, 'a'); + std::string expected = input; + std::string actual = TruncateIllegalLabel(input); + EXPECT_EQ(actual, expected); + + std::string input2(66, 'a'); + std::string expected2(63, 'a'); + std::string actual2 = TruncateIllegalLabel(input2); + EXPECT_EQ(actual, expected); +} + +TEST_F(ParseHelperTest, ParseCPUEnvTest) +{ + std::string input = "100m"; + auto actual = ParseCPUEnv(input); + EXPECT_EQ(actual, "100"); + + input = "100.1m"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, "100.1"); + + input = "0"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, ""); + + input = "0.5"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, "500"); + + input = "1.51"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, "1510"); + + input = "-1"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, ""); + + input = "10"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, "10000"); + + input = "10000000000000000"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, ""); + + input = "xxx"; + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, ""); + + input = std::to_string(INT_MAX/1000 + 1); + actual = ParseCPUEnv(input); + EXPECT_EQ(actual, ""); +} + +TEST_F(ParseHelperTest, ParseMemoryEnv) +{ + std::string input = "100Mi"; + auto actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, "100"); + + input = "100.1Mi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, "100.1"); + + input = "0Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, ""); + + input = "-1Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, ""); + + input = "1Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, "1024"); + + input = "0.5Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, "512"); + + input = "1.5Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, "1536"); + + input = "10000000000000000Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, ""); + + input = "xxx"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, ""); + + input = std::to_string(INT_MAX/1000 + 1) + "Gi"; + actual = ParseMemoryEnv(input); + EXPECT_EQ(actual, ""); +} +} \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/scaler/pool_manager_test.cpp b/functionsystem/tests/unit/function_master/scaler/pool_manager_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1d05c2d361ea9bb482ff9c2eed7f5b233e04ac07 --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/pool_manager_test.cpp @@ -0,0 +1,386 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "function_master/scaler/pool/pool_manager.h" + +namespace functionsystem::scaler::test { + +const std::string POD_POOL_INFO_STR = R"( +{ + "id": "pool1", + "size": 1, + "max_size": 1, + "scalable": false, + "group": "rg1", + "reuse": true, + "image": "runtime-manager:version1", + "init_image": "function-agent-init:version1", + "labels": { + "label1":"val1" + }, + "environment": { + "env1": "key1", + "env2": "key2" + }, + "volumes": "", + "volume_mounts": "", + "resources": { + "limits": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + }, + "requests": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + } + }, + "idle_recycle_time": { + "reserved": -1, + "scaled": 1 + }, + "runtime_class_name": "runc", + "node_selector": { + "label1":"val1" + }, + "tolerations": "", + "affinities": "", + "topology_spread_constraints": "" +} +)"; + +class PoolManagerTest : public ::testing::Test { +protected: + void SetUp() override + { + poolManager_ = std::make_shared(nullptr); + } + + void TearDown() override + { + } + + std::shared_ptr poolManager_; +}; + +std::shared_ptr GenNewPod(const std::string &podName) +{ + auto pod1 = std::make_shared(); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName(podName); + pod1->GetMetadata()->SetLabels({}); + pod1->GetMetadata()->SetAnnotations({}); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetContainerStatuses({ std::make_shared() }); + pod1->GetStatus()->GetContainerStatuses().front()->SetContainerID("runtime-manager"); + pod1->GetStatus()->SetPhase("Running"); + return pod1; +} + +std::shared_ptr GenNewDeployment(const std::string &deploymentName) +{ + auto deployment1 = std::make_shared(); + deployment1->SetMetadata(std::make_shared()); + deployment1->GetMetadata()->SetName(deploymentName); + deployment1->GetMetadata()->SetUid(deploymentName); + deployment1->SetSpec(std::make_shared()); + deployment1->GetSpec()->SetRTemplate(std::make_shared()); + deployment1->GetSpec()->GetRTemplate()->SetMetadata(std::make_shared()); + deployment1->GetSpec()->GetRTemplate()->GetMetadata()->SetName(deploymentName); + deployment1->GetSpec()->GetRTemplate()->GetMetadata()->SetLabels({{"app", deploymentName}}); + deployment1->GetSpec()->GetRTemplate()->GetMetadata()->SetAnnotations({}); + return deployment1; +} + +TEST_F(PoolManagerTest, ParsePodPool) +{ + auto podPool = poolManager_->GetOrNewPool("pool1"); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + EXPECT_EQ(podPool->maxSize, 1); + EXPECT_EQ(podPool->scalable, false); + EXPECT_EQ(podPool->idleRecycleTime.reserved, -1); + EXPECT_EQ(podPool->idleRecycleTime.scaled, 1); +} + +TEST_F(PoolManagerTest, PodEvent) +{ + auto persistentCounter = std::make_shared>(0); + poolManager_->RegisterPersistHandler([cnt(persistentCounter)](const std::string &poolID) { + (*cnt)++; + }); + auto scaleUpCounter = std::make_shared>(0); + poolManager_->RegisterScaleUpHandler([cnt(scaleUpCounter)](const std::string &poolID, bool isReserved) { + (*cnt)++; + }); + auto podPool = poolManager_->GetOrNewPool("pool1"); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + { + // pod pool is not scalable + podPool->scalable = false; + auto pod1 = GenNewPod("function-agent-pool1-1"); + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 0); + pod1->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pool2"; + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 0); + pod1->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pool1"; + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 0); + } + { + // pod pool is scalable + podPool->scalable = true; + podPool->size = 2; + podPool->maxSize = 3; + podPool->readyCount = 0; + podPool->status = static_cast(PoolState::CREATING); + auto pod1 = GenNewPod("function-agent-pool1-1"); + pod1->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pool1"; + poolManager_->OnPodUpdate(pod1); + + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->status, static_cast(PoolState::CREATING)); + EXPECT_EQ(*persistentCounter, 1); + // put same pod again + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 1); + EXPECT_EQ(*persistentCounter, 1); + EXPECT_EQ(*scaleUpCounter, 2); + // put new pod + auto pod2 = GenNewPod("function-agent-pool1-2"); + pod2->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pool1"; + poolManager_->OnPodUpdate(pod2); + EXPECT_EQ(*persistentCounter, 2); + EXPECT_EQ(*scaleUpCounter, 3); + auto pod3 = GenNewPod("function-agent-pool1-3"); + pod3->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pool2"; + poolManager_->OnPodUpdate(pod3); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 2); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->status, static_cast(PoolState::RUNNING)); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyPodSet.size(), 2); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 0); + EXPECT_EQ(*persistentCounter, 2); + EXPECT_EQ(*scaleUpCounter, 3); + // delete pod + poolManager_->OnPodDelete(pod3); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 2); + EXPECT_EQ(*persistentCounter, 2); + poolManager_->OnPodDelete(pod2); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 1); + EXPECT_EQ(*persistentCounter, 3); + // test terminating pod + EXPECT_FALSE(IsPodTerminating(pod1)); + pod1->SetStatus(std::make_shared()); + EXPECT_FALSE(IsPodTerminating(pod1)); + pod1->GetStatus()->SetContainerStatuses({ std::make_shared() }); + EXPECT_FALSE(IsPodTerminating(pod1)); + pod1->GetStatus()->GetContainerStatuses().front()->SetContainerID("test"); + EXPECT_FALSE(IsPodTerminating(pod1)); + pod1->GetStatus()->GetContainerStatuses().front()->SetState(std::make_shared()); + EXPECT_FALSE(IsPodTerminating(pod1)); + pod1->GetStatus()->GetContainerStatuses().front()->GetState()->SetTerminated(std::make_shared()); + EXPECT_TRUE(IsPodTerminating(pod1)); + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 0); + EXPECT_EQ(*persistentCounter, 4); + EXPECT_EQ(*scaleUpCounter, 5); + } + +} + +TEST_F(PoolManagerTest, PendingPodEvent) +{ + { + // pod pool is scalable, pod is pending + auto podPool1 = poolManager_->GetOrNewPool("pending-pool1"); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool1); + podPool1->scalable = true; + podPool1->size = 2; + podPool1->maxSize = 3; + podPool1->readyCount = 0; + podPool1->status = static_cast(PoolState::CREATING); + auto pod1 = GenNewPod("function-agent-pending-pool1-1"); + pod1->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pending-pool1"; + pod1->GetStatus()->SetPhase("Pending"); + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->readyCount, 0); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->pendingCreatePodSet.size(), 1); + pod1->GetStatus()->SetPhase("Running"); + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->readyCount, 1); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->pendingCreatePodSet.size(), 0); + auto pod2 = GenNewPod("function-agent-pending-pool1-2"); + pod2->GetMetadata()->GetAnnotations()["yr-pod-pool"] = "pending-pool1"; + poolManager_->OnPodUpdate(pod2); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->readyCount, 2); + EXPECT_EQ(poolManager_->GetPodPool("pending-pool1")->status, static_cast(PoolState::RUNNING)); + } +} + +TEST_F(PoolManagerTest, TryScaleUpPod) +{ + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", false).IsNone()); + auto podPool = poolManager_->GetOrNewPool("pool1"); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + { + podPool->scalable = false; + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + } + { + podPool->scalable = true; + podPool->status = static_cast(PoolState::NEW); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + podPool->status = static_cast(PoolState::FAILED); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + podPool->status = static_cast(PoolState::DELETED); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + } + { + podPool->scalable = true; + podPool->status = static_cast(PoolState::RUNNING); + // count reach maxSize + podPool->size = 1; + podPool->maxSize = 2; + podPool->pendingCreatePodSet.emplace("function-agent-pool1-1"); + podPool->readyPodSet.emplace("function-agent-pool1-2"); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + // isReserved and count reach size + podPool->maxSize = 3; + podPool->size = 2; + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + // failed to get deployment + podPool->pendingCreatePodSet.clear(); + EXPECT_TRUE(poolManager_->TryScaleUpPod("pool1", true).IsNone()); + } + { + podPool->scalable = true; + podPool->status = static_cast(PoolState::RUNNING); + // count reach maxSize + podPool->size = 1; + podPool->maxSize = 2; + podPool->pendingCreatePodSet.clear(); + podPool->readyPodSet.clear(); + auto deployment1 = GenNewDeployment("function-agent-pool1"); + deployment1->GetSpec()->GetRTemplate()->GetMetadata()->GetLabels()["key1"] = "val1"; + deployment1->GetSpec()->GetRTemplate()->GetMetadata()->GetAnnotations()["key1"] = "val1"; + poolManager_->PutDeployment(deployment1); + // scale reserved + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 0); + auto pod = poolManager_->TryScaleUpPod("pool1", true).Get(); + EXPECT_EQ(pod->GetMetadata()->GetLabels()["yr-idle-to-recycle"], "unlimited"); + EXPECT_EQ(pod->GetMetadata()->GetAnnotations()["yr-pod-pool"], "pool1"); + EXPECT_EQ(pod->GetMetadata()->GetAnnotations()["key1"], "val1"); + EXPECT_EQ(pod->GetMetadata()->GetLabels()["key1"], "val1"); + EXPECT_EQ(pod->GetMetadata()->GetOwnerReferences()[0]->GetName(), "function-agent-pool1"); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 1); + // scale scaled + auto pod1 = poolManager_->TryScaleUpPod("pool1", false).Get(); + EXPECT_EQ(pod1->GetMetadata()->GetLabels()["yr-idle-to-recycle"], "1"); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 2); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 0); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetContainerStatuses({ std::make_shared() }); + pod1->GetStatus()->GetContainerStatuses().front()->SetContainerID("runtime-manager"); + pod1->GetStatus()->SetPhase("Running"); + poolManager_->OnPodUpdate(pod1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 1); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 1); + pod->SetStatus(std::make_shared()); + pod->GetStatus()->SetContainerStatuses({ std::make_shared() }); + pod->GetStatus()->GetContainerStatuses().front()->SetContainerID("runtime-manager"); + pod->GetStatus()->SetPhase("Running"); + poolManager_->OnPodUpdate(pod); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->pendingCreatePodSet.size(), 0); + EXPECT_EQ(poolManager_->GetPodPool("pool1")->readyCount, 2); + } +} + +TEST_F(PoolManagerTest, ValidatePodPoolCreateParams) +{ + // PodId + auto podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->id = "ERROR_ID"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // PodPoolGroup + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->group = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // Size + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->size = -1; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // MaxSize + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->maxSize = 1; + podPool->size = 2; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // HorizontalPodAutoscalerSpec + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->maxSize = 1; + podPool->size = 0; + podPool->horizontalPodAutoscalerSpec = + R"({"minReplicas": 1, "maxReplicas": 2, "metrics":[{"resource": {"name":"cpu", "target":{"averageUtilization":20, "type":"Utilization"}}, "type":"Resource"}, {"resource": {"name":"memory", "target":{"averageUtilization":50, "type":"Utilization"}}, "type":"Resource"}]})"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // IdleRecycleTime.Scaled + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->idleRecycleTime.scaled = -2; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // IdleRecycleTime.Reserved + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->idleRecycleTime.reserved = -2; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // Image (lenth = 201) + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->image = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaa"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // InitImage (lenth = 201) + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->initImage = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaa"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // RuntimeClassName (lenth = 65) + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->runtimeClassName = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // PodPendingDurationThreshold + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + podPool->podPendingDurationThreshold = -1; + EXPECT_FALSE(PoolManager::ValidatePodPoolCreateParams(podPool)); + // Success + podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, podPool); + EXPECT_TRUE(PoolManager::ValidatePodPoolCreateParams(podPool)); +} +} \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/scaler/scaler_driver_test.cpp b/functionsystem/tests/unit/function_master/scaler/scaler_driver_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c13cb4ae9880fb97be63646a12346ee26ecb4cda --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/scaler_driver_test.cpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "function_master/scaler/scaler_driver.h" + +#include + +#include "common/etcd_service/etcd_service_driver.h" +#include "meta_store_client/meta_store_struct.h" +#include "function_master/common/flags/flags.h" +#include "utils/port_helper.h" + +namespace functionsystem::scaler::test { + +const GrpcSslConfig sslConfig{}; + +class ScalerDriverTest : public ::testing::Test { +protected: + inline static std::unique_ptr etcdSrvDriver_; + inline static std::string metaStoreServerHost_; + + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + etcdSrvDriver_->StopServer(); + } + + void SetUp() override + { + } + + void TearDown() override + { + } + + ScalerHandlers handlers{ .systemUpgradeHandler = [](bool isUpgrading) {}, + .localSchedFaultHandler = [](const std::string &nodeName) {} }; +}; + +TEST_F(ScalerDriverTest, DriverStartNoK8s) +{ + const char *argv2[] = { "./function_master", + R"(--log_config={ + "filepath":"/home/yr/log", + "level":"DEBUG", + "rolling":{ + "maxsize":100, + "maxfiles":1 + } + })", + "--node_id=10", + "--ip=127.0.0.1", + "--d1=1", + "--d2=1", + "--meta_store_address=127.0.0.1", + "--k8s_base_path=" }; + + functionsystem::functionmaster::Flags flags; + flags.ParseFlags(8, argv2); + + auto metaStoreTimeoutOpt = MetaStoreTimeoutOption(); + metaStoreTimeoutOpt.operationRetryTimes = 2; + metaStoreTimeoutOpt.grpcTimeout = 1000; + + auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }, sslConfig, + metaStoreTimeoutOpt); + auto kubeClient = KubeClient::CreateKubeClient(flags.GetK8sBasePath(), KubeClient::ClusterSslConfig("", "", false)); + auto driver = std::make_shared(flags, client, client, kubeClient, handlers); + EXPECT_EQ(driver->Start(), StatusCode::SUCCESS); + + (void)driver->Stop(); + driver->Await(); +} + +TEST_F(ScalerDriverTest, DriverStartNoPool) +{ + const char *argv[] = { "./function_master", + R"(--log_config={ + "filepath":"/home/yr/log", + "level":"DEBUG", + "rolling":{ + "maxsize":100, + "maxfiles":1 + } + })", + "--node_id=10", + "--ip=127.0.0.1", + "--d1=1", + "--d2=1", + "--meta_store_address=127.0.0.1", + "--k8s_base_path=127.0.0.1:443", + "--skip_k8s_tls_verify=true" }; + + functionsystem::functionmaster::Flags flags; + flags.ParseFlags(9, argv); + + auto metaStoreTimeoutOpt = MetaStoreTimeoutOption(); + metaStoreTimeoutOpt.operationRetryTimes = 2; + metaStoreTimeoutOpt.grpcTimeout = 1000; + + auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }, sslConfig, + metaStoreTimeoutOpt); + auto kubeClient = KubeClient::CreateKubeClient(flags.GetK8sBasePath(), KubeClient::ClusterSslConfig("", "", false)); + auto driver = std::make_shared(flags, client, client, kubeClient, handlers); + EXPECT_EQ(driver->Start(), StatusCode::SUCCESS); + + (void)driver->Stop(); + driver->Await(); +} + +/** + * Feature: Start sclaer with driver + * Description: start sclaer + * Steps: + * 1. set up params + * 2. Start driver + * 3. Stop driver + * + * Expectation: + * 2. StatusCode::SUCCESS + * 3. StatusCode::SUCCESS + */ +TEST_F(ScalerDriverTest, DriverStart) +{ + (void)litebus::os::Mkdir("/home/sn/scaler/config/"); + std::string poolsStr = R"( + [ + { + "name":"pool1-500-500-a", + "poolSize":1, + "requestCpu":"500m", + "requestMemory":"500Mi", + "limitCpu":"500m", + "limitMemory":"500Mi" + }, + { + "name":"pool2-600-600-a", + "poolSize":1, + "requestCpu":"600m", + "requestMemory":"600Mi", + "limitCpu":"600m", + "limitMemory":"600Mi" + } + ] + )"; + Write("/home/sn/scaler/config/functionsystem-pools.json", poolsStr); + + const char *argv[] = { "./function_master", + R"(--log_config={ + "filepath":"/home/yr/log", + "level":"DEBUG", + "rolling":{ + "maxsize":100, + "maxfiles":1 + } + })", + "--node_id=10", + "--ip=127.0.0.1", + "--d1=1", + "--d2=1", + "--meta_store_address=127.0.0.1", + "--k8s_base_path=127.0.0.1:443", + "--skip_k8s_tls_verify=true", + "--taint_tolerance_list=unavailable", + "--worker_taint_exclude_labels=node-role=edge;", + "--system_upgrade_watch_enable=true", + "--az_id=0", + "--system_upgrade_key=/hms-caas/edgems/upgrade-zones", + "--system_upgrade_address=127.0.0.1", + "--system_upgrade_address=10" }; + + functionsystem::functionmaster::Flags flags; + flags.ParseFlags(11, argv); + + auto metaStoreTimeoutOpt = MetaStoreTimeoutOption(); + metaStoreTimeoutOpt.operationRetryTimes = 2; + metaStoreTimeoutOpt.grpcTimeout = 1000; + + auto client = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }, sslConfig, + metaStoreTimeoutOpt); + auto kubeClient = KubeClient::CreateKubeClient(flags.GetK8sBasePath(), KubeClient::ClusterSslConfig("", "", false)); + auto driver = std::make_shared(flags, client, client, kubeClient, handlers); + EXPECT_EQ(driver->Start(), StatusCode::SUCCESS); + + (void)driver->Stop(); + driver->Await(); + + (void)litebus::os::Rmdir("/home/sn/scaler"); +} + +} // namespace functionsystem::scaler::test diff --git a/functionsystem/tests/unit/function_master/scaler/scaler_test.cpp b/functionsystem/tests/unit/function_master/scaler/scaler_test.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c66a5b1a07539088e5127c7f031cf136f1e9164b 100644 --- a/functionsystem/tests/unit/function_master/scaler/scaler_test.cpp +++ b/functionsystem/tests/unit/function_master/scaler/scaler_test.cpp @@ -0,0 +1,3546 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include + +#include "common/constants/actor_name.h" +#include "common/constants/constants.h" +#include "common/etcd_service/etcd_service_driver.h" +#include "common/logs/logging.h" +#include "common/metrics/metrics_adapter.h" +#include "common/resource_view/view_utils.h" +#include "common/types/instance_state.h" +#include "common/utils/files.h" +#include "common/hex/hex.h" +#include "common/kube_client/api/apps_v1_api.h" +#include "common/kube_client/model/deployment/v1_deployment.h" +#include "common/kube_client/model/deployment/v1_deployment_status.h" +#include "function_master/scaler/scaler_actor.h" +#include "mocks/mock_kube_client.h" +#include "mocks/mock_meta_store_client.h" +#include "scaler_test_actor.h" +#include "utils/future_test_helper.h" +#include "utils/generate_info.h" +#include "utils/os_utils.hpp" +#include "utils/port_helper.h" + +namespace functionsystem::scaler::test { +using namespace functionsystem::test; + +using V1Deployment = functionsystem::kube_client::model::V1Deployment; +using Object = functionsystem::kube_client::model::Object; +using V1NodeStatus = functionsystem::kube_client::model::V1NodeStatus; +using V1NodeAddress = functionsystem::kube_client::model::V1NodeAddress; +using V1ObjectMeta = functionsystem::kube_client::model::V1ObjectMeta; +using V1NodeSpec = functionsystem::kube_client::model::V1NodeSpec; +using V1DeploymentStatus = functionsystem::kube_client::model::V1DeploymentStatus; + +const std::string DELEGATE_CONTAINER_STR = R"( +{ + "image": "image", + "env": [{ + "name": "env_x", + "value": "value_x" + }, { + "name": "env_y", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "status.podIP" + } + } + }, { + "name": "env_z", + "valueFrom": { + "resourceFieldRef": { + "containerName": "delegate_container", + "resource": "resource_x", + "divisor": "divisor_x" + } + } + }], + "command": { + "command1": "100", + "command2": "1" + }, + "workingDir": "workingDir", + "args": { + "args1": "100", + "args2": "1" + }, + "uid": 1003, + "gid": 1003, + "volumeMounts": [{ + "name": "uds", + "mountPath": "/home/uds", + "subPath": "uds sub path", + "subPathExpr": "sub path expr", + "mountPropagation": "mount propagation", + "readOnly": true + }, { + "mountPath": "xx" + }, { + "name": "xx" + } + ], + "livenessProbe": { + "exec":{ + "command":["liveness","probe"] + }, + "initialDelaySeconds": 100, + "timeoutSeconds": 200, + "periodSeconds": 300, + "successThreshold": 400, + "failureThreshold": 500 + }, + "readinessProbe": { + "exec":{ + "command":["readiness","probe"] + }, + "initialDelaySeconds": 1100, + "timeoutSeconds": 1200, + "periodSeconds": 1300, + "successThreshold": 1400, + "failureThreshold": 1500 + } +} +)"; + +const std::string POD_POOL_INFO_STR = R"( +{ + "id": "pool1", + "size": 1, + "group": "rg1", + "reuse": true, + "image": "runtime-manager:version1", + "init_image": "function-agent-init:version1", + "labels": { + "label1":"val1", + "":"" + }, + "environment": { + "env1": "key1", + "env2": "key2" + }, + "volumes": "[{\"name\": \"volume-1\", \"hostPath\": { \"path\": \"/home/xxx\",\"type\": \"DirectoryOrCreate\"}}, {\"name\": \"pvc-xx\", \"persistentVolumeClaim\": {\"caimName\": \"pvc-xxx\"}}]", + "volume_mounts": "[{\"name\": \"pvc-xx\",\"mountPath\": \"/home/snuser/models\"}]", + "resources": { + "limits": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + }, + "requests": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + } + }, + "runtime_class_name": "runc", + "node_selector": { + "label1":"val1" + }, + "tolerations": "[{\"key\":\"node.kubernetes.io/disk-pressure\",\"operator\": \"Equal\", \"value\": \"true\", \"effect\": \"NoSchedule\"}]", + "affinities": "{\"nodeAffinity\": {\"requiredDuringSchedulingIgnoredDuringExecution\": {\"nodeSelectorTerms\": [{\"matchExpressions\": [{ \"key\": \"node-type\",\"operator\": \"In\",\"values\": [\"system\"]}]}]}}}", + "topology_spread_constraints": "[{\"maxSkew\":1,\"minDomains\":1,\"topologyKey\":\"zone\",\"whenUnsatisfiable\":\"DoNotSchedule\", \"matchLabelKeys\": [\"test1\",\"test2\"],\"labelSelector\":{\"matchLabels\": {\"app\":\"foo\"}} }]" +} +)"; + +const std::string POD_POOL_WITH_HPA_INFO_STR = R"( +{ + "id": "pool1", + "size": 1, + "group": "rg1", + "reuse": true, + "image": "runtime-manager:version1", + "init_image": "function-agent-init:version1", + "labels": { + "label1":"val1", + "":"" + }, + "environment": { + "env1": "key1", + "env2": "key2" + }, + "volumes": "[{\"name\": \"volume-1\", \"hostPath\": { \"path\": \"/home/xxx\",\"type\": \"DirectoryOrCreate\"}}, {\"name\": \"pvc-xx\", \"persistentVolumeClaim\": {\"caimName\": \"pvc-xxx\"}}]", + "volume_mounts": "[{\"name\": \"pvc-xx\",\"mountPath\": \"/home/snuser/models\"}]", + "resources": { + "limits": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + }, + "requests": { + "cpu": "2", + "memory": "6Gi", + "huawei.com/ascend-1980": "1" + } + }, + "affinities": "{\"nodeAffinity\": {\"requiredDuringSchedulingIgnoredDuringExecution\": {\"nodeSelectorTerms\": [{\"matchExpressions\": [{ \"key\": \"node-type\",\"operator\": \"In\",\"values\": [\"system\"]}]}]}},\"podAntiAffinity\": {\"preferredDuringSchedulingIgnoredDuringExecution\": [{\"weight\": 1,\"podAffinityTerm\": {\"labelSelector\": {\"matchExpressions\": [{\"key\": \"app\",\"operator\": \"In\",\"values\": [\"function-agent-pool-1\"]}]},\"topologyKey\": \"kubernetes.io/hostname\"}}]}}", + "horizontal_pod_autoscaler_spec": "{\"minReplicas\": 1, \"maxReplicas\": 2, \"metrics\":[{\"resource\": {\"name\":\"cpu\", \"target\":{\"averageUtilization\":20, \"type\":\"Utilization\"}}, \"type\":\"Resource\"}, {\"resource\": {\"name\":\"memory\", \"target\":{\"averageUtilization\":50, \"type\":\"Utilization\"}}, \"type\":\"Resource\"}]}" +} +)"; + +const std::string POD_POOL_TEST_STR = R"( +{ + "id": "pool1", + "group": "rg1", + "resources": { + "limits": { + "cpu": "500m", + "memory": "500Mi" + }, + "requests": { + "cpu": "500m", + "memory": "500Mi" + } + }, + "size": 1, + "reuse": true +} +)"; + +const std::string DEFAULT_DEPLOYMENT_JSON_STR = R"( +{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": { + "name": "deploy-001" + }, + "spec": { + } +} +)"; +const std::string NODE_AFFINITY_TEMPLATE_STR = R"( +{ + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{ + "matchExpressions": [{ + "key": "key1", + "operator": "In", + "values": ["val1"] + }, { + "key": "key2", + "operator": "In", + "values": ["val2"] + }], + "matchFields": [{ + "key": "key1", + "operator": "In", + "values": ["val1"] + }, { + "key": "key2", + "operator": "In", + "values": ["val2"] + }] + }, { + "matchExpressions": [{ + "key": "key3", + "operator": "In", + "values": ["val3"] + }, { + "key": "key3", + "operator": "In", + "values": ["val3"] + }], + "matchFields": [{ + "key": "key4", + "operator": "In", + "values": ["val4"] + }, { + "key": "key4", + "operator": "In", + "values": ["val4"] + }] + }] + }, + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "preference": { + "matchExpressions": [{ + "key": "node.kubernetes.io/instance-type", + "operator": "In", + "values": ["physical.kat2ne.48xlarge.8.376t.ei.c002.ondemand", "physical.kat2ne.48xlarge.8.ei.pod101.ondemand"] + }] + }, + "weight": 1 + }] +} +)"; + +const std::string instanceKey1 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/requestID1/instanceID1)"; +const std::string instanceInfoJson1 = R"({"scheduleOption":{"schedPolicyName":"monopoly"},"instanceID":"instanceID1","requestID":"requestID1","runtimeID":"runtime-1","runtimeAddress":"10.42.2.101","functionAgentID":"functionagent-pool1-776c6db574-1","functionProxyID":"siaphisprg00911","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":1,"msg":"scheduling"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + +const std::string instanceKey2 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/requestID2/instanceID2)"; +const std::string instanceInfoJson2 = R"({"scheduleOption":{"schedPolicyName":"monopoly"},"instanceID":"instanceID2","requestID":"requestID2","runtimeID":"runtime-2","runtimeAddress":"10.42.2.102","functionAgentID":"functionagent-pool1-776c6db574-2","functionProxyID":"siaphisprg00912","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":2,"msg":"creating"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + +const std::string instanceKey3 = R"(/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/requestID3/instanceID3)"; +const std::string instanceInfoJson3 = R"({"instanceID":"instanceID3","requestID":"requestID3","runtimeID":"runtime-3","runtimeAddress":"10.42.2.103","functionAgentID":"functionagent-pool1-776c6db574-3","functionProxyID":"siaphisprg00913","function":"12345678901234561234567890123456/0-system-faasExecutorPython3.9/$latest","scheduleTimes":1,"instanceStatus":{"code":3,"msg":"running"},"jobID":"job-12345678","parentID":"4e7cd507-8645-4600-b33c-f045f13e4beb","deployTimes":1,"version":"1"})"; + + +class ScalerTest : public ::testing::Test { +protected: + void SetUp() override + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + auto client = std::make_shared(); + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); + metaStorageAccessor_ = std::make_shared(metaStoreClient_); + counter = std::make_shared>(0); + ScalerParams scalerParams{ .k8sNamespace = DEFAULT_NAMESPACE, + .gracePeriodSeconds = 25, + .systemUpgradeParam = { + .isEnabled = true, + .systemUpgradeKey = DEFAULT_SYSTEM_UPGRADE_KEY, + .azID = 1, + .systemUpgradeWatcher = metaStorageAccessor_, + .handlers = { .systemUpgradeHandler = [](bool isUpgrading) {}, + .localSchedFaultHandler = [](const std::string &nodeName) {}, + .evictAgentHandler = [cnt(counter)](const std::string &localID, const std::shared_ptr &req){ + (*cnt)++; + return Status::OK(); + }} + + } }; + scalerParams.poolConfigPath = "/home/sn/scaler/config/functionsystem-pools.json"; + scalerParams.agentTemplatePath = "/home/sn/scaler/template/function-agent.json"; + scalerParams.enableFrontendPool = true; + actor_ = std::make_shared(SCALER_ACTOR, client, metaStorageAccessor_, scalerParams); + litebus::Spawn(actor_); + + litebus::Async(actor_->GetAID(), &ScalerActor::UpdateLeaderInfo, GetLeaderInfo(actor_->GetAID())); + + testActor_ = std::make_shared("ScalerTestActor"); + litebus::Spawn(testActor_); + + pools_ = std::make_shared>(); + scaler::ResourcePool pool1{ .name = "600-600-1pool", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "600m" }, { MEMORY_RESOURCE, "600Mi" } }, + .limitResources = { { CPU_RESOURCE, "600m" }, { MEMORY_RESOURCE, "600Mi" } } }; + pools_->push_back(pool1); + scaler::ResourcePool pool2{ .name = "500-500-2pool", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "500m" }, { MEMORY_RESOURCE, "500Mi" } }, + .limitResources = { { CPU_RESOURCE, "500m" }, { MEMORY_RESOURCE, "500Mi" } } }; + pools_->push_back(pool2); + + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + // mock template deployment + actor_->SetTemplateDeployment(localVarResult); + } + + void TearDown() override + { + metaStorageAccessor_->metaClient_ = metaStoreClient_; // for replace origin client while using mockClient + + litebus::Terminate(actor_->GetAID()); + litebus::Terminate(testActor_->GetAID()); + + litebus::Await(actor_); + litebus::Await(testActor_); + + // delete all data after terminate actor. Otherwise, the test case occasionally fails. + metaStorageAccessor_->Delete("/", true).Get(); + + metaStoreClient_ = nullptr; + metaStorageAccessor_ = nullptr; + actor_ = nullptr; + testActor_ = nullptr; + etcdSrvDriver_->StopServer(); + etcdSrvDriver_ = nullptr; + } + +protected: + std::unique_ptr etcdSrvDriver_; + std::shared_ptr actor_; + std::shared_ptr metaStoreClient_; + std::shared_ptr metaStorageAccessor_; + std::shared_ptr testActor_; + std::shared_ptr> pools_; + std::vector> podPools_; + std::shared_ptr> counter; + std::string metaStoreServerHost_; + + static resource_view::InstanceInfo CreateInstance(const std::string &id, int code) + { + resource_view::InstanceInfo output; + output.set_instanceid(id); + output.set_requestid(INSTANCE_PATH_PREFIX + "/001"); + output.set_functionagentid("function-agent-001"); + output.mutable_instancestatus()->set_code(code); + output.mutable_scheduleoption()->set_schedpolicyname("monopoly"); + output.mutable_labels()->Add("label"); + return output; + } + + messages::CreateAgentResponse GetTestResponse() + { + return litebus::Async(testActor_->GetAID(), &ScalerTestActor::GetResponse).Get(); + } + + messages::UpdateNodeTaintResponse GetUpdateNodeTaintResponse() + { + return litebus::Async(testActor_->GetAID(), &ScalerTestActor::GetTaintResponse).Get(); + } + + static std::shared_ptr GetReadyPod() + { + auto pod = std::make_shared(); + auto containerStatus = std::make_shared(); + containerStatus->SetContainerID("docker://1234567"); + containerStatus->SetName("runtime"); + containerStatus->SetReady(true); + std::vector> statuses; + statuses.push_back(containerStatus); + auto status = std::make_shared(); + status->SetHostIP("127.0.0.1"); + status->SetContainerStatuses(statuses); + pod->SetStatus(status); + auto metaData = std::make_shared(); + metaData->SetUid("uid-123"); + metaData->SetName("function-agent-123"); + metaData->SetLabels({}); + metaData->GetLabels()["yr-reuse"] = "false"; + pod->SetMetadata(metaData); + auto podSpec = std::make_shared(); + podSpec->SetNodeName("node-001"); + pod->SetSpec(podSpec); + return pod; + } + + static std::shared_ptr GetReadyNode() + { + auto address = std::make_shared(); + address->SetType("InternalIP"); + address->SetAddress("127.0.0.1"); + auto status = std::make_shared(); + status->SetAddresses({ address }); + auto node = std::make_shared(); + auto spec = std::make_shared(); + node->SetSpec(spec); + node->GetSpec()->SetTaints({}); + node->SetStatus(status); + auto metadata = std::make_shared(); + metadata->SetName("127.0.0.1"); + node->SetMetadata(metadata); + return node; + } + + std::vector> GetTaintNodeList() + { + std::string taintKey = "test-taint"; + actor_->member_->proxyTaintKey = taintKey; + std::vector> nodeList; + auto node1 = GetReadyNode(); + node1->GetMetadata()->SetName("node1"); + std::vector> taints; + taints.emplace_back(std::make_shared()); + auto taint = std::make_shared(); + taint->SetKey("test-taint"); + taints.emplace_back(taint); + node1->GetSpec()->SetTaints(taints); + actor_->member_->nodes["node1"] = node1; + nodeList.emplace_back(node1); + auto node2 = GetReadyNode(); + node2->GetMetadata()->SetName("node2"); + actor_->member_->nodes["node2"] = node2; + nodeList.emplace_back(node2); + return nodeList; + } + + static bool ContainValue(const std::shared_ptr &body, const std::string &value) + { + YRLOG_INFO("body value is {}", body->ToJson().dump()); + return body->ToJson().dump().find(value) != std::string::npos; + } + + static ::resources::Resources GetResources() + { + ::resources::Resources resources; + ::resources::Resource cpuResource; + cpuResource.set_name("CPU"); + cpuResource.mutable_scalar()->set_value(500); + cpuResource.mutable_scalar()->set_limit(500); + ::resources::Resource memoryResource; + memoryResource.set_name("Memory"); + memoryResource.mutable_scalar()->set_value(500); + memoryResource.mutable_scalar()->set_limit(500); + resources.mutable_resources()->operator[]("CPU").CopyFrom(cpuResource); + resources.mutable_resources()->operator[]("Memory").CopyFrom(memoryResource); + return resources; + } + + // if etcd watch is create slowly, sometimes it will trigger etcd event twice + void WaitRegisterComplete() + { + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->member_->metaStorageAccessor->metaClient_->metaStoreClientMgr_->GetKvClient("")->readyRecords_.size() >= 3; }); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + litebus::Promise> errPromise; + errPromise.SetFailed(404); + bool isRead = false; + EXPECT_CALL(*mockClient, ReadNamespacedPod) + .WillRepeatedly(testing::DoAll(testing::Assign(&isRead, true), testing::Return(errPromise.GetFuture()))); + resource_view::InstanceInfo instance = + CreateInstance(INSTANCE_PATH_PREFIX + "/ins", static_cast(InstanceState::FATAL)); + instance.set_functionagentid("function-agent-not-found"); + instance.mutable_scheduleoption()->set_schedpolicyname(""); + std::string insJson; + TransToJsonFromInstanceInfo(insJson, instance); + metaStorageAccessor_->Put(INSTANCE_PATH_PREFIX + "/ins", insJson).Get(); + EXPECT_AWAIT_TRUE([&isRead]() -> bool { return isRead; }); + } + + static std::vector GetWatchEvents(const std::string &key, const std::string &jsonStr) + { + KeyValue kv; + kv.set_key(key); + kv.set_value(jsonStr); + KeyValue prevKv; + auto event = WatchEvent{ EventType::EVENT_TYPE_PUT, kv, prevKv }; + std::vector events{ event }; + return events; + } + + static litebus::Future> GetErrorDeploymentFuture(int errorCode) + { + litebus::Promise> errPromise; + errPromise.SetFailed(errorCode); + return errPromise.GetFuture(); + } +}; + +/** + * Test instance deleted before pod ready + * + * Steps: + * 1. mock k8s client, templates + * 2. create agent + * 3. mock event, delete instance id + * 4. mock event, pod is ready + * + * Expect: + * 1. delete pod is called + */ +TEST_F(ScalerTest, CreatePodAutoCleaning) +{ + // mock k8s client + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + // mock template deployment + actor_->SetTemplateDeployment(localVarResult); + + // make a suitable create agent request + auto req = std::make_shared(); + auto instanceinfo = view_utils::Get1DInstance(); + instanceinfo.set_requestid("requestid123"); + instanceinfo.set_instanceid("instanceid123"); + req->mutable_instanceinfo()->CopyFrom(instanceinfo); + req->mutable_instanceinfo()->mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + + litebus::Future> createPodCalledArg; + EXPECT_CALL(*mockClient, CreateNamespacedPod) + .WillOnce(testing::DoAll(test::FutureArg<1>(&createPodCalledArg), testing::Return(GetReadyPod()))); + + litebus::Future deletePodCalledArg; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(test::FutureArg<0>(&deletePodCalledArg), testing::Return(GetReadyPod()))) + .WillOnce(testing::DoAll(test::FutureArg<0>(&deletePodCalledArg), testing::Return(GetReadyPod()))); + + // Checkpoint: check craete pod request is called + litebus::Future listPodCalledArg; + + testActor_->CreateAgent(actor_->GetAID(), req->SerializeAsString()); + + // Checkpoint: check create pod request is called + ASSERT_AWAIT_READY(createPodCalledArg); + req->mutable_instanceinfo()->set_functionagentid(createPodCalledArg.Get()->GetMetadata()->GetName()); + actor_->HandleInstanceDelete(req->instanceinfo()); + + // mock pod is ready event + auto pod = GetReadyPod(); + pod->GetMetadata()->SetName(createPodCalledArg.Get()->GetMetadata()->GetName()); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_ADD, pod); + + // Expect: delete pod is called + ASSERT_AWAIT_READY(deletePodCalledArg); + EXPECT_EQ(deletePodCalledArg.Get(), createPodCalledArg.Get()->GetMetadata()->GetName()); + litebus::Async(actor_->GetAID(), &ScalerActor::OnPodModified, K8sEventType::EVENT_TYPE_DELETE, pod); + ASSERT_AWAIT_TRUE([&]() -> bool {return actor_->member_->podNameMap.size() == 0;}); +} + +TEST_F(ScalerTest, CreatePodAutoCleaningWithSameInstanceID) +{ + // mock k8s client + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + localVarResult->FromJson(localVarJson); + // mock template deployment + actor_->SetTemplateDeployment(localVarResult); + // make a suitable create agent request + auto req = std::make_shared(); + auto instanceinfo = view_utils::Get1DInstance(); + instanceinfo.set_requestid("requestid123"); + instanceinfo.set_instanceid("same-ins001"); + req->mutable_instanceinfo()->CopyFrom(instanceinfo); + req->mutable_instanceinfo()->mutable_scheduleoption()->set_schedpolicyname(MONOPOLY_SCHEDULE); + litebus::Future> createPodCalledArg; + litebus::Future> createPodCalledArg1; + auto createPod2 = GetReadyPod(); + createPod2->GetMetadata()->SetName("function-agent-ins002"); + EXPECT_CALL(*mockClient, CreateNamespacedPod) + .WillOnce(testing::DoAll(test::FutureArg<1>(&createPodCalledArg), testing::Return(GetReadyPod()))) + .WillOnce(testing::DoAll(test::FutureArg<1>(&createPodCalledArg1), testing::Return(GetReadyPod()))); + + litebus::Future deletePodCalledArg; + litebus::Future deletePodCalledArg1; + litebus::Future deletePodCalledArg2; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(test::FutureArg<0>(&deletePodCalledArg), testing::Return(GetReadyPod()))) + .WillOnce(testing::DoAll(test::FutureArg<0>(&deletePodCalledArg1), testing::Return(GetReadyPod()))); + + testActor_->CreateAgent(actor_->GetAID(), req->SerializeAsString()); + // Checkpoint: check create pod request is called + ASSERT_AWAIT_READY(createPodCalledArg); + req->mutable_instanceinfo()->mutable_instancestatus()->set_code(6); + req->mutable_instanceinfo()->set_functionproxyid("node-001"); + actor_->HandleInstancePut(req->instanceinfo()); + ASSERT_AWAIT_READY(deletePodCalledArg); + req->mutable_instanceinfo()->set_functionproxyid("InstanceManagerOwner"); + actor_->HandleInstanceDelete(req->instanceinfo()); + // create agent again + req->mutable_instanceinfo()->mutable_instancestatus()->set_code(1); + req->mutable_instanceinfo()->set_functionproxyid("node-001"); + testActor_->CreateAgent(actor_->GetAID(), req->SerializeAsString()); + ASSERT_AWAIT_READY(createPodCalledArg1); + req->mutable_instanceinfo()->set_functionproxyid("InstanceManagerOwner"); + actor_->HandleInstanceDelete(req->instanceinfo()); + ASSERT_AWAIT_READY(deletePodCalledArg1); +} + +TEST_F(ScalerTest, CreateFunctionPoolsMockSuccess) +{ + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + litebus::Future> mockDeploymentFuture; + EXPECT_CALL(*mockClient.get(), MockCreateNamespacedDeployment(testing::_)) + .WillRepeatedly(testing::DoAll(test::FutureArg<0>(&mockDeploymentFuture))); + ASSERT_AWAIT_TRUE([&]() -> bool { + return litebus::Async(actor_->GetAID(), &ScalerActor::CreateResourcePools, pools_).Get() == + static_cast(0); + }); + ASSERT_AWAIT_READY(mockDeploymentFuture); + auto mockDeployment = mockDeploymentFuture.Get(); + std::string expectMetadata = R"({"labels":{"app":"function-agent-600-600-1pool","yr-reuse":"false"}})"; + EXPECT_STREQ(mockDeployment->GetSpec()->GetRTemplate()->GetMetadata()->ToJson().dump().c_str(), + expectMetadata.c_str()); + std::string expectAffinity = + "{\"podAntiAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"podAffinityTerm\":{\"labelSelector\":{\"matchLabels\":{\"app\":\"function-agent-600-600-1pool\"}},\"topologyKey\":\"kubernetes.io/hostname\"},\"weight\":100}],\"requiredDuringSchedulingIgnoredDuringExecution\":[]}}"; + EXPECT_STREQ(mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetAffinity()->ToJson().dump().c_str(), + expectAffinity.c_str()); + + ASSERT_AWAIT_TRUE([&]() -> bool { + return litebus::Async(actor_->GetAID(), &ScalerActor::GetPoolDeploymentsMap).Get().size() >= 2; + }); + scaler::ResourcePool pool1{ .name = "600-600-1pool", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "700m" }, { MEMORY_RESOURCE, "700Mi" } }, + .limitResources = { { CPU_RESOURCE, "700m" }, { MEMORY_RESOURCE, "700Mi" } } }; + + for (unsigned long int i = 0; i < pools_->size(); i++) { + if (pools_->at(i).name == "600-600-1pool") { + pools_->erase(pools_->begin() + i); + pools_->push_back(pool1); + } + } + litebus::Async(actor_->GetAID(), &ScalerActor::CreateResourcePools, pools_); + ASSERT_AWAIT_TRUE([&]() -> bool { + auto m = litebus::Async(actor_->GetAID(), &ScalerActor::GetPoolDeploymentsMap).Get(); + if (auto poolIter = m.find("function-agent-600-600-2pool"); poolIter != m.end()) { + return true; + } + return false; + }); +} + +TEST_F(ScalerTest, IsSameDeploymentConfig) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + scaler::ResourcePool pool1{ .name = "600-600-1pool", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "600m" }, { MEMORY_RESOURCE, "600Mi" } }, + .limitResources = { { CPU_RESOURCE, "600m" }, { MEMORY_RESOURCE, "600Mi" } } }; + + std::shared_ptr body; + auto deployment = mockClient->CreateNamespacedDeployment("", body).Get(); + EXPECT_TRUE(actor_->IsSameDeploymentConfig(pool1, deployment)); + pool1.limitResources[MEMORY_RESOURCE] = "700Mi"; + EXPECT_FALSE(actor_->IsSameDeploymentConfig(pool1, deployment)); + pool1.limitResources[CPU_RESOURCE] = "700m"; + EXPECT_FALSE(actor_->IsSameDeploymentConfig(pool1, deployment)); + pool1.requestResources[MEMORY_RESOURCE] = "700Mi"; + EXPECT_FALSE(actor_->IsSameDeploymentConfig(pool1, deployment)); + pool1.requestResources[CPU_RESOURCE] = "700m"; + EXPECT_FALSE(actor_->IsSameDeploymentConfig(pool1, deployment)); + pool1.poolSize = 2; + EXPECT_FALSE(actor_->IsSameDeploymentConfig(pool1, deployment)); +} + + TEST_F(ScalerTest, LoadFunctionPodPoolsConfigSuccess) +{ + (void)litebus::os::Rmdir("/home/sn/scaler/config/"); + (void)litebus::os::Mkdir("/home/sn/scaler/config/"); + auto writeStatus = Write("/home/sn/scaler/config/functionsystem-pools.json", "fake json"); + YRLOG_INFO("write json file result: {}", writeStatus); + // Failure + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + poolManager->LoadPodPoolsConfig("/home/sn/scaler/config/functionsystem-pools.json"); + auto localPoolMap = poolManager->GetLocalPodPools(); + ASSERT_AWAIT_TRUE([&]() -> bool { return localPoolMap.empty(); }); + // Success + std::string poolsStr = + R"({"pools":[{"id":"pool1-500-500-a","resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"500m","memory":"500Mi"}},"size":1,"reuse":true},{"id":"pool2-600-600-a","resources":{"limits":{"cpu":"600m","memory":"600Mi"},"requests":{"cpu":"600m","memory":"600Mi"}},"size":2}]})"; + writeStatus = Write("/home/sn/scaler/config/functionsystem-pools.json", poolsStr); + YRLOG_INFO("write json file result: {}", writeStatus); + poolManager->LoadPodPoolsConfig("/home/sn/scaler/config/functionsystem-pools.json"); + localPoolMap = poolManager->GetLocalPodPools(); + ASSERT_AWAIT_TRUE([&]() -> bool { return localPoolMap.size() == 2; }); + EXPECT_TRUE(localPoolMap["pool1-500-500-a"]->reuse); + EXPECT_FALSE(localPoolMap["pool2-600-600-a"]->reuse); + (void)litebus::os::Rmdir("/home/sn/scaler/config/"); +} + +const std::string DELEGATE_CONTAINER_VOLUME_MOUNTS_NO = R"( +{ + "image": "image", + "env": [], + "command":{ + "command1": "100", + "command2": "1" + }, + "args": { + "args1": "100", + "args2": "1" + }, + "workingDir": "workingDir", + "uid": 1003, + "gid": 1003 +} +)"; + +const std::string DELEGATE_CONTAINER_VOLUME_MOUNTS_INVALID = R"( +{ + "image": "image", + "env": [], + "command": { + "command1": "100", + "command2": "1" + }, + "workingDir": "workingDir", + "args": { + "args1": "100", + "args2": "1" + }, + "uid": 1003, + "gid": 1003, + "volumeMounts": {} +} +)"; + +const std::string DELEGATE_CONTAINER_VOLUME_MOUNTS_NO_NAME = R"( +{ + "image": "image", + "env": [], + "command": { + "command1": "100", + "command2": "1" + }, + "workingDir": "workingDir", + "args": { + "args1": "100", + "args2": "1" + }, + "uid": 1003, + "gid": 1003, + "volumeMounts": [{ + "name": "uds", + "mountPath": "/home/uds" + }, + { + "mountPath": "/home/uds/invalid" + } + ] +} +)"; + +const std::string DELEGATE_CONTAINER_NO_UID_GID_AND_SENSITIVE_VOLUME_MOUNT = R"( +{ + "image": "image", + "env": [], + "command": { + "command1": "100", + "command2": "1" + }, + "volumeMounts": [ + { + "name": "sts-config", + "mountPath": "/home/uds" + }, + { + "name": "uds", + "mountPath": "/home/uds" + } + ], + "workingDir": "workingDir", + "args": { + "args1": "100", + "args2": "1" + } +} +)"; + +TEST_F(ScalerTest, ParseContainerInfoTest) +{ + ::resources::InstanceInfo instanceInfo; + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_STR; + + litebus::Option> delegateContainerOp; + ParseDelegateContainer(instanceInfo, delegateContainerOp); + EXPECT_TRUE(delegateContainerOp.IsSome()); + + auto delegateContainer = delegateContainerOp.Get(); + EXPECT_EQ(delegateContainer->GetSecurityContext()->GetRunAsGroup(), 1003); + EXPECT_EQ(delegateContainer->GetSecurityContext()->GetRunAsUser(), 1003); + + // readiness + EXPECT_TRUE(delegateContainer->ReadinessProbeIsSet()); + EXPECT_TRUE(delegateContainer->GetReadinessProbe()->ExecIsSet()); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetExec()->GetCommand().at(0), "readiness"); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetExec()->GetCommand().at(1), "probe"); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetInitialDelaySeconds(), 1100); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetTimeoutSeconds(), 1200); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetPeriodSeconds(), 1300); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetSuccessThreshold(), 1400); + EXPECT_EQ(delegateContainer->GetReadinessProbe()->GetFailureThreshold(), 1500); + + // liveness + EXPECT_TRUE(delegateContainer->LivenessProbeIsSet()); + EXPECT_TRUE(delegateContainer->GetLivenessProbe()->ExecIsSet()); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetExec()->GetCommand().at(0), "liveness"); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetExec()->GetCommand().at(1), "probe"); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetInitialDelaySeconds(), 100); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetTimeoutSeconds(), 200); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetPeriodSeconds(), 300); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetSuccessThreshold(), 400); + EXPECT_EQ(delegateContainer->GetLivenessProbe()->GetFailureThreshold(), 500); + + auto envs = delegateContainer->GetEnv(); + EXPECT_EQ(envs.size(), static_cast(3)); + EXPECT_EQ(envs.at(0)->GetName(), "env_x"); + EXPECT_EQ(envs.at(0)->GetValue(), "value_x"); + + EXPECT_EQ(envs.at(1)->GetName(), "env_y"); + EXPECT_EQ(envs.at(1)->GetValueFrom()->GetFieldRef()->GetApiVersion(), "v1"); + EXPECT_EQ(envs.at(1)->GetValueFrom()->GetFieldRef()->GetFieldPath(), "status.podIP"); + + EXPECT_EQ(envs.at(2)->GetName(), "env_z"); + EXPECT_EQ(envs.at(2)->GetValueFrom()->GetResourceFieldRef()->GetContainerName(), "delegate_container"); + EXPECT_EQ(envs.at(2)->GetValueFrom()->GetResourceFieldRef()->GetResource(), "resource_x"); + EXPECT_EQ(envs.at(2)->GetValueFrom()->GetResourceFieldRef()->GetDivisor(), "divisor_x"); + + EXPECT_EQ(delegateContainer->GetVolumeMounts().size(), static_cast(1)); + for (const auto &mount : delegateContainer->GetVolumeMounts()) { + if (mount->GetName() == "uds") { + EXPECT_EQ(mount->GetMountPath(), "/home/uds"); + EXPECT_EQ(mount->GetSubPath(), "uds sub path"); + EXPECT_EQ(mount->GetMountPropagation(), "mount propagation"); + EXPECT_TRUE(mount->IsReadOnly()); + } + } + + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_VOLUME_MOUNTS_NO; + ParseDelegateContainer(instanceInfo, delegateContainerOp); + EXPECT_TRUE(delegateContainerOp.Get()->GetVolumeMounts().empty()); + + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_VOLUME_MOUNTS_INVALID; + ParseDelegateContainer(instanceInfo, delegateContainerOp); + EXPECT_TRUE(delegateContainerOp.Get()->GetVolumeMounts().empty()); + + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_VOLUME_MOUNTS_NO_NAME; + ParseDelegateContainer(instanceInfo, delegateContainerOp); + EXPECT_TRUE(delegateContainerOp.IsSome()); + EXPECT_EQ(delegateContainerOp.Get()->GetVolumeMounts().size(), static_cast(1)); + EXPECT_EQ(delegateContainerOp.Get()->GetVolumeMounts().at(0)->GetName(), "uds"); + + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = + DELEGATE_CONTAINER_NO_UID_GID_AND_SENSITIVE_VOLUME_MOUNT; + ParseDelegateContainer(instanceInfo, delegateContainerOp); + EXPECT_TRUE(delegateContainerOp.IsSome()); + EXPECT_TRUE(delegateContainerOp.Get()->GetSecurityContext()->RunAsNonRootIsSet()); + EXPECT_FALSE(delegateContainerOp.Get()->GetSecurityContext()->IsRunAsNonRoot()); +} + +const std::string DELEGATE_VOLUMES = R"( +[{ + "name": "uds-volume", + "hostPath": { + "type": "type", + "path": "/home/uds" + } +}, { + "name": "empty-volume", + "emptyDir": { + "medium": "medium", + "sizeLimit": "sizeLimit" + } +}, { + "name": "secret-volume", + "secret": { + "secretName": "secret name", + "defaultMode": 1, + "items": [{ + "key": "password", + "mode": 511, + "path": "/home/xxx" + }], + "optional": true + } +}, { + "name": "config-map-volume", + "configMap": { + "defaultMode": 440, + "name": "xx-config", + "items": [{ + "mode": 1, + "key": "xx-key", + "path": "/home/xxx" + }] + } +}, { + "emptyDir": {} +}] +)"; + +const std::string DELEGATE_HOST_ALIASES = R"( +{ + "ip1": [ + "hostname1-1" + ], + "ip2": [ + "hostname2-1", + "hostname2-2" + ], + "ip3": [] +} +)"; + +TEST_F(ScalerTest, ParseDelegateVolumesTest) +{ + ::resources::InstanceInfo instanceInfo; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUMES") = DELEGATE_VOLUMES; + + std::vector> volumes; + ParseDelegateVolumes(instanceInfo, volumes); + EXPECT_EQ(volumes.size(), static_cast(4)); // emptyDir is invalid, because no name. + + EXPECT_EQ(volumes.at(0)->GetName(), "uds-volume"); + EXPECT_EQ(volumes.at(0)->GetHostPath()->GetType(), "type"); + + EXPECT_EQ(volumes.at(1)->GetName(), "empty-volume"); + EXPECT_EQ(volumes.at(1)->GetEmptyDir()->GetMedium(), "medium"); + EXPECT_EQ(volumes.at(1)->GetEmptyDir()->GetSizeLimit(), "sizeLimit"); + + EXPECT_EQ(volumes.at(2)->GetName(), "secret-volume"); + EXPECT_EQ(volumes.at(2)->GetSecret()->GetSecretName(), "secret name"); + EXPECT_EQ(volumes.at(2)->GetSecret()->GetDefaultMode(), 1); + EXPECT_TRUE(volumes.at(2)->GetSecret()->IsOptional()); + + EXPECT_EQ(volumes.at(2)->GetSecret()->GetItems().size(), static_cast(1)); + EXPECT_EQ(volumes.at(2)->GetSecret()->GetItems().at(0)->GetKey(), "password"); + EXPECT_EQ(volumes.at(2)->GetSecret()->GetItems().at(0)->GetMode(), 511); + EXPECT_EQ(volumes.at(2)->GetSecret()->GetItems().at(0)->GetPath(), "/home/xxx"); + + EXPECT_EQ(volumes.at(3)->GetName(), "config-map-volume"); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetName(), "xx-config"); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetDefaultMode(), 440); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetItems().size(), static_cast(1)); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetItems().at(0)->GetMode(), 1); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetItems().at(0)->GetKey(), "xx-key"); + EXPECT_EQ(volumes.at(3)->GetConfigMap()->GetItems().at(0)->GetPath(), "/home/xxx"); + + std::vector> volumes_illegal; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUMES") = "{]"; + ParseDelegateVolumes(instanceInfo, volumes_illegal); // invalid + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUMES") = "{}"; + ParseDelegateVolumes(instanceInfo, volumes_illegal); // not array + EXPECT_TRUE(volumes_illegal.empty()); +} + +TEST_F(ScalerTest, ParseDelegateVolumeMountsTest) +{ + ::resources::InstanceInfo instanceInfo; + std::vector> volumeMounts; + ParseDelegateVolumeMounts(instanceInfo,"DELEGATE_VOLUME_MOUNTS", volumeMounts); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUME_MOUNTS") = "{]"; + ParseDelegateVolumeMounts(instanceInfo,"DELEGATE_VOLUME_MOUNTS", volumeMounts); + EXPECT_TRUE(volumeMounts.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUME_MOUNTS") = "{}"; + ParseDelegateVolumeMounts(instanceInfo,"DELEGATE_VOLUME_MOUNTS", volumeMounts); + EXPECT_TRUE(volumeMounts.empty()); + + const std::string DELEGATE_VOLUME_MOUNTS = R"( + [{ + "name": "delegate-container-log", + "mountPath": "/var/lib/docker/", + "subPath": "delegate-container-log sub path", + "subPathExpr": "delegate-container-log sub path expr", + "mountPropagation": "delegate-container-log mount propagation" + }, { + "name": "kubepods_burstable", + "mountPath": "/sys/fs/cgroup/memory/kubepods/burstable/", + "subPath": "kubepods_burstable sub path", + "subPathExpr": "kubepods_burstable sub path expr", + "mountPropagation": "kubepods_burstable mount propagation", + "readOnly": true + }, { + "name": "invalid-mount_path" + }, { + "mountPath": "/sys/fs/cgroup/memory/kubepods/burstable/" + }] + )"; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUME_MOUNTS") = DELEGATE_VOLUME_MOUNTS; + ParseDelegateVolumeMounts(instanceInfo, "DELEGATE_VOLUME_MOUNTS", volumeMounts); + EXPECT_EQ(volumeMounts.size(), static_cast(2)); + + EXPECT_FALSE(volumeMounts.at(0)->IsReadOnly()); + EXPECT_EQ(volumeMounts.at(0)->GetName(), "delegate-container-log"); + EXPECT_EQ(volumeMounts.at(0)->GetMountPath(), "/var/lib/docker/"); + + EXPECT_TRUE(volumeMounts.at(1)->IsReadOnly()); + EXPECT_EQ(volumeMounts.at(1)->GetName(), "kubepods_burstable"); + EXPECT_EQ(volumeMounts.at(1)->GetSubPath(), "kubepods_burstable sub path"); +} + +const std::string DELEGATE_SIDECAR = R"( +[{ + "name": "name", + "image": "image", + "resourceRequirements": { + "limits":{ + "cpu":"100", + "mem":"101" + }, + "requests":{ + "cpu":"102", + "mem":"103" + } + }, + "lifecycle":{ + "preStop":{ + "exec":{ + "command":["pre","stop"] + } + }, + "postStart":{ + "exec":{ + "command":["post","start"] + } + } + } +}] +)"; + +TEST_F(ScalerTest, ParseDelegateSidecarsTest) +{ + ::resources::InstanceInfo instanceInfo; + std::vector> sidecars; + ParseDelegateSidecars(instanceInfo, sidecars); + EXPECT_TRUE(sidecars.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_SIDECARS") = "{]"; + ParseDelegateSidecars(instanceInfo, sidecars); + EXPECT_TRUE(sidecars.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_SIDECARS") = "{}"; + ParseDelegateSidecars(instanceInfo, sidecars); + EXPECT_TRUE(sidecars.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_SIDECARS") = DELEGATE_SIDECAR; + ParseDelegateSidecars(instanceInfo, sidecars); + EXPECT_EQ(sidecars.size(), static_cast(1)); + + auto container = sidecars.at(0); + EXPECT_EQ(container->GetName(), "name"); + EXPECT_EQ(container->GetImage(), "image"); + + EXPECT_EQ(container->GetResources()->GetLimits().at("cpu"), "100"); + EXPECT_EQ(container->GetResources()->GetLimits().at("mem"), "101"); + EXPECT_EQ(container->GetResources()->GetRequests().at("mem"), "103"); + EXPECT_EQ(container->GetResources()->GetRequests().at("cpu"), "102"); + + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().size(), static_cast(2)); + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().at(0), "pre"); + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().at(1), "stop"); + + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().size(), static_cast(2)); + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().at(1), "start"); + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().at(0), "post"); +} + +TEST_F(ScalerTest, ParseDelegateInitContainersTest) +{ + ::resources::InstanceInfo instanceInfo; + std::vector> initContainers; + ParseDelegateSidecars(instanceInfo, initContainers); + EXPECT_TRUE(initContainers.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_CONTAINERS") = "{]"; + ParseDelegateInitContainers(instanceInfo, initContainers); + EXPECT_TRUE(initContainers.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_CONTAINERS") = "{}"; + ParseDelegateInitContainers(instanceInfo, initContainers); + EXPECT_TRUE(initContainers.empty()); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_CONTAINERS") = DELEGATE_SIDECAR; + ParseDelegateInitContainers(instanceInfo, initContainers); + EXPECT_EQ(initContainers.size(), static_cast(1)); + + auto container = initContainers.at(0); + EXPECT_EQ(container->GetName(), "name"); + EXPECT_EQ(container->GetImage(), "image"); + + EXPECT_EQ(container->GetResources()->GetLimits().at("cpu"), "100"); + EXPECT_EQ(container->GetResources()->GetLimits().at("mem"), "101"); + EXPECT_EQ(container->GetResources()->GetRequests().at("cpu"), "102"); + EXPECT_EQ(container->GetResources()->GetRequests().at("mem"), "103"); + + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().size(), static_cast(2)); + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().at(0), "pre"); + EXPECT_EQ(container->GetLifecycle()->GetPreStop()->GetExec()->GetCommand().at(1), "stop"); + + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().size(), static_cast(2)); + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().at(0), "post"); + EXPECT_EQ(container->GetLifecycle()->GetPostStart()->GetExec()->GetCommand().at(1), "start"); +} + +const std::string DELEGATE_TOLERATIONS = R"( +[{ +"key": "key1", +"operator": "Equal", +"value": "value", +"effect": "NoSchedule" +}, { +"key": "key2", +"operator": "Equal", +"value": "value", +"effect": "NoSchedule" +}] +)"; + +const std::string DELEGATE_ANNOTATIONS = R"( +{ + "key1": "value1", + "key2": "value2" +} +)"; + +const std::string DELEGATE_POD_SECCOMP_PROFILE = R"( +{ "type": "RuntimeDefault" } +)"; + +const std::string DELEGATE_NODE_AFFINITY = R"( +{ + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{ + "matchExpressions": [{ + "key": "node.kubernetes.io/instance-type", + "operator": "In", + "values": ["physical.kat2ne.48xlarge.8.376t.ei.c002.ondemand", "physical.kat2ne.48xlarge.8.ei.pod101.ondemand"] + }] + }] + } +} +)"; + +const std::string DELEGATE_AFFINITY = R"( +{ + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + "faasfrontend": "" + } + }, + "topologyKey": "topology.kubernetes.io/zone" + }, + "weight": 100 + }] + } +} +)"; + +const std::string DELEGATE_NODE_PREFERRED_AFFINITY = R"( +{ + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "preference": { + "matchExpressions": [{ + "key": "node.kubernetes.io/instance-type", + "operator": "In", + "values": ["physical.kat2ne.48xlarge.8.376t.ei.c002.ondemand", "physical.kat2ne.48xlarge.8.ei.pod101.ondemand"] + }] + }, + "weight": 1 + }] +} +)"; + +const std::string DELEGATE_RUNTIME_MANAGER = R"( +{ + "image": "testImage", + "env": [{"name":"SOLOMON_APP_ID","value":"redis-bridge"},{"name":"SOLOMON_CONFIG_CENTER_URL"},{"name":"SOLOMON_DOCKER_ENV","value":"kwe_perf"}] +} +)"; + +const std::string DELEGATE_INIT_VOLUME_MOUNTS = R"( +[{ + "name": "delegate-container-init", + "mountPath": "/opt/huawei/logs" +}] +)"; + +const std::string DELEGATE_INTI_ENVS = R"( +[ + { + "name": "env1", + "value": "value1" + }, + { + "name": "env2", + "value": "value2" + } +] +)"; + +const std::string DELEGATE_POD_INIT_LABELS = R"( +{ + "intK1": "val1", + "intK2": "val2", + "securityGroup": "123456" +} +)"; + +TEST_F(ScalerTest, CreatePodTemplateSpec) +{ + ::resources::InstanceInfo instanceInfo; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_VOLUMES") = DELEGATE_VOLUMES; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_HOST_ALIASES") = DELEGATE_HOST_ALIASES; + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_STR; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_SIDECARS") = DELEGATE_SIDECAR; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_CONTAINERS") = DELEGATE_SIDECAR; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_TOLERATIONS") = DELEGATE_TOLERATIONS; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_POD_ANNOTATIONS") = DELEGATE_ANNOTATIONS; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_POD_SECCOMP_PROFILE") = DELEGATE_POD_SECCOMP_PROFILE; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_NODE_AFFINITY") = DELEGATE_NODE_AFFINITY; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_AFFINITY") = DELEGATE_AFFINITY; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_VOLUME_MOUNTS") = DELEGATE_INIT_VOLUME_MOUNTS; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_INIT_ENV") = DELEGATE_INTI_ENVS; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_POD_INIT_LABELS") = DELEGATE_POD_INIT_LABELS; + + std::vector> volumes; + ParseDelegateVolumes(instanceInfo, volumes); + + std::vector> delegateHostAliases; + ParseDelegateHostAliases(instanceInfo, delegateHostAliases); + + litebus::Option> delegateContainer; + ParseDelegateContainer(instanceInfo, delegateContainer); + std::vector> delegateInitVolumeMounts; + ParseDelegateVolumeMounts(instanceInfo, "DELEGATE_INIT_VOLUME_MOUNTS", delegateInitVolumeMounts); + std::vector> delegateSidecars; + ParseDelegateSidecars(instanceInfo, delegateSidecars); + std::vector> tolerations; + ParseDelegateTolerations(instanceInfo, tolerations); + std::vector> initEnvs; + ParseDelegateInitEnv(instanceInfo, initEnvs); + auto annotations = ParseDelegateAnnotations(instanceInfo); + EXPECT_EQ(annotations.size(), static_cast(2)); + instanceInfo.add_labels("k1:v1"); + auto podInitLabels = GetPodLabelsForCreate(instanceInfo); + EXPECT_EQ(podInitLabels.size(), static_cast(4)); + EXPECT_TRUE(podInitLabels.find("k1") != podInitLabels.end() && podInitLabels.find("k1")->second == "v1"); + nlohmann::json deploymentJSON = nlohmann::json::parse(DEPLOYMENT_JSON); + auto templateDeployment = std::make_shared(); + templateDeployment->FromJson(deploymentJSON); + + scaler::ResourcePool pool{ .name = "", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "500m" }, { MEMORY_RESOURCE, "500Mi" } }, + .limitResources = { { CPU_RESOURCE, "500m" }, { MEMORY_RESOURCE, "500Mi" } } }; + + pool.delegateVolumes = volumes; + pool.delegateHostAliases = delegateHostAliases; + pool.delegateContainer = delegateContainer; + pool.delegateSidecars = delegateSidecars; + pool.delegateTolerations = tolerations; + pool.delegateInitEnvs = initEnvs; + pool.delegateSecCompProfile = ParseSeccompProfile(instanceInfo); + pool.delegateInitVolumeMounts = delegateInitVolumeMounts; + std::unordered_map> affinityTypeLabels; + pool.affinity = ParseAffinity(affinityTypeLabels); + ParseAffinityFromCreateOpts(instanceInfo, pool.affinity); + ParseNodeAffinity(instanceInfo, pool.affinity); + + auto spec = actor_->GeneratePodTemplateSpec(pool, templateDeployment); + EXPECT_EQ(spec->GetSpec()->GetVolumes().size(), static_cast(18)); + int32_t count = 0; + for (const auto &item : spec->GetSpec()->GetVolumes()) { + if (item->GetName() == "uds-volume" || item->GetName() == "empty-volume" || + item->GetName() == "secret-volume") { + count++; + } + } + // There are 3 valid volumes variables in JSON. + EXPECT_EQ(count, 3); + + EXPECT_EQ(spec->GetSpec()->GetTolerations().size(), static_cast(2)); + EXPECT_EQ(spec->GetSpec()->GetSecurityContext()->GetSeccompProfile()->GetType(), "RuntimeDefault"); + EXPECT_EQ(spec->GetSpec()->GetAffinity()->GetNodeAffinity()->RequiredDuringSchedulingIgnoredDuringExecutionIsSet(), + true); + EXPECT_EQ(spec->GetSpec()->GetContainers().size(), static_cast(4)); + + for (auto container: spec->GetSpec()->GetContainers()) { + YRLOG_INFO("container name : {}", container->GetName()); + } + // There are 3 environment variables in JSON. + // By default, ip and port of datasystem are added in code. + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetEnv().size(), static_cast(5)); + + auto env = spec->GetSpec()->GetContainers().at(0)->GetEnv().at(2); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetEnv().at(2)->GetName(), "env_z"); + EXPECT_EQ(env->GetValueFrom()->GetResourceFieldRef()->GetContainerName(), "delegate_container"); + + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetEnv().at(3)->GetName(), "DATASYSTEM_HOST"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetEnv().at(4)->GetName(), "DATASYSTEM_PORT"); + + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetVolumeMounts().size(), static_cast(1)); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetVolumeMounts().at(0)->GetName(), "uds"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetVolumeMounts().at(0)->GetMountPath(), "/home/uds"); + + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetResources()->GetLimits()["cpu"], "3"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetResources()->GetLimits()["memory"], "6Gi"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetResources()->GetLimits()["cpu"], "500m"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetResources()->GetLimits()["memory"], "500Mi"); + + EXPECT_EQ(spec->GetSpec()->GetInitContainers().size(), static_cast(1)); + EXPECT_EQ(spec->GetSpec()->GetInitContainers()[0]->GetVolumeMounts().size(), static_cast(3)); + EXPECT_EQ(spec->GetSpec()->GetInitContainers()[0]->GetEnv().size(), static_cast(2)); + + EXPECT_TRUE(spec->GetSpec()->HostAliasesIsSet()); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().size(), static_cast(2)); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(0)->GetIp(), "ip1"); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(0)->GetHostnames().size(), static_cast(1)); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(0)->GetHostnames().at(0), "hostname1-1"); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(1)->GetIp(), "ip2"); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(1)->GetHostnames().size(), static_cast(2)); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(1)->GetHostnames().at(0), "hostname2-1"); + EXPECT_EQ(spec->GetSpec()->GetHostAliases().at(1)->GetHostnames().at(1), "hostname2-2"); + + EXPECT_TRUE(spec->GetSpec()->AutomountServiceAccountTokenIsSet()); + EXPECT_FALSE(spec->GetSpec()->IsAutomountServiceAccountToken()); + EXPECT_FALSE(spec->GetSpec()->ServiceAccountNameIsSet()); + + // create container without uid/gid + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = + DELEGATE_CONTAINER_NO_UID_GID_AND_SENSITIVE_VOLUME_MOUNT; + + // create system function + instanceInfo.mutable_createoptions()->operator[](RESOURCE_OWNER_KEY) = SYSTEM_OWNER_VALUE; + + ParseDelegateContainer(instanceInfo, delegateContainer); + pool.delegateContainer = delegateContainer; + pool.isSystemFunc = IsSystemFunction(instanceInfo); + actor_->member_->etcdAuthType = "TLS"; + actor_->AddVolumesAndMountsForSystemFunc(pool); + + spec = actor_->GeneratePodTemplateSpec(pool, templateDeployment); + EXPECT_TRUE(spec->GetSpec()->GetContainers().at(0)->GetSecurityContext()->RunAsNonRootIsSet()); + EXPECT_FALSE(spec->GetSpec()->GetContainers().at(0)->GetSecurityContext()->IsRunAsNonRoot()); + EXPECT_FALSE(spec->GetSpec()->GetContainers().at(0)->GetSecurityContext()->RunAsGroupIsSet()); + EXPECT_FALSE(spec->GetSpec()->GetContainers().at(0)->GetSecurityContext()->RunAsUserIsSet()); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(0)->GetVolumeMounts().size(), static_cast(1)); + + EXPECT_TRUE(spec->GetSpec()->AutomountServiceAccountTokenIsSet()); + EXPECT_TRUE(spec->GetSpec()->IsAutomountServiceAccountToken()); + EXPECT_TRUE(spec->GetSpec()->ServiceAccountNameIsSet()); + EXPECT_EQ(spec->GetSpec()->GetServiceAccountName(), "system-function"); + + // create delegate runtime manager + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_RUNTIME_MANAGER") = DELEGATE_RUNTIME_MANAGER; + litebus::Option> delegateRuntimeManager; + ParseDelegateRuntimeManager(instanceInfo, delegateRuntimeManager); + pool.delegateRuntimeManager = delegateRuntimeManager; + auto prevEnvSize = spec->GetSpec()->GetContainers().at(1)->GetEnv().size(); + spec = actor_->GeneratePodTemplateSpec(pool, templateDeployment); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetImage(), "testImage"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetEnv().size(), prevEnvSize + 3); + + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_NODE_AFFINITY") = DELEGATE_NODE_PREFERRED_AFFINITY; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_AFFINITY") = DELEGATE_AFFINITY; + ParseNodeAffinity(instanceInfo, pool.affinity); + ParseAffinityFromCreateOpts(instanceInfo, pool.affinity); + spec = actor_->GeneratePodTemplateSpec(pool, templateDeployment); + EXPECT_EQ(spec->GetSpec()->GetAffinity()->GetNodeAffinity()->PreferredDuringSchedulingIgnoredDuringExecutionIsSet(), + true); + EXPECT_EQ(spec->GetSpec()->GetAffinity()->GetPodAntiAffinity()->PreferredDuringSchedulingIgnoredDuringExecutionIsSet(), + true); +} + +TEST_F(ScalerTest, CreateAgentTest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), "invalid_request"); + auto createAgentRequest = std::make_shared(); + ::resources::InstanceInfo info; + info.set_requestid("test-request-01"); + info.set_instanceid("test-instance-01"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-01"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::GS_START_CREATE_POD_FAILED; }); + + actor_->member_->templateDeployment = nullptr; + actor_->member_->agentTemplatePath = ""; + ::resources::Resources resources; + info.mutable_resources()->CopyFrom(resources); + info.set_requestid("test-request-error-deployment"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-error-deployment"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::GS_START_CREATE_POD_FAILED; }); + + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + actor_->SetTemplateDeployment(localVarResult); + + info.set_requestid("test-request-02"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-02"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::GS_START_CREATE_POD_FAILED; }); + info.mutable_resources()->CopyFrom(GetResources()); + info.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = "invalid json"; + info.set_requestid("test-request-03"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + + litebus::Promise> promise; + promise.SetFailed(100); + EXPECT_CALL(*mockClient, CreateNamespacedPod).WillOnce(testing::Return(promise.GetFuture())); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-03"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::GS_START_CREATE_POD_FAILED; }); + + auto pod = GetReadyPod(); + EXPECT_CALL(*mockClient, CreateNamespacedPod).WillRepeatedly(testing::Return(pod)); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-03"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { + litebus::Async(actor_->GetAID(), &ScalerActor::OnPodModified, K8sEventType::EVENT_TYPE_MODIFIED, pod); + return GetTestResponse().code() == StatusCode::SUCCESS; + }); + + info.set_requestid("test-request-04"); + info.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_STR; + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { + litebus::Async(actor_->GetAID(), &ScalerActor::OnPodModified, K8sEventType::EVENT_TYPE_MODIFIED, pod); + return GetTestResponse().requestid() == "test-request-04"; + }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::SUCCESS; }); + ASSERT_AWAIT_TRUE([&]() -> bool { + auto creteOpt = GetTestResponse().updatedcreateoptions(); + auto iter = creteOpt.find(DELEGATE_CONTAINER_ID_KEY); + return iter != creteOpt.end() && iter->second == "1234567"; + }); +} + +TEST_F(ScalerTest, CreateAgentTestWithAffinity) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto pod = GetReadyPod(); + + litebus::Future> podArgFuture; + EXPECT_CALL(*mockClient, CreateNamespacedPod) + .WillRepeatedly(testing::DoAll(test::FutureArg<1>(&podArgFuture), testing::Return(pod))); + + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + + auto localVarResult = std::make_shared(); + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + litebus::Async(actor_->GetAID(), &ScalerActor::SetTemplateDeployment, localVarResult); + + ::resources::InstanceInfo info; + + info.set_requestid("test-request-01"); + info.mutable_labels()->Add("label"); + info.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_STR; + info.mutable_createoptions()->operator[]("DELEGATE_POD_ANNOTATIONS") = DELEGATE_ANNOTATIONS; + info.mutable_createoptions()->operator[]("DELEGATE_POD_INIT_LABELS") = DELEGATE_POD_INIT_LABELS; + info.mutable_resources()->CopyFrom(GetResources()); + info.mutable_scheduleoption()->mutable_affinity()->mutable_instanceaffinity()->mutable_affinity()->operator[]( + "antiAffinityInstance") = AffinityType::PreferredAntiAffinity; + info.mutable_scheduleoption()->mutable_affinity()->mutable_instanceaffinity()->mutable_affinity()->operator[]( + "affinityInstance") = AffinityType::RequiredAffinity; + + auto createAgentRequest = std::make_shared(); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { + litebus::Async(actor_->GetAID(), &ScalerActor::OnPodModified, K8sEventType::EVENT_TYPE_MODIFIED, pod); + return GetTestResponse().requestid() == "test-request-01"; + }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::SUCCESS; }); + ASSERT_AWAIT_TRUE([&]() -> bool { + auto creteOpt = GetTestResponse().updatedcreateoptions(); + auto iter = creteOpt.find(DELEGATE_CONTAINER_ID_KEY); + return iter != creteOpt.end() && iter->second == "1234567"; + }); + + ASSERT_AWAIT_READY(podArgFuture); + auto podArg = podArgFuture.Get(); + + EXPECT_TRUE(podArg->GetMetadata()->GetLabels().find("label") != podArg->GetMetadata()->GetLabels().end()); + EXPECT_TRUE(podArg->GetMetadata()->GetLabels().find("securityGroup") != podArg->GetMetadata()->GetLabels().end()); + EXPECT_TRUE(podArg->GetMetadata()->GetLabels()["securityGroup"] == "123456"); + std::string expectAffinity = + "{\"podAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[]," + "\"requiredDuringSchedulingIgnoredDuringExecution\":[{\"labelSelector\":{\"matchLabels\":{\"affinityInstance\":" + "\"\"}},\"topologyKey\":\"kubernetes.io/" + "hostname\"}]},\"podAntiAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"podAffinityTerm\":{" + "\"labelSelector\":{\"matchLabels\":{\"antiAffinityInstance\":\"\"}},\"topologyKey\":\"kubernetes.io/" + "hostname\"},\"weight\":100}],\"requiredDuringSchedulingIgnoredDuringExecution\":[]}}"; + EXPECT_STREQ(podArg->GetSpec()->GetAffinity()->ToJson().dump().c_str(), expectAffinity.c_str()); + EXPECT_EQ(podArg->GetMetadata()->GetAnnotations().size(), static_cast(3)); + EXPECT_EQ(podArg->GetMetadata()->GetAnnotations()["yr-default"], "yr-default"); +} + +TEST_F(ScalerTest, CreatePodTestCPUBind) +{ + ::resources::InstanceInfo instanceInfo; + instanceInfo.mutable_createoptions()->operator[](DELEGATE_CONTAINER) = DELEGATE_CONTAINER_STR; + + litebus::Option> delegateContainer; + ParseDelegateContainer(instanceInfo, delegateContainer); + nlohmann::json deploymentJSON = nlohmann::json::parse(DEPLOYMENT_JSON); + auto templateDeployment = std::make_shared(); + templateDeployment->FromJson(deploymentJSON); + + scaler::ResourcePool pool{ .name = "", + .poolSize = 1, + .requestResources = { { CPU_RESOURCE, "24000m" }, { MEMORY_RESOURCE, "500Mi" } }, + .limitResources = { { CPU_RESOURCE, "24000m" }, { MEMORY_RESOURCE, "500Mi" } } }; + pool.delegateContainer = delegateContainer; + pool.isNeedBindCPU = IsNeedBindCPU(pool.limitResources); + + auto spec = actor_->GeneratePodTemplateSpec(pool, templateDeployment); + EXPECT_EQ(spec->GetSpec()->GetContainers().size(), static_cast(3)); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetResources()->GetLimits()["cpu"], "1000m"); + EXPECT_EQ(spec->GetSpec()->GetContainers().at(1)->GetResources()->GetLimits()["memory"], "900Mi"); +} + +TEST_F(ScalerTest, WatchInstanceTest) +{ + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + mockClient->InitOwnerReference("default", "function-master"); + mockClient->GetOwnerReference()->SetUid("abc"); + actor_->SetKubeClient(mockClient); + resource_view::InstanceInfo instance0 = + CreateInstance(INSTANCE_PATH_PREFIX + "/000", static_cast(InstanceState::SCHEDULING)); + metaStorageAccessor_->Put(instance0.instanceid(), "invalid json").Get(); + + auto pod1 = std::make_shared(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("127.0.0.1"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("function-agent-001"); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + pod1->GetMetadata()->GetLabels()["app"] = "function-agent"; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::Return(pod1)); + + auto pod2 = std::make_shared(); + pod2->SetStatus(std::make_shared()); + pod2->GetStatus()->SetPodIP("127.0.0.2"); + pod2->SetMetadata(std::make_shared()); + pod2->GetMetadata()->SetName("function-agent-002"); + pod2->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + pod2->GetMetadata()->GetLabels()["app"] = "function-agent"; + + std::string jsonString1; + resource_view::InstanceInfo instance1 = + CreateInstance(INSTANCE_PATH_PREFIX + "/001", static_cast(InstanceState::SCHEDULING)); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString1, instance1)); + metaStorageAccessor_->Put(instance1.instanceid(), jsonString1).Get(); + + std::string jsonString2; + resource_view::InstanceInfo instance2 = + CreateInstance(INSTANCE_PATH_PREFIX + "/002", static_cast(InstanceState::CREATING)); + (*instance2.mutable_createoptions())["DELEGATE_POD_LABELS"] = R"({"123":"123"})"; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString2, instance2)); + + std::string jsonString3; + resource_view::InstanceInfo instance3 = + CreateInstance(INSTANCE_PATH_PREFIX + "/003", static_cast(InstanceState::FATAL)); + (*instance3.mutable_createoptions())["DELEGATE_POD_LABELS"] = R"({"123":"123"})"; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString3, instance3)); + metaStorageAccessor_->Put(instance3.instanceid(), jsonString3).Get(); + + jsonString3 = ""; + instance3 = CreateInstance(INSTANCE_PATH_PREFIX + "/003", static_cast(InstanceState::FAILED)); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString3, instance3)); + metaStorageAccessor_->Put(instance3.instanceid(), jsonString3).Get(); + + std::string jsonString4; + resource_view::InstanceInfo instance4 = + CreateInstance(INSTANCE_PATH_PREFIX + "/004", static_cast(InstanceState::FATAL)); + instance4.set_functionagentid("function-agent-004"); + instance4.mutable_scheduleoption()->set_schedpolicyname(""); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString4, instance4)); + litebus::Promise> errPromise; + errPromise.SetFailed(500); + bool isRead = false; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillOnce(testing::DoAll(testing::Assign(&isRead, true), testing::Return(errPromise.GetFuture()))); + metaStorageAccessor_->Put(instance4.instanceid(), jsonString4).Get(); + EXPECT_AWAIT_TRUE([&isRead]() -> bool { return isRead; }); + std::string jsonString5; + resource_view::InstanceInfo instance5 = + CreateInstance(INSTANCE_PATH_PREFIX + "/005", static_cast(InstanceState::CREATING)); + (*instance5.mutable_createoptions())["DELEGATE_POD_LABELS"] = R"(123)"; + instance5.add_labels("label"); + instance5.add_labels("tenantId:123"); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString5, instance5)); + + std::string jsonString6; + resource_view::InstanceInfo instance6 = + CreateInstance(INSTANCE_PATH_PREFIX + "/006", static_cast(InstanceState::RUNNING)); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString6, instance6)); + + std::string jsonString7; + resource_view::InstanceInfo instance7 = + CreateInstance(INSTANCE_PATH_PREFIX + "/007", static_cast(InstanceState::SUB_HEALTH)); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString7, instance7)); + + std::string jsonString8; + resource_view::InstanceInfo instance8 = + CreateInstance(INSTANCE_PATH_PREFIX + "/008", static_cast(InstanceState::SCHEDULE_FAILED)); + instance8.set_functionagentid(""); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString8, instance8)); + + std::string jsonString9; + resource_view::InstanceInfo instance9 = + CreateInstance(INSTANCE_PATH_PREFIX + "/009", static_cast(InstanceState::FATAL)); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString9, instance9)); + instance9.set_functionproxyid(INSTANCE_MANAGER_OWNER); + + litebus::Future> body1; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::Return(pod1)); + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillOnce(testing::DoAll(FutureArg<2>(&body1), testing::Return(pod1))); + metaStorageAccessor_->Put(instance5.instanceid(), jsonString5).Get(); + ASSERT_AWAIT_READY(body1); + EXPECT_TRUE(ContainValue(body1.Get(), "\"tenantId\":\"123\"")); + EXPECT_TRUE(ContainValue(body1.Get(), "\"label\":\"\"")); + EXPECT_TRUE(ContainValue(body1.Get(), "function-master")); + + bool isFinished = false; + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(pod1))); + metaStorageAccessor_->Put(instance5.instanceid(), jsonString5).Get(); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + + jsonString5 = ""; + instance5.mutable_createoptions()->clear(); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString5, instance5)); + isFinished = false; + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(pod1))); + metaStorageAccessor_->Put(instance5.instanceid(), jsonString5).Get(); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + + isFinished = false; + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(pod1))); + metaStorageAccessor_->Put(instance2.instanceid(), jsonString2).Get(); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + + litebus::Future> body; + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillOnce(testing::DoAll(FutureArg<2>(&body), testing::Return(pod1))); + metaStorageAccessor_->Put(instance6.instanceid(), jsonString6).Get(); + ASSERT_AWAIT_READY(body); + EXPECT_TRUE(ContainValue(body.Get(), "\"instanceStatus\":\"running\"")); + + body = {}; + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillRepeatedly(testing::DoAll(FutureArg<2>(&body), testing::Return(pod1))); + metaStorageAccessor_->Put(instance7.instanceid(), jsonString7).Get(); + ASSERT_AWAIT_READY(body); + EXPECT_TRUE(ContainValue(body.Get(), "\"instanceStatus\":\"subHealth\"")); + + litebus::Future podName = ""; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(FutureArg<0>(&podName), testing::Return(pod1))); + metaStorageAccessor_->Put(instance9.instanceid(), jsonString9).Get(); + metaStorageAccessor_->Put(instance8.instanceid(), jsonString8).Get(); + ASSERT_AWAIT_READY(podName); + EXPECT_EQ(podName.Get(), "function-agent-001"); + + bool isDeleteFinished = false; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillRepeatedly(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(pod2))); + metaStorageAccessor_->Delete(instance8.instanceid()).Get(); + metaStorageAccessor_->Delete(instance9.instanceid()).Get(); + metaStorageAccessor_->Delete(instance7.instanceid()).Get(); + metaStorageAccessor_->Delete(instance6.instanceid()).Get(); + metaStorageAccessor_->Delete(instance5.instanceid()).Get(); + metaStorageAccessor_->Delete(instance4.instanceid()).Get(); + metaStorageAccessor_->Delete(instance3.instanceid()).Get(); + metaStorageAccessor_->Delete(instance0.instanceid()).Get(); + metaStorageAccessor_->Delete(instance2.instanceid()).Get(); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); +} + +TEST_F(ScalerTest, UpdateLabelPodNotFound) +{ + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + resource_view::InstanceInfo instanceInvalid = + CreateInstance(INSTANCE_PATH_PREFIX + "/000", static_cast(InstanceState::RUNNING)); + instanceInvalid.set_functionagentid(FUNCTION_AGENT_ID_PREFIX + "127.0.0.2-8080"); + std::string jsonStringInvalid; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonStringInvalid, instanceInvalid)); + + auto isFinished = false; + litebus::Future> body; + litebus::Future> body2; + litebus::Promise> errPromise; + errPromise.SetFailed(500); + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(errPromise.GetFuture()))); + metaStorageAccessor_->Put(instanceInvalid.instanceid(), jsonStringInvalid).Get(); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); +} + +TEST_F(ScalerTest, UpdateLabelSyncTest) +{ + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto pod1 = std::make_shared(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("127.0.0.1"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("function-agent-001"); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + pod1->GetMetadata()->GetLabels()["123"] = "1234"; + + resource_view::InstanceInfo instance = + CreateInstance(INSTANCE_PATH_PREFIX + "/001", static_cast(InstanceState::RUNNING)); + (*instance.mutable_createoptions())["DELEGATE_POD_LABELS"] = R"({"123":"123"})"; + std::string jsonString1; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString1, instance)); + + (*instance.mutable_createoptions())["DELEGATE_POD_LABELS"] = R"({"123":"1234"})"; + std::string jsonString2; + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString2, instance)); + + auto isFinished = false; + litebus::Future> body; + litebus::Future> body2; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::Return(pod1)); + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillOnce(testing::DoAll(FutureArg<2>(&body), testing::Return(pod1))) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), FutureArg<2>(&body2), testing::Return(pod1))).WillRepeatedly(testing::Return(pod1)); + metaStorageAccessor_->Put(instance.instanceid(), jsonString1).Get(); + metaStorageAccessor_->Put(instance.instanceid(), jsonString2).Get(); + ASSERT_AWAIT_READY(body); + EXPECT_TRUE(ContainValue(body.Get(), "\"123\"")); + + ASSERT_AWAIT_READY(body2); + EXPECT_TRUE(ContainValue(body2.Get(), "\"123\"")); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + + auto isDeleteFinished = false; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(pod1))); + metaStorageAccessor_->Delete(instance.instanceid()).Get(); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); +} + +/*case + * @title: 监å¬åˆ°æœ‰æ•ˆçš„ds-workerä¿¡æ¯ + * @type: Function test + * @ar: Scaler感知ds-worker存活 + * @dts: + * @precondition: 1.åˆå§‹åŒ–ScalerActorç±»åŠå…¶ä¾èµ– + * @precondition: 2.åˆå§‹åŒ–MetaStore Service + * @step: 1.ScalerActor注册Watch ds-worker事件 + * @step: 2.å‘MetaStore中Put有效的ds-workerä¿¡æ¯ + * @step: 3.å‘MetaStore中Deleteå‰ä¸€æ­¥Putçš„ds-workerä¿¡æ¯ + * @expect: 1.监å¬åˆ°Put事件åŽï¼ŒworkerAliveNodes-worker所在节点IP，且调用mockClientçš„PatchNode方法 + * @expect: 2.监å¬åˆ°Delete事件åŽï¼ŒworkerAliveNodes中ä¸å­˜åœ¨ds-worker所在节点IP，且调用mockClientçš„PatchNode方法 + * @author: w00521000 + */ +TEST_F(ScalerTest, UpdateNodeTaintTest) +{ + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + litebus::Async(actor_->GetAID(), &ScalerActor::SetKubeClient, mockClient); + const char *argv[] = { "./function_master", "--node_id=node1", "--ip=127.0.0.1:22771", + "--meta_store_address=127.0.0.1:32279", + "--self_taint_prefix=az1/" }; + functionmaster::Flags flags; + ASSERT_TRUE(flags.ParseFlags(5, argv).IsNone()); + actor_->ParseParams(flags); + + auto address = std::make_shared(); + address->SetType("InternalIP"); + address->SetAddress("7.189.21.43"); + auto status = std::make_shared(); + status->SetAddresses({ address }); + auto node = std::make_shared(); + node->SetStatus(status); + + auto metadata = std::make_shared(); + metadata->SetName("7.189.21.43 node"); + node->SetMetadata(metadata); + + auto spec = std::make_shared(); + node->SetSpec(spec); + node->GetSpec()->SetTaints({}); + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_ADD, node); + + // add taint + std::shared_ptr nodeList = std::make_shared(); + std::vector> items{node}; + nodeList->SetItems(items); + + bool isListFinished = false; + EXPECT_CALL(*mockClient, ListNode) + .WillOnce(testing::DoAll(testing::Assign(&isListFinished, true), testing::Return(nodeList))); + + bool isPatchFinished = false; + litebus::Future> body; + EXPECT_CALL(*mockClient, PatchNode) + .WillOnce(testing::DoAll(FutureArg<1>(&body), testing::Assign(&isPatchFinished, true), testing::Return(node))); + + auto req = std::make_shared(); + req->set_requestid("req-001"); + req->set_key("is-function-proxy-unready"); + req->set_healthy(false); + req->set_ip("7.189.21.43"); + testActor_->UpdateNodeTaints(actor_->GetAID(), req->SerializeAsString()); + EXPECT_AWAIT_TRUE([&isListFinished]() -> bool { return isListFinished; }); + EXPECT_AWAIT_TRUE([&isPatchFinished]() -> bool { return isPatchFinished; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetUpdateNodeTaintResponse().requestid() == "req-001"; }); + EXPECT_TRUE(ContainValue(body.Get(), "\"az1/is-function-proxy-unready\"")); + + // remove taint + isPatchFinished = false; + isListFinished = false; + std::shared_ptr nodeEmptyList = std::make_shared(); + EXPECT_CALL(*mockClient, ListNode) + .WillOnce(testing::DoAll(testing::Return(nodeEmptyList))) + .WillOnce(testing::DoAll(testing::Assign(&isListFinished, true), testing::Return(nodeList))); + + std::vector> Taints; + auto taint = std::make_shared(); + taint->SetKey("az1/is-function-proxy-unready"); + Taints.push_back(taint); + node->GetSpec()->SetTaints(Taints); + + actor_->SetIsUpgrading(true); + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_ADD, node); + + actor_->SetIsUpgrading(false); + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_ADD, node); + + litebus::Future> body1; + EXPECT_CALL(*mockClient, PatchNode) + .WillOnce(testing::DoAll(FutureArg<1>(&body1), testing::Assign(&isPatchFinished, true), testing::Return(node))); + req->set_requestid("req-002"); + req->set_key("is-function-proxy-unready"); + req->set_healthy(true); + req->set_ip("7.189.21.43"); + testActor_->UpdateNodeTaints(actor_->GetAID(), req->SerializeAsString()); + testActor_->UpdateNodeTaints(actor_->GetAID(), req->SerializeAsString()); + EXPECT_AWAIT_TRUE([&isListFinished]() -> bool { return isListFinished; }); + EXPECT_AWAIT_TRUE([&isPatchFinished]() -> bool { return isPatchFinished; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetUpdateNodeTaintResponse().requestid() == "req-002"; }); +} + +/** + * Feature: PodPoolTest + * Description: pod pool CRUD + * Steps: + * 1. Sync deployment and pod pool info + * 2. receive pod pool add event + * 3. receive pod pool update event + * 4. receive pod pool delete event + * Expectation: + * 1. Add Pod Pool + * 2. Update Pod Pool + * 3. Delete Pod pool + */ +TEST_F(ScalerTest, PodPoolTest) +{ + std::string mem1 = "1Gi"; + std::string cpu1 = "1"; + std::string errMemVal = "mb122"; + std::string errCpuVal = "xxxx"; + EXPECT_EQ("1024", ParseMemoryEnv(mem1)); + EXPECT_EQ("1000", ParseCPUEnv(cpu1)); + EXPECT_EQ("", ParseMemoryEnv(errMemVal)); + EXPECT_EQ("", ParseCPUEnv(errCpuVal)); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->curStatus_ == "master"; }); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + auto oldPoolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, oldPoolInfo); + oldPoolInfo->status = 1; + oldPoolInfo->readyCount = 0; + oldPoolInfo->id = "pool2"; + oldPoolInfo->group ="group2"; + litebus::Future> mockDeploymentFuture; + metaStorageAccessor_->Put("/yr/podpools/info/" + oldPoolInfo->id, poolManager->TransToJsonFromPodPoolInfo(oldPoolInfo)).Get(); + actor_->SyncDeploymentAndPodPool().Get(); + EXPECT_TRUE(poolManager->GetPodPool("pool2") != nullptr); + EXPECT_EQ(static_cast(3), poolManager->GetPoolDeploymentsMap().size()); + EXPECT_CALL(*mockClient.get(), MockCreateNamespacedDeployment(testing::_)) + .WillRepeatedly(testing::DoAll(test::FutureArg<0>(&mockDeploymentFuture))); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + ASSERT_AWAIT_READY(mockDeploymentFuture); + auto mockDeployment = mockDeploymentFuture.Get(); + EXPECT_EQ("function-agent-pool1", mockDeployment->GetMetadata()->GetName()); + EXPECT_EQ("rg1", mockDeployment->GetSpec()->GetRTemplate()->GetMetadata()->GetLabels()["rg1"]); + EXPECT_EQ("runc", mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetRuntimeClassName()); + EXPECT_EQ("val1", mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetNodeSelector()["label1"]); + EXPECT_EQ("node.kubernetes.io/disk-pressure", mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetTolerations()[0]->GetKey()); + EXPECT_TRUE(mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetAffinity()->GetPodAntiAffinity()->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_EQ(true, mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->TopologySpreadConstraintsIsSet()); + EXPECT_EQ(1, mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetTopologySpreadConstraints()[0]->GetMaxSkew()); + EXPECT_EQ(1, mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetTopologySpreadConstraints()[0]->GetLabelSelector()->GetMatchLabels().size()); + EXPECT_EQ(static_cast(3), + mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetInitContainers()[0]->GetVolumeMounts().size()); + std::shared_ptr deploymentStatus = std::make_shared(); + deploymentStatus->SetAvailableReplicas(1); + deploymentStatus->SetReplicas(1); + mockDeployment->SetStatus(deploymentStatus); + mockDeployment->GetMetadata()->GetLabels()[poolInfo->group] = poolInfo->group; + actor_->OnDeploymentModified(K8sEventType::EVENT_TYPE_MODIFIED, mockDeployment); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1")->readyCount == 1; }); + poolInfo->status = 2; + poolInfo->size = 2; + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + std::shared_ptr deploymentStatus1 = std::make_shared(); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1")->status == 3; }); + deploymentStatus1->SetAvailableReplicas(2); + deploymentStatus1->SetReplicas(0); + mockDeployment->SetStatus(deploymentStatus1); + actor_->OnDeploymentModified(K8sEventType::EVENT_TYPE_MODIFIED, mockDeployment); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1")->status == 4; }); + poolInfo->status = 6; + litebus::Future> mockDeleteDeploymentFuture; + bool isDeleteFinished = false; + EXPECT_CALL(*mockClient.get(), DeleteNamespacedDeployment) + .WillOnce(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(mockDeployment))).WillRepeatedly(testing::Return(mockDeployment)); + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1") == nullptr; }); + + nlohmann::json deployJson = nlohmann::json::parse(DEFAULT_DEPLOYMENT_JSON_STR); + std::shared_ptr deploy3 = std::make_shared(); + deploy3->FromJson(deployJson); + deploy3->GetMetadata()->SetName("function-agent-pool3"); + EXPECT_TRUE(poolManager->GetPodPool("pool3") == nullptr); + isDeleteFinished = false; + auto mockClient1 = std::make_shared(); + actor_->SetKubeClient(mockClient1); + EXPECT_CALL(*mockClient1.get(), DeleteNamespacedDeployment) + .WillOnce(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(mockDeployment))).WillRepeatedly(testing::Return(mockDeployment)); + actor_->OnDeploymentModified(K8sEventType::EVENT_TYPE_MODIFIED, deploy3); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); +} + +TEST_F(ScalerTest, PodPoolTestWithDeleteError) +{ + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->curStatus_ == "master"; }); + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->status = 6; + poolInfo->id = "pool3"; + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + EXPECT_CALL(*mockClient.get(), DeleteNamespacedDeployment) + .WillOnce(testing::Return(GetErrorDeploymentFuture(500))); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool3") != nullptr && poolManager->GetPodPool("pool3")->status == 5; + }); + } + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->status = 6; + poolInfo->id = "pool4"; + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + EXPECT_CALL(*mockClient.get(), DeleteNamespacedDeployment) + .WillOnce(testing::Return(GetErrorDeploymentFuture(404))); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool4") == nullptr; }); + } +} + +TEST_F(ScalerTest, PodPoolTestWithCreateError) +{ + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->curStatus_ == "master"; }); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->id = "pool-error-0"; + actor_->member_->templateDeployment = nullptr; + actor_->member_->agentTemplatePath = ""; + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-error-0") != nullptr && poolManager->GetPodPool("pool-error-0")->status == 5; + }); + } + { + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + localVarResult->FromJson(localVarJson); + actor_->SetTemplateDeployment(localVarResult); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->id = "pool-error-1"; + auto deployment = std::make_shared(); + deployment->SetMetadata(std::make_shared()); + deployment->GetMetadata()->SetName("function-agent-pool-error-1"); + actor_->member_->poolManager->PutDeployment(deployment); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-error-1") != nullptr && poolManager->GetPodPool("pool-error-1")->status == 5; + }); + } + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->id = "pool-error-2"; + litebus::Future> future; + future.SetFailed(StatusCode::FAILED); + actor_->member_->k8sNamespace = "mock"; + EXPECT_CALL(*mockClient.get(), MockCreateNamespacedDeployment0).WillOnce(testing::Return(future)); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-error-2") != nullptr && poolManager->GetPodPool("pool-error-2")->status == 5; + }); + actor_->member_->k8sNamespace = DEFAULT_NAMESPACE; + } + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->id = "pool-hpa-error-0"; + poolInfo->horizontalPodAutoscalerSpec = "invalid json"; + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-hpa-error-0") != nullptr && poolManager->GetPodPool("pool-hpa-error-0")->status == 5; + }); + } + { + litebus::Future> future; + future.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(future)); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->id = "pool-hpa-error-1"; + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-hpa-error-1") != nullptr && poolManager->GetPodPool("pool-hpa-error-1")->status == 5; + }); + } +} + +TEST_F(ScalerTest, PodPoolTestWithUpdateError) +{ + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->curStatus_ == "master"; }); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->id = "pool-hpa-error-0"; + poolInfo->status = 2; + poolInfo->horizontalPodAutoscalerSpec = "invalid json"; + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-hpa-error-0") != nullptr && poolManager->GetPodPool("pool-hpa-error-0")->status == 5; + }); + } + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->id = "pool-hpa-error-1"; + poolInfo->status = 2; + litebus::Future> future; + future.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(future)); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-hpa-error-1") != nullptr && poolManager->GetPodPool("pool-hpa-error-1")->status == 5; + }); + } + { + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->id = "pool-hpa-error-2"; + poolInfo->status = 2; + auto hpaList = std::make_shared(); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(hpaList)); + litebus::Future> future; + future.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockClient.get(), MOCKCreateNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(future)); + actor_->UpdatePodPoolEvent( + GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { + return poolManager->GetPodPool("pool-hpa-error-2") != nullptr && poolManager->GetPodPool("pool-hpa-error-2")->status == 5; + }); + } +} + +/** + * Feature: PodPoolWithHPATest + * Description: pod pool CRUD + * Steps: + * 1. Sync deployment and pod pool info with HPA + * 2. receive pod pool add event + * 3. receive pod pool update event + * 4. receive pod pool delete event + * Expectation: + * 1. Add Pod Pool + * 2. Update Pod Pool + * 3. Delete Pod pool + */ +#if 0 +TEST_F(ScalerTest, PodPoolWithHPATest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + actor_->SyncDeploymentAndPodPool().Get(); + actor_->Register(); + WaitRegisterComplete(); + actor_->SetKubeClient(mockClient); + auto poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->readyCount = 0; + poolInfo->size = 0; + EXPECT_TRUE(poolInfo->group == "rg1"); + EXPECT_TRUE(poolInfo->horizontalPodAutoscalerSpec == R"({"minReplicas": 1, "maxReplicas": 2, "metrics":[{"resource": {"name":"cpu", "target":{"averageUtilization":20, "type":"Utilization"}}, "type":"Resource"}, {"resource": {"name":"memory", "target":{"averageUtilization":50, "type":"Utilization"}}, "type":"Resource"}]})"); + EXPECT_TRUE(poolInfo->id == "pool1"); + + auto hpa1 = std::make_shared(); + hpa1->SetApiVersion("autoscaling/v2beta2"); + hpa1->SetKind("HorizontalPodAutoscaler"); + auto metaData = std::make_shared(); + metaData->SetName("function-agent-pool1-hpa"); + hpa1->SetMetadata(metaData); + EXPECT_CALL(*mockClient.get(), MOCKCreateNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillOnce(testing::Return(hpa1)); + + auto hpaList0 = std::make_shared(); + auto hpaList1 = std::make_shared(); + hpaList1->SetItems({hpa1}); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillOnce(testing::Return(hpaList0)) + .WillRepeatedly(testing::Return(hpaList1)); + + litebus::Future> mockDeploymentFuture; + EXPECT_CALL(*mockClient.get(), MockCreateNamespacedDeployment(testing::_)) + .WillRepeatedly(testing::DoAll(test::FutureArg<0>(&mockDeploymentFuture))); + + // Create + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo))); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1") != nullptr && poolManager->GetPodPool("pool1")->status == 1; }); + auto pool1Info = poolManager->GetPodPool("pool1"); + EXPECT_TRUE(pool1Info != nullptr); + EXPECT_TRUE(pool1Info->status == 1); + EXPECT_TRUE(pool1Info->id == "pool1"); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + pool1Info->id).IsSome(); }); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + pool1Info->id).Get().find(R"("status":1)") != std::string::npos; }); + + ASSERT_AWAIT_READY(mockDeploymentFuture); + EXPECT_EQ("function-agent-pool1", mockDeploymentFuture.Get()->GetMetadata()->GetName()); + EXPECT_EQ("rg1", mockDeploymentFuture.Get()->GetSpec()->GetRTemplate()->GetMetadata()->GetLabels()["rg1"]); + auto deploymentStatus = std::make_shared(); + deploymentStatus->SetAvailableReplicas(1); + deploymentStatus->SetReplicas(1); + mockDeploymentFuture.Get()->SetStatus(deploymentStatus); + actor_->OnDeploymentModified(K8sEventType::EVENT_TYPE_MODIFIED, mockDeploymentFuture.Get()); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1")->readyCount == 1; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + pool1Info->id).Get().find(R"("status":4)") != std::string::npos; }); + + // Update + EXPECT_CALL(*mockClient.get(), MOCKPatchNamespacedHorizontalPodAutoscaler(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(hpa1)); + pool1Info->status = 2; + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + pool1Info->id, poolManager->TransToJsonFromPodPoolInfo(pool1Info))); + auto deploymentStatus1 = std::make_shared(); + deploymentStatus1->SetAvailableReplicas(2); + deploymentStatus1->SetReplicas(0); + mockDeploymentFuture.Get()->SetStatus(deploymentStatus1); + actor_->OnDeploymentModified(K8sEventType::EVENT_TYPE_MODIFIED, mockDeploymentFuture.Get()); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + pool1Info->id).Get().find(R"("status":4)") != std::string::npos; }); + + // Delete + auto status = std::make_shared(); + auto mockClient3 = std::make_shared(); + actor_->SetKubeClient(mockClient3); + EXPECT_CALL(*mockClient3.get(), MOCKDeleteNamespacedHorizontalPodAutoscaler(testing::_, testing::_, testing::_, testing::_)) + .WillRepeatedly(testing::Return(status)); + bool isDeleteFinished = false; + EXPECT_CALL(*mockClient3.get(), DeleteNamespacedDeployment) + .WillRepeatedly(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(mockDeploymentFuture.Get()))); + pool1Info->status = 6; + actor_->UpdatePodPoolEvent(GetWatchEvents("/yr/podpools/info/" + pool1Info->id, poolManager->TransToJsonFromPodPoolInfo(pool1Info))); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return poolManager->GetPodPool("pool1") == nullptr; }); + metaStorageAccessor_->Delete("/yr/podpools/info/", true).Get(); +} +#endif + +/** + * Feature: PodPoolWithHPATest + * Description: pod pool CRUD + * Steps: + * 1. Sync deployment and pod pool info with HPA + * 2. receive pod pool add event + * 3. receive pod pool update event + * 4. receive pod pool delete event + * Expectation: + * 1. Add Pod Pool + * 2. Update Pod Pool + * 3. Delete Pod pool + */ +TEST_F(ScalerTest, CreatePodPoolWhenHPAExistTest) +{ + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + poolInfo->readyCount = 0; + poolInfo->status = 0; + poolInfo->size = 0; + EXPECT_TRUE(poolInfo->group == "rg1"); + EXPECT_TRUE(poolInfo->id == "pool1"); + EXPECT_TRUE(poolInfo->horizontalPodAutoscalerSpec == R"({"minReplicas": 1, "maxReplicas": 2, "metrics":[{"resource": {"name":"cpu", "target":{"averageUtilization":20, "type":"Utilization"}}, "type":"Resource"}, {"resource": {"name":"memory", "target":{"averageUtilization":50, "type":"Utilization"}}, "type":"Resource"}]})"); + + auto hpa1 = std::make_shared(); + hpa1->SetKind("HorizontalPodAutoscaler"); + hpa1->SetApiVersion("autoscaling/v2beta2"); + auto metaData = std::make_shared(); + metaData->SetName("function-agent-pool1-hpa"); + hpa1->SetMetadata(metaData); + + auto hpaList1 = std::make_shared(); + hpaList1->SetItems({hpa1}); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(hpaList1)); + + // Create + metaStorageAccessor_->Put("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo)).Get(); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + poolInfo->id).Get().find(R"("status":5)") != std::string::npos; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + poolInfo->id).Get().find(R"("msg":"create HPA failed")") != std::string::npos; }); + metaStorageAccessor_->Delete("/yr/podpools/info/", true).Get(); +} +TEST_F(ScalerTest, CreatePodPoolWithInvalidData) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + actor_->SyncDeploymentAndPodPool().Get(); + litebus::Future> mockDeploymentFuture; + EXPECT_CALL(*mockClient.get(), MockCreateNamespacedDeployment(testing::_)) + .WillRepeatedly(testing::DoAll(test::FutureArg<0>(&mockDeploymentFuture))); + actor_->Register(); + WaitRegisterComplete(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + poolInfo->id = "errPool1"; + poolInfo->affinities = "{\"requiredDuringSchedulingIgnoredDuringExecution\": {\"nodeSelectorTerms\": [{\"matchExpressions\": [{ \"key\": \"node-type\",\"operator\": \"In\",\"values\": [\"system\"]}]}]}}, }}"; + poolInfo->tolerations = "[{\"key\":\"node.kubernetes.io/disk-pressure\",\"operator\": \"Equal\", \"value\": \"true\", \"effect\": \"NoSchedule\"]"; + metaStorageAccessor_->Put("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo)).Get(); + ASSERT_AWAIT_READY(mockDeploymentFuture); + auto mockDeployment = mockDeploymentFuture.Get(); + EXPECT_EQ("function-agent-errPool1", mockDeployment->GetMetadata()->GetName()); + EXPECT_EQ(static_cast(0), + mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetTolerations().size()); + EXPECT_FALSE(mockDeployment->GetSpec()->GetRTemplate()->GetSpec()->GetAffinity()->NodeAffinityIsSet()); + metaStorageAccessor_->Delete("/yr/podpools/info", true).Get(); +} + +TEST_F(ScalerTest, UpdatePodPoolWhenHPANotExistTest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_WITH_HPA_INFO_STR, poolInfo); + poolInfo->status = 2; + poolInfo->readyCount = 0; + poolInfo->size = 0; + EXPECT_TRUE(poolInfo->id == "pool1"); + EXPECT_TRUE(poolInfo->group == "rg1"); + EXPECT_TRUE(poolInfo->horizontalPodAutoscalerSpec == R"({"minReplicas": 1, "maxReplicas": 2, "metrics":[{"resource": {"name":"cpu", "target":{"averageUtilization":20, "type":"Utilization"}}, "type":"Resource"}, {"resource": {"name":"memory", "target":{"averageUtilization":50, "type":"Utilization"}}, "type":"Resource"}]})"); + + auto hpaList0 = std::make_shared(); + EXPECT_CALL(*mockClient.get(), MOCKListNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(hpaList0)); + + auto hpa1 = std::make_shared(); + hpa1->SetApiVersion("autoscaling/v2beta2"); + hpa1->SetKind("HorizontalPodAutoscaler"); + auto metaData = std::make_shared(); + metaData->SetName("function-agent-pool1-hpa"); + hpa1->SetMetadata(metaData); + EXPECT_CALL(*mockClient.get(), MOCKCreateNamespacedHorizontalPodAutoscaler(testing::_, testing::_)) + .WillRepeatedly(testing::Return(hpa1)); + + actor_->Register(); + WaitRegisterComplete(); + actor_->SetKubeClient(mockClient); + // Update + metaStorageAccessor_->Put("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo)).Get(); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + poolInfo->id).Get().find(R"("status":5)") != std::string::npos; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return metaStorageAccessor_->Get("/yr/podpools/info/" + poolInfo->id).Get().find(R"("msg":"pool not found")") != std::string::npos; }); + metaStorageAccessor_->Delete("/yr/podpools/info", true).Get(); +} + +TEST_F(ScalerTest, SystemUpgradeTest) +{ + actor_->Register(); + WaitRegisterComplete(); + GetTaintNodeList(); + EXPECT_FALSE(actor_->GetIsUpgrading()); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"(fake json)"); + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 0, "status": 1})"); + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 0, "status": 100})"); + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"status": 1})"); + ASSERT_AWAIT_TRUE([&]() { return actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 1, "status": 2})"); + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 1, "status": 1})"); + ASSERT_AWAIT_TRUE([&]() { return actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 1, "status": 2})"); + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Put(DEFAULT_SYSTEM_UPGRADE_KEY + "/zoneid", R"({"azId": 1, "status": 1})"); + ASSERT_AWAIT_TRUE([&]() { return actor_->GetIsUpgrading(); }); + + metaStorageAccessor_->Delete(DEFAULT_SYSTEM_UPGRADE_KEY, true); + + ASSERT_AWAIT_TRUE([&]() { return !actor_->GetIsUpgrading(); }); +} + +TEST_F(ScalerTest, GetPodDeleteOptionsTest) +{ + auto opt = actor_->GetPodDeleteOptions("podName"); + EXPECT_TRUE(opt.IsSome()); + EXPECT_EQ(opt.Get()->GetGracePeriodSeconds(), 25); + + opt = actor_->GetPodDeleteOptions("podName", true); + EXPECT_TRUE(opt.IsSome()); + EXPECT_EQ(opt.Get()->GetGracePeriodSeconds(), 25); + + opt = actor_->GetPodDeleteOptions("podName", false); + EXPECT_TRUE(opt.IsSome()); + EXPECT_FALSE(opt.Get()->GracePeriodSecondsIsSet()); +} + +TEST_F(ScalerTest, SlaveTest) +{ + auto member = std::make_shared(); + auto slaveBusiness = std::make_shared(actor_, member); + slaveBusiness->OnChange(); + slaveBusiness->UpdatePodLabels({}, false); + slaveBusiness->DeletePod(""); + slaveBusiness->CreateResourcePools(nullptr); + slaveBusiness->CreatePodPools({}); + slaveBusiness->CreateAgent({}, "", ""); + slaveBusiness->CreatePodPool(nullptr); + slaveBusiness->UpdatePodPool(nullptr); + slaveBusiness->DeletePodPool(nullptr); + slaveBusiness->UpdateNodeTaint({}, "", ""); + slaveBusiness->DeletePodNotBindInstance(nullptr); + slaveBusiness->MigratePodInstanceWithTaints("", ""); + slaveBusiness->MigrateNodeInstanceWithTaints(nullptr); + slaveBusiness->UpdatePodLabelsWithoutInstance(nullptr); + slaveBusiness->PersistencePoolInfo(""); + slaveBusiness->ScaleUpPodByPoolID("", false, "", litebus::AID()); +} + +TEST_F(ScalerTest, ParseLabelWithAnnotationTest) +{ + resource_view::InstanceInfo instance1 = + CreateInstance(INSTANCE_PATH_PREFIX + "/ins-0001", static_cast(InstanceState::RUNNING)); + (*instance1.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{}"; + auto readyPod = GetReadyPod(); + HandlePodLabelsWithAnnotation(instance1, readyPod, false); + EXPECT_EQ(1, readyPod->GetMetadata()->GetLabels().size()); + EXPECT_EQ(0, readyPod->GetMetadata()->GetAnnotations().size()); + (*instance1.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{\"label1\":\"va1\", \"label2\":\"val2\",\"\":\"\"}"; + HandlePodLabelsWithAnnotation(instance1, readyPod, false); + EXPECT_EQ(3, readyPod->GetMetadata()->GetLabels().size()); + EXPECT_EQ(1, readyPod->GetMetadata()->GetAnnotations().size()); + HandlePodLabelsWithAnnotation(instance1, readyPod, true); + EXPECT_EQ(1, readyPod->GetMetadata()->GetLabels().size()); + EXPECT_EQ(0, readyPod->GetMetadata()->GetAnnotations().size()); + auto newPod = GetReadyPod(); + newPod->GetMetadata()->GetLabels()["label2"] = "val2"; + newPod->GetMetadata()->GetLabels()["label3"] = "val3"; + newPod->GetMetadata()->GetLabels()["label4"] = "val4"; + resource_view::InstanceInfo instance2 = + CreateInstance(INSTANCE_PATH_PREFIX + "/ins-0002", static_cast(InstanceState::FATAL)); + (*instance2.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{\"label2\":\"val2\"}"; + resource_view::InstanceInfo instance3 = + CreateInstance(INSTANCE_PATH_PREFIX + "/ins-0003", static_cast(InstanceState::SCHEDULE_FAILED)); + (*instance3.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{\"label3\":\"va3\"}"; + newPod->GetMetadata()->GetAnnotations()["yr-labels-" + instance2.instanceid()] = "{\"label2\":\"va2\"}"; + newPod->GetMetadata()->GetAnnotations()["yr-labels-" + instance3.instanceid()] = "{\"label3\":\"va3\"}"; + HandlePodLabelsWithAnnotation(instance2, newPod, false); + HandlePodLabelsWithAnnotation(instance3, newPod, false); + EXPECT_EQ(2, newPod->GetMetadata()->GetLabels().size()); + EXPECT_EQ(0, newPod->GetMetadata()->GetAnnotations().size()); +} + +TEST_F(ScalerTest, WatchSharedInstanceTest) +{ + // 1. Delete all instances + metaStorageAccessor_->Delete(INSTANCE_PATH_PREFIX, true).Get(); + // 2. Watch instance's events + litebus::Async(actor_->GetAID(), &ScalerActor::Register); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); // synchronize + auto pod1 = std::make_shared(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("127.0.0.1"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("function-agent-001"); + pod1->GetMetadata()->SetLabels({}); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + pod1->GetMetadata()->SetAnnotations({}); + std::string jsonString0; + resource_view::InstanceInfo instance0 = + CreateInstance(INSTANCE_PATH_PREFIX + "/00x", static_cast(InstanceState::RUNNING)); + instance0.set_instanceid("00x"); + (*instance0.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{\"label1\":\"val1\"}"; + instance0.mutable_scheduleoption()->set_schedpolicyname("shared"); + instance0.set_functionagentid(""); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString0, instance0)); + std::string jsonString1; + resource_view::InstanceInfo instance1 = + CreateInstance(INSTANCE_PATH_PREFIX + "/001", static_cast(InstanceState::RUNNING)); + instance1.set_instanceid("001"); + (*instance1.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{\"label1\":\"val1\"}"; + instance1.mutable_scheduleoption()->set_schedpolicyname("shared"); + ASSERT_TRUE(TransToJsonFromInstanceInfo(jsonString1, instance1)); + litebus::Future> body; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillOnce(testing::Return(pod1)); + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillOnce(testing::DoAll(FutureArg<2>(&body), testing::Return(pod1))); + // 3. Put two instances to MetaStore. + metaStorageAccessor_->Put(INSTANCE_PATH_PREFIX + "/00x", jsonString0).Get(); + metaStorageAccessor_->Put(INSTANCE_PATH_PREFIX + "/001", jsonString1).Get(); + ASSERT_AWAIT_READY(body); + EXPECT_TRUE(ContainValue(body.Get(), "label1")); + EXPECT_TRUE(ContainValue(body.Get(), "yr-labels-" + instance1.instanceid())); + body = {}; + pod1->GetMetadata()->GetLabels()["label1"] = "val1"; + pod1->GetMetadata()->GetAnnotations()["yr-labels-" + instance1.instanceid()] = "{\"label1\":\"val1\"}"; + litebus::Future> body1; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::Return(pod1)); + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillRepeatedly(testing::DoAll(FutureArg<2>(&body1), testing::Return(pod1))); + // 4. Delete the second instance. + metaStorageAccessor_->Delete(INSTANCE_PATH_PREFIX + "/001").Get(); + ASSERT_AWAIT_READY(body1); + EXPECT_TRUE(ContainValue(body1.Get(), "label1")); + EXPECT_TRUE(ContainValue(body1.Get(), "yr-labels-" + instance1.instanceid())); + EXPECT_FALSE(ContainValue(body1.Get(), "yr-default")); + EXPECT_TRUE(pod1->GetMetadata()->GetAnnotations().find("yr-default") != pod1->GetMetadata()->GetAnnotations().end()); + actor_->SetPodMap({}); // synchronize + actor_->SetNodeMap({}); // synchronize +} + +TEST_F(ScalerTest, UpdatePodLabelsWithoutInstanceTest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto newPod = GetReadyPod(); + newPod->GetMetadata()->GetLabels()["label2"] = "val2"; + newPod->GetMetadata()->GetLabels()["label3"] = "val3"; + newPod->GetMetadata()->GetLabels()["label4"] = "val4"; + newPod->GetMetadata()->SetAnnotations({}); + newPod->GetMetadata()->GetAnnotations()["yr-labels-ins-00033333"] = "{\"label2\":\"va2\"}"; + newPod->GetMetadata()->GetAnnotations()["yr-labels-ins-00044444"] = "{\"label3\":\"va3\"}"; + litebus::Future> body; + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillOnce(testing::DoAll(FutureArg<2>(&body), testing::Return(newPod))); + actor_->UpdatePodLabelsWithoutInstance(newPod); + ASSERT_AWAIT_READY(body); + EXPECT_TRUE(ContainValue(body.Get(), "/metadata/labels/label2")); + EXPECT_TRUE(ContainValue(body.Get(), "/metadata/labels/label3")); +} + +TEST_F(ScalerTest, MigrateInstanceTest) +{ + actor_->Register(); + WaitRegisterComplete(); + const char *argv[] = { "./function_master","--node_id=node1", "--ip=127.0.0.1:22771", + "--meta_store_address=127.0.0.1:32279", + "--self_taint_prefix=az1/" }; + functionmaster::Flags flags; + ASSERT_TRUE(flags.ParseFlags(5, argv).IsNone()); + actor_->ParseParams(flags); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto node = GetReadyNode(); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + node->GetMetadata()->SetName("10.10.10.01"); + actor_->MigratePodInstanceWithTaints("10.10.10.01", "function-agent-001"); + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + actor_->SetNodeMap(nodeMap); + actor_->MigratePodInstanceWithTaints("10.10.10.01", "function-agent-001"); + auto pod1 = std::make_shared(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("10.0.0.1"); + pod1->GetStatus()->SetHostIP("10.10.10.01"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("function-agent-001"); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + pod1->GetMetadata()->GetLabels()["app"] = "function-agent-001"; + auto podSpec = std::make_shared(); + podSpec->SetTerminationGracePeriodSeconds(60); + pod1->SetSpec(podSpec); + bool isFinished = false; + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillOnce(testing::Return(pod1)); + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillRepeatedly(testing::DoAll(testing::Assign(&isFinished, true),testing::Return(pod1))); + std::string jsonString1; + resource_view::InstanceInfo instance1 = + CreateInstance(INSTANCE_PATH_PREFIX + "/0011", static_cast(InstanceState::RUNNING)); + instance1.set_functionagentid("function-agent-001"); + instance1.set_functionproxyid("10.10.10.01"); + TransToJsonFromInstanceInfo(jsonString1, instance1); + metaStorageAccessor_->Put(instance1.instanceid(), jsonString1).Get(); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + auto nodeTaint = std::make_shared(); + nodeTaint->SetKey("az1/is-ds-worker-unready"); + nodeTaint->SetValue("true"); + nodeTaint->SetEffect("PreferNoSchedule"); + node->GetSpec()->GetTaints().push_back(nodeTaint); + nodeMap["10.10.10.01"] = node; + actor_->SetNodeMap(nodeMap); + actor_->MigratePodInstanceWithTaints("10.10.10.01", "function-agent-001"); + EXPECT_EQ(*counter, 1); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); +} + +TEST_F(ScalerTest, WatchEvictedNodeTaint) +{ + actor_->Register(); + WaitRegisterComplete(); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); + + auto localActor = std::make_shared("evicted-node" + LOCAL_SCHED_FUNC_AGENT_MGR_ACTOR_NAME_POSTFIX); + litebus::Spawn(localActor); + + auto node = GetReadyNode(); + node->GetMetadata()->SetName("evicted-node"); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + auto nodeTaint = std::make_shared(); + // add taint + nodeTaint->SetKey("evicted-taint-key"); + nodeTaint->SetValue("true"); + nodeTaint->SetEffect("PreferNoSchedule"); + node->GetSpec()->SetTaints({}); + node->GetSpec()->GetTaints().push_back(nodeTaint); + actor_->SetEvictedTaintKey("evicted-taint-key"); + + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_ADD, node); + EXPECT_AWAIT_TRUE([&]() -> bool { return localActor->GetLocalStatusRequest().status() == 4; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->member_->hasEvictedTaint; }); + + // remove taint + node->GetSpec()->SetTaints({}); + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_ADD, node); + EXPECT_AWAIT_TRUE([&]() -> bool { return localActor->GetLocalStatusRequest().status() == 1; }); + EXPECT_AWAIT_TRUE([&]() -> bool { return !actor_->member_->hasEvictedTaint; }); + + // create pod - after pod instances migration is completed, the annotation is removed + node->GetSpec()->GetTaints().push_back(nodeTaint); + auto pod = std::make_shared(); + pod->SetStatus(std::make_shared()); + pod->GetStatus()->SetHostIP("10.10.10.01"); + pod->SetMetadata(std::make_shared()); + pod->GetMetadata()->SetName("function-agent-001"); + pod->GetMetadata()->GetAnnotations()["cluster-autoscaler.kubernetes.io/safe-to-evict"] = "false"; + std::unordered_map> podNameMap; + podNameMap["function-agent-001"] = pod; + actor_->SetPodMap(podNameMap); + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + actor_->SetNodeMap(nodeMap); + + bool isFinished = false, isFinished1 = false; + EXPECT_CALL(*mockClient, PatchNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return(pod))) + .WillOnce(testing::DoAll(testing::Assign(&isFinished1, true), testing::Return(pod))); + actor_->MigratePodInstanceWithTaints("10.10.10.01", "function-agent-001"); + EXPECT_EQ(*counter, 1); + EXPECT_AWAIT_TRUE([&isFinished]() -> bool { return isFinished; }); + // add annotation + resource_view::InstanceInfo instance = CreateInstance(INSTANCE_PATH_PREFIX + "/0011", static_cast(InstanceState::RUNNING)); + instance.set_functionagentid("function-agent-001"); + instance.set_functionproxyid("node1"); + instance.mutable_scheduleoption()->set_schedpolicyname(""); + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillOnce(testing::Return(pod)); + actor_->HandleInstancePut(instance); + EXPECT_AWAIT_TRUE([&isFinished1]() -> bool { return isFinished1; }); + actor_->OnNodeModified(K8sEventType::EVENT_TYPE_DELETE, node); + ASSERT_AWAIT_TRUE([&]() -> bool {return actor_->member_->nodes.size() == 0;}); + litebus::Terminate(localActor->GetAID()); + litebus::Await(localActor); + localActor = nullptr; +} + +TEST_F(ScalerTest, KeySyncerGetFailedOrEmptyTest) +{ + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + auto originClient = metaStorageAccessor_->GetMetaClient(); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + { // for get failed + std::shared_ptr rep = std::make_shared(); + rep->status = Status(StatusCode::FAILED, ""); + + auto future = actor_->SystemUpgradeSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_FALSE(future.Get().status.IsOk()); + future = actor_->InstanceInfoSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_FALSE(future.Get().status.IsOk()); + future = actor_->PodPoolSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_FALSE(future.Get().status.IsOk()); + } + { // for get response is empty + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + + auto future = actor_->SystemUpgradeSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + future = actor_->InstanceInfoSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + future = actor_->PodPoolSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + } + + // for reset originClient, if not reset, will delete failed in teardown() + metaStorageAccessor_->metaClient_ = originClient; +} + +TEST_F(ScalerTest, SystemUpgradeSyncerSuccessTest) +{ + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + auto originClient = metaStorageAccessor_->GetMetaClient(); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + actor_->member_->systemUpgradeParam.systemUpgradeWatcher = metaStorageAccessor_; + + { // for get response is not empty + KeyValue getKeyValue; + getKeyValue.set_key(DEFAULT_SYSTEM_UPGRADE_KEY+"AZ") ; + getKeyValue.set_value("Node1"); + + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + rep->kvs.emplace_back(getKeyValue); + + auto future = actor_->SystemUpgradeSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + } + // for reset originClient, if not reset, will delete failed in teardown() + metaStorageAccessor_->metaClient_ = originClient; +} + +std::vector GenerateResponseEvent() +{ + std::vector events; + std::list> kvMap = { + {instanceKey1, instanceInfoJson1}, + {instanceKey2, instanceInfoJson2}, + {instanceKey3, instanceInfoJson3}, + }; + + for (auto elem : kvMap) { + KeyValue inst1; + inst1.set_value(elem.second); + inst1.set_key(elem.first) ; + + WatchEvent event{ .eventType = EVENT_TYPE_PUT, .kv = inst1, .prevKv = {} }; + events.emplace_back(event); + } + return events; +} + +std::vector> GeneratePodlist() +{ + auto pod1 = std::make_shared(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("10.42.2.101"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("functionagent-pool1-776c6db574-1"); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + + auto pod2 = std::make_shared(); + pod2->SetStatus(std::make_shared()); + pod2->GetStatus()->SetPodIP("10.42.2.102"); + pod2->SetMetadata(std::make_shared()); + pod2->GetMetadata()->SetName("functionagent-pool1-776c6db574-2"); + pod2->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + + auto pod3 = std::make_shared(); + pod3->SetStatus(std::make_shared()); + pod3->GetStatus()->SetPodIP("10.42.2.103"); + pod3->SetMetadata(std::make_shared()); + pod3->GetMetadata()->SetName("functionagent-pool1-776c6db574-3"); + pod3->GetMetadata()->GetLabels()["yr-reuse"] = "false"; + + std::vector> podlist({pod1, pod2, pod3}); + return podlist; +} + +TEST_F(ScalerTest, InstanceInfoSyncerTest) +{ + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + auto originClient = metaStorageAccessor_->GetMetaClient(); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + + // for get response is not empty + auto events = GenerateResponseEvent(); + auto podlist = GeneratePodlist(); + std::vector targetPod({0}); // only put instance1 into cache + for (auto i : targetPod) { + EXPECT_CALL(*mockClient, ReadNamespacedPod).WillRepeatedly(testing::Return(podlist[i])); + EXPECT_CALL(*mockClient, PatchNamespacedPod).WillRepeatedly(testing::Return(podlist[i])); + std::vector tmpEvents; + tmpEvents.emplace_back(events[i]); + actor_->UpdateInstanceEvent(tmpEvents); + } + + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + rep->kvs.emplace_back(events[0].kv); + rep->kvs.emplace_back(events[1].kv); + + std::string jsonString1; + resource_view::InstanceInfo instance1 = CreateInstance("instanceID4", static_cast(InstanceState::RUNNING)); + instance1.set_functionagentid("function-agent-001"); + instance1.set_functionproxyid("10.10.10.01"); + instance1.set_lowreliability(true); + TransToJsonFromInstanceInfo(jsonString1, instance1); + KeyValue inst1; + inst1.set_value(jsonString1); + inst1.set_key("/sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-system-faasExecutorPython3.9/version/$latest/defaultaz/requestID4/instanceID4"); + rep->kvs.emplace_back(inst1); + + auto future = actor_->InstanceInfoSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + + // instance3 expect to delete + EXPECT_TRUE(actor_->member_->instanceId2PodName.count("instanceID3") == 0); + EXPECT_TRUE(actor_->member_->podName2InstanceId.count("functionagent-pool1-776c6db574-3") == 0); + + // instance2 expect to add + EXPECT_TRUE(actor_->member_->instanceId2PodName.count("instanceID2") == 1); + EXPECT_TRUE(actor_->member_->podName2InstanceId.count("functionagent-pool1-776c6db574-2") == 1); + + EXPECT_TRUE(actor_->member_->instanceId2PodName.count("instanceID4") == 0); + // for reset originClient, if not reset, will delete failed in teardown() + metaStorageAccessor_->metaClient_ = originClient; +} + +std::vector GenerateResponsePodPoolEvent() +{ + + auto key1 = R"(/yr/podpools/info/pool-not-incache)"; + auto value1 = R"({"affinities":"","environment":null,"group":"rg1","horizontal_pod_autoscaler_spec":"","id":"pool-not-incache","image":"","init_image":"","labels":null,"msg":"Running","node_selector":null,"ready_count":1,"resources":{"liits":{"cpu":"600m","memory":"512Mi"},"requests":{"cpu":"600m","memory":"512Mi"}},"reuse":false,"runtime_class_name":"","size":1,"status":4,"tolerations":"","volume_mounts":"","volumes":""})"; + + auto key2 = R"(/yr/podpools/info/pool-not-inetcd)"; + auto value2 = R"({"affinities":"","environment":null,"group":"rg1","horizontal_pod_autoscaler_spec":"","id":"pool-not-inetcd","image":"","init_image":"","labels":null,"msg":"Running","node_selector":null,"ready_count":1,"resources":{"liits":{"cpu":"600m","memory":"512Mi"},"requests":{"cpu":"600m","memory":"512Mi"}},"reuse":false,"runtime_class_name":"","size":1,"status":4,"tolerations":"","volume_mounts":"","volumes":""})"; + + auto key3 = R"(/yr/podpools/info/pool-inboth)"; + auto value3status1 = R"({"affinities":"","environment":null,"group":"rg1","horizontal_pod_autoscaler_spec":"","id":"pool-inboth","image":"","init_image":"","labels":null,"msg":"Running","node_selector":null,"ready_count":1,"resources":{"liits":{"cpu":"600m","memory":"512Mi"},"requests":{"cpu":"600m","memory":"512Mi"}},"reuse":false,"runtime_class_name":"","size":1,"status":1,"tolerations":"","volume_mounts":"","volumes":""})"; + auto value3status4 = R"({"affinities":"","environment":null,"group":"rg1","horizontal_pod_autoscaler_spec":"","id":"pool-inboth","image":"","init_image":"","labels":null,"msg":"Running","node_selector":null,"ready_count":1,"resources":{"liits":{"cpu":"600m","memory":"512Mi"},"requests":{"cpu":"600m","memory":"512Mi"}},"reuse":false,"runtime_class_name":"","size":1,"status":4,"tolerations":"","volume_mounts":"","volumes":""})"; + + auto key4 = R"(/yr/podpools/info/pool-inboth-delete)"; + auto value4 = R"({"affinities":"","environment":null,"group":"rg1","horizontal_pod_autoscaler_spec":"","id":"pool-inboth-delete","image":"","init_image":"","labels":null,"msg":"Running","node_selector":null,"ready_count":1,"resources":{"liits":{"cpu":"600m","memory":"512Mi"},"requests":{"cpu":"600m","memory":"512Mi"}},"reuse":false,"runtime_class_name":"","size":1,"status":6,"tolerations":"","volume_mounts":"","volumes":""})"; + + std::vector events; + std::list> kvMap = { + {key1, value1}, + {key2, value2}, + {key3, value3status1}, + {key3, value3status4}, + {key4, value4}, + }; + + for (auto elem : kvMap) { + KeyValue inst1; + inst1.set_key(elem.first) ; + inst1.set_value(elem.second); + WatchEvent event{ .eventType = EVENT_TYPE_PUT, .kv = inst1, .prevKv = {} }; + events.emplace_back(event); + } + return events; +} + +TEST_F(ScalerTest, PodPoolSyncerTest) +{ + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + auto originClient = metaStorageAccessor_->GetMetaClient(); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + + auto events = GenerateResponsePodPoolEvent(); + std::vector targetPod({1, 2, 4}); // put key2, key3(status 1),, key4 in cache + for (auto i : targetPod) { + auto podPool = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(events[i].kv.value(), podPool); + PoolManager::TransToPoolInfoFromJson(events[i].kv.value(), + actor_->member_->poolManager->GetOrNewPool(podPool->id)); + } + + // put key1, key3(status 4), key4 in etcd + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + rep->kvs.emplace_back(events[0].kv); + rep->kvs.emplace_back(events[3].kv); + rep->kvs.emplace_back(events[4].kv); + + auto future = actor_->PodPoolSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + + // test exist in etcd but not in cache, need update + EXPECT_TRUE(actor_->member_->poolManager->GetPodPool("pool-not-incache") != nullptr); + + // test exist both in etcd but cache and value isn't consistent, need update + EXPECT_TRUE(actor_->member_->poolManager->GetPodPool("pool-inboth")->status == static_cast(PoolState::RUNNING)); + + // test poolId not in etcd, need to remove from poolMap + EXPECT_TRUE(actor_->member_->poolManager->GetPodPool("pool-inboth-inetcd") == nullptr); + + // test exist both in etcd but cache and value is consistent, but state is DELETED, reUpdate + EXPECT_TRUE(actor_->member_->poolManager->GetDeployment(POOL_NAME_PREFIX +"pool-inboth-delete") == nullptr); + // for reset originClient, if not reset, will delete failed in teardown() + metaStorageAccessor_->metaClient_ = originClient; +} + +TEST_F(ScalerTest, DeletePodRequest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + { + // delete error + auto req = std::make_shared(); + req->set_requestid("req-123"); + req->set_functionagentid("function-agent-0001"); + litebus::Promise> errPromise; + errPromise.SetFailed(504); + litebus::Future deletePodCalledArg; + EXPECT_CALL(*mockClient, DeleteNamespacedPod).WillOnce(testing::Return(errPromise.GetFuture())); + litebus::Future> deletePodResponseArg; + EXPECT_CALL(*testActor_.get(), MockDeletePodResponse).WillOnce(test::FutureArg<0>(&deletePodResponseArg)); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::DeletePodRequest, actor_->GetAID(), req->SerializeAsString()); + ASSERT_AWAIT_READY(deletePodResponseArg); + EXPECT_EQ(deletePodResponseArg.Get()->code(), 504); + } + + { + // delete success with 404 + auto req = std::make_shared(); + req->set_requestid("req-123"); + req->set_functionagentid("function-agent-0001"); + litebus::Promise> errPromise; + errPromise.SetFailed(404); + EXPECT_CALL(*mockClient, DeleteNamespacedPod).WillOnce(testing::Return(errPromise.GetFuture())); + litebus::Future> deletePodResponseArg; + EXPECT_CALL(*testActor_.get(), MockDeletePodResponse).WillOnce(test::FutureArg<0>(&deletePodResponseArg)); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::DeletePodRequest, actor_->GetAID(), req->SerializeAsString()); + ASSERT_AWAIT_READY(deletePodResponseArg); + EXPECT_EQ(deletePodResponseArg.Get()->code(), 0); + } + + { + // delete success + auto req = std::make_shared(); + req->set_requestid("req-123"); + req->set_functionagentid("function-agent-0001"); + EXPECT_CALL(*mockClient, DeleteNamespacedPod).WillOnce(testing::Return(GetReadyPod())); + litebus::Future> deletePodResponseArg; + EXPECT_CALL(*testActor_.get(), MockDeletePodResponse).WillOnce(test::FutureArg<0>(&deletePodResponseArg)); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::DeletePodRequest, actor_->GetAID(), req->SerializeAsString()); + ASSERT_AWAIT_READY(deletePodResponseArg); + EXPECT_EQ(deletePodResponseArg.Get()->code(), 0); + } +} + +TEST_F(ScalerTest, ResourceIsSyncedTest) { + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + + std::vector> podItems; + std::shared_ptr podList = std::make_shared(); + podList->SetItems(podItems); + EXPECT_CALL(*mockClient, ListNamespacedPod).WillRepeatedly(testing::DoAll(testing::Return(podList))); + litebus::Future> mockDeploymentFuture; + bool isDeleteFinished = false; + EXPECT_CALL(*mockClient.get(), DeleteNamespacedDeployment) + .Times(2) + .WillRepeatedly(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(mockDeploymentFuture))); + auto member = std::make_shared(); + auto masterBusiness = std::make_shared(actor_, actor_->member_); + actor_->member_->isSynced = true; + masterBusiness->OnChange(); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); + + std::shared_ptr poolInfo = std::make_shared(); + PoolManager::TransToPoolInfoFromJson(POD_POOL_INFO_STR, poolInfo); + metaStorageAccessor_->Put("/yr/podpools/info/" + poolInfo->id, poolManager->TransToJsonFromPodPoolInfo(poolInfo)).Get(); + EXPECT_CALL(*mockClient.get(), DeleteNamespacedDeployment).Times(0); + actor_->member_->isSynced = false; + masterBusiness->OnChange(); + EXPECT_EQ(2, actor_->GetPoolDeploymentsMap().size()); + metaStorageAccessor_->Delete("/yr/podpools/info", true).Get(); +} + +TEST_F(ScalerTest, DeletePodNotBindInstanceTest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); + + EXPECT_AWAIT_TRUE([&]() -> bool { return actor_->curStatus_ == "master"; }); + auto node = GetReadyNode(); + node->GetMetadata()->SetName("10.10.10.01"); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + //actor_->SetNodeMap(nodeMap); + + auto pod1 = GetReadyPod(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPodIP("10.0.0.1"); + pod1->GetStatus()->SetHostIP("10.10.10.01"); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName("function-agent-001"); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "true"; + pod1->GetMetadata()->GetLabels()["app"] = "function-agent"; + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + ASSERT_AWAIT_TRUE([&]() -> bool {return actor_->member_->podNameMap.size() == 1;}); + pod1->GetMetadata()->GetLabels()["schedule-policy"] = "monopoly"; + auto isDeleteFinished = false; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(pod1))); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + EXPECT_AWAIT_TRUE([&isDeleteFinished]() -> bool { return isDeleteFinished; }); +} + +TEST_F(ScalerTest, MergeNodeAffinityCoverage) +{ + ::resources::InstanceInfo instanceInfo; + EXPECT_FALSE(IsAggregationMergePolicy(instanceInfo)); + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_NODE_AFFINITY_POLICY") = ""; + EXPECT_FALSE(IsAggregationMergePolicy(instanceInfo)); + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_NODE_AFFINITY_POLICY") = "coverage"; + bool mergePolicy = IsAggregationMergePolicy(instanceInfo); + auto nodeAffinity1 = std::make_shared(); + nlohmann::json json = nlohmann::json::parse(NODE_AFFINITY_TEMPLATE_STR); + nodeAffinity1->FromJson(json); + // src and dest is null + EXPECT_TRUE(MergeNodeAffinity(nullptr, nullptr, mergePolicy) == nullptr); + // src is null + auto mergeAff = MergeNodeAffinity(nullptr, nodeAffinity1, mergePolicy); + EXPECT_TRUE(mergeAff->RequiredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_TRUE(mergeAff->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + // dest is null + mergeAff = MergeNodeAffinity(nodeAffinity1, nullptr, mergePolicy); + EXPECT_TRUE(mergeAff->RequiredDuringSchedulingIgnoredDuringExecutionIsSet()); + EXPECT_TRUE(mergeAff->PreferredDuringSchedulingIgnoredDuringExecutionIsSet()); + auto nodeAffinity2 = std::make_shared(); + nodeAffinity2->FromJson(json); + nodeAffinity2->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms()[0]->GetMatchExpressions()[0]->SetKey("test"); + nodeAffinity2->GetPreferredDuringSchedulingIgnoredDuringExecution()[0]->SetWeight(999); + // both src and dest set affinity + mergeAff = MergeNodeAffinity(nodeAffinity1, nodeAffinity2, mergePolicy); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms()[0]->GetMatchExpressions()[0]->GetKey(), "test"); + EXPECT_EQ(mergeAff->GetPreferredDuringSchedulingIgnoredDuringExecution()[0]->GetWeight(), 999); +} + +TEST_F(ScalerTest, MergeNodeAffinityAggregation) +{ + ::resources::InstanceInfo instanceInfo; + instanceInfo.mutable_createoptions()->operator[]("DELEGATE_NODE_AFFINITY_POLICY") = "aggregation"; + bool mergePolicy = IsAggregationMergePolicy(instanceInfo); + auto nodeAffinity1 = std::make_shared(); + nlohmann::json json = nlohmann::json::parse(NODE_AFFINITY_TEMPLATE_STR); + nodeAffinity1->FromJson(json); + auto nodeAffinity2 = std::make_shared(); + nodeAffinity2->FromJson(json); + nodeAffinity2->UnsetRequiredDuringSchedulingIgnoredDuringExecution(); + // src set require, dest not set require + auto mergeAff = MergeNodeAffinity(nodeAffinity1, nodeAffinity2, mergePolicy); + EXPECT_EQ(mergeAff->GetPreferredDuringSchedulingIgnoredDuringExecution().size(), 2); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().size(), 2); + // src not set require, dest set require + mergeAff = MergeNodeAffinity(nodeAffinity2, nodeAffinity1, mergePolicy); + EXPECT_EQ(mergeAff->GetPreferredDuringSchedulingIgnoredDuringExecution().size(), 2); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().size(), 2); + nodeAffinity2->FromJson(json); + mergeAff = MergeNodeAffinity(nodeAffinity1, nodeAffinity2, mergePolicy); + EXPECT_EQ(mergeAff->GetPreferredDuringSchedulingIgnoredDuringExecution().size(), 2); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().size(), 4); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution() + ->GetNodeSelectorTerms()[0] + ->GetMatchExpressions() + .size(), + 4); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution() + ->GetNodeSelectorTerms()[0] + ->GetMatchFields() + .size(), + 4); + // exceed limit + nodeAffinity1->FromJson(json); + nodeAffinity2->FromJson(json); + auto nodeSelectorJson = + nodeAffinity1->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms()[0]->ToJson(); + for (int i = 0; i < 11; i++) { + auto item = std::make_shared(); + item->FromJson(nodeSelectorJson); + nodeAffinity1->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().emplace_back(item); + nodeAffinity2->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().emplace_back(item); + } + mergeAff = MergeNodeAffinity(nodeAffinity1, nodeAffinity2, mergePolicy); + EXPECT_EQ(mergeAff->GetRequiredDuringSchedulingIgnoredDuringExecution()->GetNodeSelectorTerms().size(), 100); +} + +TEST_F(ScalerTest, AbnormalPodAlarm) +{ + auto masterBusiness = std::make_shared(actor_, actor_->member_); + actor_->business_ = masterBusiness; + metrics::MetricsAdapter::GetInstance().enabledInstruments_.insert(metrics::YRInstrument::YR_POD_ALARM); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); + + auto node = GetReadyNode(); + node->GetMetadata()->SetName("10.10.10.01"); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + actor_->SetNodeMap(nodeMap); + auto pod1 = GetReadyPod(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPhase("Failed"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) != alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); + + auto pod2 = GetReadyPod(); + pod2->SetStatus(std::make_shared()); + pod2->GetStatus()->SetPhase("Unknown"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod2); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) != alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); + + pod2->GetStatus()->SetPhase("Succeeded"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod2); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_; + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); + + pod2->GetStatus()->SetPhase("Failed"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_DELETE, pod2); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_; + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); +} + +TEST_F(ScalerTest, PendingPodAlarm) +{ + auto masterBusiness = std::make_shared(actor_, actor_->member_); + actor_->business_ = masterBusiness; + metrics::MetricsAdapter::GetInstance().enabledInstruments_.insert(metrics::YRInstrument::YR_POD_ALARM); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); + auto node = GetReadyNode(); + node->GetMetadata()->SetName("10.10.10.01"); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + + auto pod1 = GetReadyPod(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPhase("Pending"); + std::map annotations = {{PENDING_THRESHOLD, "15"}}; + pod1->GetMetadata()->SetAnnotations(annotations); + + // event comes first time, pending duration does not exceed threshold, no alarm + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + auto receivedTime = std::chrono::system_clock::now() - std::chrono::seconds(10); + actor_->member_->pendingPods["function-agent-123"] = receivedTime; + actor_->business_->CheckPodStatus(); + auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + + // event comes second time, pending duration does not exceed threshold, no alarm + pod1->GetStatus()->SetStartTime(kube_client::utility::Datetime("2023-03-15T12:34:56Z")); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + EXPECT_EQ(actor_->member_->pendingPods["function-agent-123"].time_since_epoch().count(), receivedTime.time_since_epoch().count()); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + + // pending duration does not exceed threshold, send alarm + actor_->member_->pendingPods["function-agent-123"] = receivedTime - std::chrono::seconds(20); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) != alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + + // pod phase turns to Running, no alarm + pod1->GetStatus()->SetPhase("Running"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + EXPECT_TRUE(actor_->member_->pendingPods.find("function-agent-123") == actor_->member_->pendingPods.end()); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + + // pod deletion action will send Modified event(with Pending phase) first, then Delete event + pod1->GetStatus()->SetPhase("Pending"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + EXPECT_TRUE(actor_->member_->pendingPods.find("function-agent-123") != actor_->member_->pendingPods.end()); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); + + actor_->OnPodModified(K8sEventType::EVENT_TYPE_DELETE, pod1); + EXPECT_TRUE(actor_->member_->pendingPods.find("function-agent-123") == actor_->member_->pendingPods.end()); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + metrics::MetricsAdapter::GetInstance().GetAlarmHandler().alarmMap_.clear(); +} + +TEST_F(ScalerTest, PendingPodClear) +{ + auto masterBusiness = std::make_shared(actor_, actor_->member_); + actor_->business_ = masterBusiness; + metrics::MetricsAdapter::GetInstance().enabledInstruments_.insert(metrics::YRInstrument::YR_POD_ALARM); + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + actor_->SetNodeMap({}); + auto node = GetReadyNode(); + node->GetMetadata()->SetName("10.10.10.01"); + node->GetStatus()->GetAddresses()[0]->SetAddress("10.10.10.01"); + + std::unordered_map> nodeMap; + nodeMap["10.10.10.01"] = node; + actor_->SetNodeMap(nodeMap); + + // pod exists in pendingPods, but its info does not exist in podNameMap + auto pod1 = GetReadyPod(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetPhase("Pending"); + pod1->GetStatus()->SetStartTime(kube_client::utility::Datetime("2023-03-15T12:34:56Z")); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + actor_->member_->podNameMap.clear(); + actor_->business_->CheckPodStatus(); + auto alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); + + // pod exists in pendingPods, but its status is not pending anymore + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + actor_->member_->podNameMap.find(pod1->GetMetadata()->GetName())->second->GetStatus()->SetPhase("Succeeded"); + actor_->business_->CheckPodStatus(); + alarmMap = metrics::MetricsAdapter::GetInstance().GetAlarmHandler().GetAlarmMap(); + EXPECT_TRUE(alarmMap.find(metrics::POD_ALARM) == alarmMap.end()); + EXPECT_TRUE(actor_->member_->pendingPods.empty()); +} + +TEST_F(ScalerTest, CreateAgentByPoolIDTest) +{ + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + litebus::Async(actor_->GetAID(), &ScalerActor::SetTemplateDeployment, localVarResult); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + auto counter = std::make_shared>(0); + poolManager->RegisterScaleUpHandler([cnt(counter)](const std::string &poolID, bool isReserved) { + (*cnt)++; + }); + + auto createAgentRequest = std::make_shared(); + ::resources::InstanceInfo info; + info.set_requestid("test-request-01"); + info.set_instanceid("test-instance-01"); + info.mutable_createoptions()->operator[]("AFFINITY_POOL_ID") = "pool1"; + info.mutable_resources()->CopyFrom(GetResources()); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + { + // poolID is not found + info.set_requestid("test-request-01"); + info.set_instanceid("test-instance-01"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-01"; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().code() == StatusCode::GS_START_CREATE_POD_FAILED; }); + EXPECT_EQ(GetTestResponse().message(), "failed to scale up pod"); + } + auto podPool = poolManager->GetOrNewPool("pool1"); + podPool->size = 1; + podPool->maxSize = 2; + podPool->readyCount = 0; + podPool->scalable = true; + podPool->status = static_cast(PoolState::RUNNING); + podPool->idleRecycleTime.scaled = 10; + podPool->idleRecycleTime.reserved = 20; + auto deployment1 = std::make_shared(); + deployment1->FromJson(localVarJson); + deployment1->GetMetadata()->SetName("function-agent-pool1"); + poolManager->PutDeployment(deployment1); + { + litebus::Promise> promise1; + litebus::Promise> promise2; + // success scale pod + EXPECT_CALL(*mockClient, CreateNamespacedPod) + .WillOnce(testing::Invoke([promise1](const std::string &rNamespace, const std::shared_ptr &body) { + promise1.SetValue(body); + return body; + })) + .WillOnce(testing::Invoke([promise2](const std::string &rNamespace, const std::shared_ptr &body) { + promise2.SetValue(body); + return body; + })); + info.set_requestid("test-request-02"); + info.set_instanceid("test-instance-02"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_READY(promise1.GetFuture()); + auto pod1 = promise1.GetFuture().Get(); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetContainerStatuses({}); + pod1->GetStatus()->SetPhase("Running"); + EXPECT_EQ(pod1->GetMetadata()->GetLabels()["yr-idle-to-recycle"], "20"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-02"; }); + EXPECT_EQ(GetTestResponse().code(), 0); + EXPECT_EQ(poolManager->GetPodPool("pool1")->readyCount, 1); + EXPECT_EQ(*counter, 2); + info.set_requestid("test-request-03"); + info.set_instanceid("test-instance-03"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_READY(promise2.GetFuture()); + auto pod2 = promise2.GetFuture().Get(); + EXPECT_EQ(pod2->GetMetadata()->GetLabels()["yr-idle-to-recycle"], "10"); + pod2->SetStatus(std::make_shared()); + pod2->GetStatus()->SetContainerStatuses({}); + pod2->GetStatus()->SetPhase("Running"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod2); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-03"; }); + EXPECT_EQ(GetTestResponse().code(), 0); + EXPECT_EQ(poolManager->GetPodPool("pool1")->readyCount, 2); + EXPECT_EQ(*counter, 4); + info.set_requestid("test-request-04"); + info.set_instanceid("test-instance-04"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-04"; }); + EXPECT_EQ(GetTestResponse().code(), StatusCode::GS_START_CREATE_POD_FAILED); + EXPECT_EQ(poolManager->GetPodPool("pool1")->readyCount, 2); + EXPECT_EQ(*counter, 4); + + // delete pod + EXPECT_CALL(*mockClient, DeleteNamespacedPod).WillOnce(testing::Return(pod2)); + actor_->DoDeletePod(pod2->GetMetadata()->GetName()); + ASSERT_AWAIT_TRUE([&]() -> bool { return *counter == 5; }); + EXPECT_TRUE(poolManager->GetPodPool("pool1")->deletingPodSet.find(pod2->GetMetadata()->GetName()) != poolManager->GetPodPool("pool1")->deletingPodSet.end()); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_DELETE, pod2); + ASSERT_AWAIT_TRUE([&]() -> bool { return *counter == 6; }); + EXPECT_TRUE(poolManager->GetPodPool("pool1")->deletingPodSet.find(pod2->GetMetadata()->GetName()) == poolManager->GetPodPool("pool1")->deletingPodSet.end()); + } + { + litebus::Promise> promise; + promise.SetFailed(100); + EXPECT_CALL(*mockClient, CreateNamespacedPod).WillRepeatedly(testing::Return(promise.GetFuture())); + // scale up fail to create pod + podPool->maxSize = 3; + info.set_requestid("test-request-05"); + info.set_instanceid("test-instance-05"); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + createAgentRequest->mutable_instanceinfo()->CopyFrom(info); + litebus::Async(testActor_->GetAID(), &ScalerTestActor::CreateAgent, actor_->GetAID(), + createAgentRequest->SerializeAsString()); + ASSERT_AWAIT_TRUE([&]() -> bool { return GetTestResponse().requestid() == "test-request-05"; }); + EXPECT_EQ(GetTestResponse().code(), StatusCode::GS_START_CREATE_POD_FAILED); + EXPECT_EQ(poolManager->GetPodPool("pool1")->readyCount, 1); + EXPECT_EQ(*counter, 7); + EXPECT_EQ(poolManager->GetPodPool("pool1")->pendingCreatePodSet.size(), 0); + } +} + +TEST_F(ScalerTest, LoadFunctionAgentTemplate) +{ + (void)litebus::os::Rmdir("/home/sn/scaler/template/"); + actor_->SetTemplateDeployment(nullptr); + auto status = litebus::Async(actor_->GetAID(), &ScalerActor::SyncTemplateDeployment); + EXPECT_EQ(status.Get(), StatusCode::FILE_NOT_FOUND); + + (void)litebus::os::Mkdir("/home/sn/scaler/template/"); + auto writeStatus = Write("/home/sn/scaler/template/function-agent.json", "fake json"); + status = litebus::Async(actor_->GetAID(), &ScalerActor::SyncTemplateDeployment); + ASSERT_AWAIT_READY(status); + EXPECT_EQ(status.Get(), StatusCode::JSON_PARSE_ERROR); + + writeStatus = Write("/home/sn/scaler/template/function-agent.json", DEPLOYMENT_JSON); + YRLOG_INFO("write json file result: {}", writeStatus); + status = litebus::Async(actor_->GetAID(), &ScalerActor::SyncTemplateDeployment); + ASSERT_AWAIT_READY(status); + EXPECT_EQ(status.Get(), StatusCode::SUCCESS); + (void)litebus::os::Rmdir("/home/sn/scaler/"); +} + +TEST_F(ScalerTest, SyncPodsTest) { + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + std::shared_ptr poolManager = std::make_shared(metaStorageAccessor_); + actor_->SetPoolManager(poolManager); + + litebus::Future> failedFuture; + failedFuture.SetFailed(StatusCode::FAILED); + EXPECT_CALL(*mockClient, ListNamespacedPod).WillOnce(testing::Return(failedFuture)); + auto future = actor_->SyncPodAndInstance(); + future.Get(); + EXPECT_TRUE(future.IsError()); + + std::shared_ptr podList = std::make_shared(); + std::vector> podItems; + podItems.emplace_back(nullptr); + auto pod1 = GetReadyPod(); + pod1->GetMetadata()->SetName("error-pod"); + podItems.emplace_back(pod1); + auto pod2 = GetReadyPod(); + pod2->UnsetStatus(); + pod2->GetMetadata()->SetUid("unready-failed"); + pod2->GetMetadata()->SetName("function-agent-unready-failed"); + podItems.emplace_back(pod2); + podItems.emplace_back(GetReadyPod()); + podList->SetItems(podItems); + EXPECT_CALL(*mockClient, ListNamespacedPod).WillRepeatedly(testing::DoAll(testing::Return(podList))); + future = actor_->SyncPodAndInstance(); + EXPECT_TRUE(future.Get().IsOk()); + EXPECT_TRUE(actor_->member_->podNameMap.size() == 2); + EXPECT_TRUE(actor_->waitForReadyPods_.find("unready-failed") != actor_->waitForReadyPods_.end()); + bool isDeleteFinished = false; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillRepeatedly(testing::DoAll(testing::Assign(&isDeleteFinished, true), testing::Return(GetReadyPod()))); + actor_->waitForReadyPods_["unready-failed"].SetFailed(StatusCode::FAILED); + ASSERT_AWAIT_TRUE([&]() -> bool { return isDeleteFinished; }); +} + +TEST_F(ScalerTest, SystemFunctionPodManager) +{ + // mock k8s client + auto mockClient = std::make_shared(); + actor_->SetKubeClient(mockClient); + actor_->SetPodMap({}); + auto scaleUpCounter = std::make_shared>(0); + actor_->member_->frontendManager->RegisterScaleUpHandler( + [cnt(scaleUpCounter)](const std::string &podName, uint32_t instanceNumber) { cnt->fetch_add(instanceNumber); }); + + auto pod1 = GetReadyPod(); + pod1->GetMetadata()->GetLabels()["yr-reuse"] = "true"; + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod1); + actor_->member_->frontendManager->CheckSystemFunctionNeedScale(); + + auto pod2 = GetReadyPod(); + pod2->GetMetadata()->SetName("function-agent-a-500m-2048mi-faasfrontend-1"); + pod2->GetSpec()->SetNodeName("node-0002"); + actor_->OnPodModified(K8sEventType::EVENT_TYPE_MODIFIED, pod2); + // Expect: delete pod is called + litebus::Future deletePodCalledArg; + EXPECT_CALL(*mockClient, DeleteNamespacedPod) + .WillOnce(testing::DoAll(test::FutureArg<0>(&deletePodCalledArg), testing::Return(GetReadyPod()))); + actor_->member_->frontendManager->CheckSystemFunctionNeedScale(); + ASSERT_AWAIT_READY(deletePodCalledArg); +} +} // namespace functionsystem::scaler::test diff --git a/functionsystem/tests/unit/function_master/scaler/scaler_test_actor.h b/functionsystem/tests/unit/function_master/scaler/scaler_test_actor.h new file mode 100644 index 0000000000000000000000000000000000000000..2c7501ed34595fa3577d3a940258071b1982a1f5 --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/scaler_test_actor.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FUNCTIONSYSTEM_TEST_UNIT_SCALER_TEST_ACTOR_H +#define FUNCTIONSYSTEM_TEST_UNIT_SCALER_TEST_ACTOR_H + +#include +#include + +#include "actor/actor.hpp" +#include "async/async.hpp" +#include "common/proto/pb/message_pb.h" +#include "common/status/status.h" +#include "function_master/scaler/scaler_actor.h" + +namespace functionsystem::test { + +class ScalerTestActor : public litebus::ActorBase { +public: + explicit ScalerTestActor(const std::string &name) : ActorBase(name) + { + response_ = std::make_shared(); + } + ~ScalerTestActor() override = default; + + void Init() override + { + Receive("CreateAgentResponse", &ScalerTestActor::CreateAgentResponse); + Receive("UpdateNodeTaintsResponse", &ScalerTestActor::UpdateTaintsResponse); + Receive("UpdateLocalStatus", &ScalerTestActor::UpdateLocalStatus); + Receive("DeletePodResponse", &ScalerTestActor::DeletePodResponse); + } + + void CreateAgent(const litebus::AID &to, std::string msg) + { + Send(to, "CreateAgent", std::move(msg)); + } + + void UpdateNodeTaints(const litebus::AID &to, std::string msg) + { + Send(to, "UpdateNodeTaints", std::move(msg)); + } + + void DeletePodRequest(const litebus::AID &to, std::string msg) + { + Send(to, "DeletePod", std::move(msg)); + } + + void CreateAgentResponse(const litebus::AID &from, std::string &&name, std::string &&msg) + { + auto createAgentResponse = std::make_shared(); + if (!createAgentResponse->ParseFromString(msg)) { + YRLOG_ERROR("failed to parse response for CreateAgent."); + return; + } + response_ = createAgentResponse; + } + + void UpdateTaintsResponse(const litebus::AID &from, std::string &&name, std::string &&msg) + { + auto updateTaintResponse = std::make_shared(); + if (!updateTaintResponse->ParseFromString(msg)) { + YRLOG_ERROR("failed to parse response for UpdateTaints"); + return; + } + taintResponse_ = updateTaintResponse; + } + + void UpdateLocalStatus(const litebus::AID &from, std::string &&name, std::string &&msg) + { + auto localStatus = std::make_shared(); + if (!localStatus->ParseFromString(msg)) { + YRLOG_ERROR("failed to parse response for updateLocalStatus"); + return; + } + localStatus_ = localStatus; + } + + messages::CreateAgentResponse GetResponse() + { + if (response_) { + return *response_; + } + return {}; + } + + messages::UpdateNodeTaintResponse GetTaintResponse() + { + if (taintResponse_) { + return *taintResponse_; + } + return {}; + } + + messages::UpdateLocalStatusRequest GetLocalStatusRequest() + { + if (localStatus_) { + return *localStatus_; + } + return {}; + } + + void DeletePodResponse(const litebus::AID &from, std::string &&name, std::string &&msg) + { + auto deletePodResponse = std::make_shared(); + if (!deletePodResponse->ParseFromString(msg)) { + YRLOG_ERROR("failed to parse request for DeletePod."); + return; + } + MockDeletePodResponse(deletePodResponse); + } + MOCK_METHOD1(MockDeletePodResponse, void(const std::shared_ptr)); + +private: + std::shared_ptr response_; + std::shared_ptr taintResponse_; + std::shared_ptr localStatus_; +}; + +} // namespace functionsystem::test + +#endif // FUNCTIONSYSTEM_TEST_UNIT_SCALER_TEST_ACTOR_H diff --git a/functionsystem/tests/unit/function_master/scaler/system_function_pod_manager_test.cpp b/functionsystem/tests/unit/function_master/scaler/system_function_pod_manager_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b8d97ebe628f74aac54c92d85d29e4f0188f8462 --- /dev/null +++ b/functionsystem/tests/unit/function_master/scaler/system_function_pod_manager_test.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include "common/kube_client/kube_client.h" +#include + +#define private public +#include "function_master/scaler/system_function_pod_manager/system_function_pod_manager.h" + +namespace functionsystem::scaler::test { + +class SystemFunctionPodManagerTest : public ::testing::Test { +protected: + void SetUp() override + { + std::unordered_map labelSelector = {{ "reuse", "true"}}; + frontendManager_ = std::make_shared("faasfrontend", labelSelector); + } + + void TearDown() override + { + } + + std::shared_ptr frontendManager_; +}; + +std::shared_ptr GenPodForManager(const std::string &podName) +{ + auto pod1 = std::make_shared(); + pod1->SetMetadata(std::make_shared()); + pod1->GetMetadata()->SetName(podName); + pod1->GetMetadata()->SetLabels({ {"reuse", "true"}}); + pod1->GetMetadata()->SetAnnotations({}); + pod1->SetStatus(std::make_shared()); + pod1->GetStatus()->SetContainerStatuses({ std::make_shared() }); + pod1->GetStatus()->GetContainerStatuses().front()->SetContainerID("runtime-manager"); + pod1->GetStatus()->SetPhase("Running"); + pod1->SetSpec(std::make_shared()); + pod1->GetSpec()->SetNodeName("node001"); + return pod1; +} + +TEST_F(SystemFunctionPodManagerTest, PodEvent) +{ + { + // normal pod + auto pod = GenPodForManager("function-agent-pod1"); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 1); + frontendManager_->OnPodDelete(pod); + auto pod1 = GenPodForManager("function-agent-pod2"); + pod1->GetSpec()->SetNodeName("node002"); + frontendManager_->OnPodDelete(pod1); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + } + { + // frontend pod + auto pod = GenPodForManager("function-agent-a-500m-2048mi-faasfrontend-1"); + pod->GetMetadata()->SetLabels({}); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2SystemFuncPodNames_.size(), 1); + frontendManager_->OnPodDelete(pod); + auto pod1 = GenPodForManager("function-agent-a-500m-2048mi-faasfrontend-2"); + pod1->GetSpec()->SetNodeName("node002"); + frontendManager_->OnPodDelete(pod1); + EXPECT_EQ(frontendManager_->nodeID2SystemFuncPodNames_.size(), 0); + } + { + auto pod = GenPodForManager("function-agent-pod1"); + // pod not ready + pod->UnsetStatus(); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + // pod is not match + pod = GenPodForManager("function-agent-pod1"); + pod->GetMetadata()->SetLabels({ {"reuse", "false"}}); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + pod->GetMetadata()->SetLabels({}); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + } + { + // pod terminating event + auto pod = GenPodForManager("function-agent-pod1"); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 1); + pod->GetStatus()->GetContainerStatuses().front()->SetState(std::make_shared()); + pod->GetStatus()->GetContainerStatuses().front()->GetState()->SetTerminated(std::make_shared()); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + } + { + auto pod = GenPodForManager("function-agent-pod1"); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 1); + pod->GetStatus()->SetPhase("Failed"); + pod->GetStatus()->SetContainerStatuses({}); + pod->GetStatus()->UnsetContainerStatuses(); + frontendManager_->OnPodUpdate(pod); + EXPECT_EQ(frontendManager_->nodeID2PodNames_.size(), 0); + } +} + +TEST_F(SystemFunctionPodManagerTest, CheckSystemFunctionNeedScale) +{ + frontendManager_->CheckSystemFunctionNeedScale(); + auto scaleDownCounter = std::make_shared>(0); + auto scaleUpCounter = std::make_shared>(0); + frontendManager_->RegisterScaleUpHandler([cnt(scaleUpCounter)](const std::string &podName, uint32_t instanceNumber){ + cnt->fetch_add(instanceNumber); + }); + frontendManager_->RegisterScaleDownHandler([cnt(scaleDownCounter)](const std::string &podName){ + (*cnt)++; + }); + frontendManager_->CheckSystemFunctionNeedScale(); + EXPECT_EQ(*scaleUpCounter, 0); + frontendManager_->nodeID2PodNames_["node001"] = {"function-agent-pool1"}; + frontendManager_->nodeID2SystemFuncPodNames_["node001"] = {"function-agent-a-500m-2048mi-faasfrontend-1"}; + frontendManager_->nodeID2SystemFuncPodNames_["node002"] = {"function-agent-a-500m-2048mi-faasfrontend-2"}; + frontendManager_->CheckSystemFunctionNeedScale(); + EXPECT_EQ(*scaleDownCounter, 1); + frontendManager_->nodeID2PodNames_.clear(); + frontendManager_->nodeID2SystemFuncPodNames_.clear(); + auto pod1 = GenPodForManager("function-agent-pod1"); + auto pod2 = GenPodForManager("function-agent-pod2"); + frontendManager_->OnPodUpdate(pod1); + pod2->GetSpec()->SetNodeName("node002"); + frontendManager_->OnPodUpdate(pod2); + frontendManager_->CheckSystemFunctionNeedScale(); + EXPECT_EQ(*scaleUpCounter, 3); +} +} \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/system_function_loader/CMakeLists.txt b/functionsystem/tests/unit/function_master/system_function_loader/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..95c7a6f33adc9e3e926408037cc8d40cd949e435 --- /dev/null +++ b/functionsystem/tests/unit/function_master/system_function_loader/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} SYSTEM_FUNCTION_LOADER_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${SYSTEM_FUNCTION_LOADER_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_actor_test.cpp b/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_actor_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fb203477cfd04fc0c962821bf22bcdfb48b26e28 --- /dev/null +++ b/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_actor_test.cpp @@ -0,0 +1,1417 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "function_master/system_function_loader/bootstrap_actor.h" + +#include +#include +#include + + +#include "common/etcd_service/etcd_service_driver.h" +#include "common/utils/meta_store_kv_operation.h" +#include "httpd/http_actor.hpp" +#include "mocks/mock_instance_manager.h" +#include "utils/future_test_helper.h" +#include "utils/generate_info.h" +#include "utils/os_utils.hpp" +#include "utils/port_helper.h" + +namespace functionsystem::system_function_loader::test { +using namespace functionsystem::test; + +const std::string SYSTEM_FUNC_CONFIG_PATH = "/home/sn/function/config"; +const std::string SYSTEM_FUNC_CONFIG_FILE = "system-function-config.json"; +const std::string SYSTEM_FUNC_PAYLOAD_PATH = "/home/sn/function/payload"; +const std::string SYSTEM_FUNC_META_PATH = "/home/sn/function/system-function-meta"; + +const std::unordered_map payloadContent = { + { "faas-scheduler-config.json", + R"({"0-system-faasscheduler":{"systemFunctionName":"0-system-faascontroller","signal":65,"payload":{"version":"v1"}}})" }, + { "faas-frontend-config.json", + R"({"0-system-faasfrontend":{"systemFunctionName":"0-system-faascontroller","signal":66,"payload":{"version":"v1","instanceNum":2}}})" }, +}; + +const std::string controllerV0 = + R"({"funcMetaData":{"layers":[],"name":"faascontroller","description":"","version":"v0","functionUrn":"sn:cn:yrk:0:function:faascontroller","functionVersionUrn":"sn:cn:yrk:0:function:faascontroller:v0","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"local","code_path":"/home/sn/function"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}, "extendedMetaData":{"instance_meta_data":{"maxInstance":100, "minInstance":0, "concurrentNum":10, "cacheInstance":0}}})"; +const std::string controllerV1 = + R"({"funcMetaData":{"layers":[],"name":"faascontroller","description":"","version":"v1","functionUrn":"sn:cn:yrk:0:function:faascontroller","functionVersionUrn":"sn:cn:yrk:0:function:faascontroller:v1","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"local","code_path":"/home/sn/function"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}, "extendedMetaData":{"instance_meta_data":{"maxInstance":100, "minInstance":0, "concurrentNum":10, "cacheInstance":0}}})"; +const std::string schedulerV1 = + R"({"funcMetaData":{"layers":[],"name":"faasscheduler","description":"","version":"v1","functionUrn":"sn:cn:yrk:0:function:faasscheduler","functionVersionUrn":"sn:cn:yrk:0:function:faasscheduler:v1","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"local","code_path":"/home/sn/function"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}, "extendedMetaData":{"instance_meta_data":{"maxInstance":100, "minInstance":0, "concurrentNum":10, "cacheInstance":0}}})"; +const std::string frontendV1 = + R"({"funcMetaData":{"layers":[],"name":"faasfrontend","description":"","version":"v1","functionUrn":"sn:cn:yrk:0:function:faasfrontend","functionVersionUrn":"sn:cn:yrk:0:function:faasfrontend:v1","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"local","code_path":"/home/sn/function"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}, "extendedMetaData":{"instance_meta_data":{"maxInstance":100, "minInstance":0, "concurrentNum":10, "cacheInstance":0}}})"; +const std::string controllerV2 = + R"({"funcMetaData":{"layers":[],"name":"faascontroller","description":"","version":"v2","functionUrn":"sn:cn:yrk:0:function:faascontroller","functionVersionUrn":"sn:cn:yrk:0:function:faascontroller:v2","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"local","code_path":"/home/sn/function"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}, "extendedMetaData":{"instance_meta_data":{"maxInstance":100, "minInstance":0, "concurrentNum":10, "cacheInstance":0}}})"; + +const std::unordered_map metaContent = { + { "faas-controller-meta.json", controllerV1 }, + { "faas-frontend-meta.json", frontendV1 }, + { "faas-scheduler-meta.json", schedulerV1 }, +}; + +void GenSystemFunctionConfigFile(const std::string &content) +{ + if (!litebus::os::ExistPath(SYSTEM_FUNC_CONFIG_PATH)) { + litebus::os::Mkdir(SYSTEM_FUNC_CONFIG_PATH); + } + auto filePath = SYSTEM_FUNC_CONFIG_PATH + "/" + SYSTEM_FUNC_CONFIG_FILE; + + std::ofstream outfile; + outfile.open(filePath.c_str()); + outfile << content << std::endl; + outfile.close(); +} + +void GenSystemFunctionPayloadFiles(const std::unordered_map &content) +{ + if (!litebus::os::ExistPath(SYSTEM_FUNC_PAYLOAD_PATH)) { + litebus::os::Mkdir(SYSTEM_FUNC_PAYLOAD_PATH); + } + for (auto iter = content.begin(); iter != content.end(); ++iter) { + auto filePath = SYSTEM_FUNC_PAYLOAD_PATH + "/" + iter->first; + std::ofstream outfile; + outfile.open(filePath.c_str()); + outfile << iter->second << std::endl; + outfile.close(); + } +} + +void GenSystemFunctionMetaFiles(const std::unordered_map &content) +{ + if (!litebus::os::ExistPath(SYSTEM_FUNC_META_PATH)) { + litebus::os::Mkdir(SYSTEM_FUNC_META_PATH); + } + for (auto iter = content.begin(); iter != content.end(); ++iter) { + auto filePath = SYSTEM_FUNC_META_PATH + "/" + iter->first; + std::ofstream outfile; + outfile.open(filePath.c_str()); + outfile << iter->second << std::endl; + outfile.close(); + } +} + +class GlobalScheduler : public functionsystem::global_scheduler::GlobalSched { +public: + MOCK_METHOD(litebus::Future, Schedule, (const std::shared_ptr &), (override)); + MOCK_METHOD(litebus::Future>, GetLocalAddress, (const std::string &), (override)); +}; + +class MockLocalActor : public litebus::ActorBase { +public: + MockLocalActor() : litebus::ActorBase("mock-LocalSchedInstanceCtrlActor") + { + } + MOCK_METHOD(void, ForwardCustomSignalRequest, (const litebus::AID &, std::string &&, std::string &&)); + + void SendForwardCustomSignalResponse(const litebus::AID &to, const std::string &requestID) + { + messages::ForwardKillResponse rsp; + rsp.set_requestid(requestID); + rsp.set_code(StatusCode::SUCCESS); + (void)Send(to, "ForwardCustomSignalResponse", rsp.SerializeAsString()); + } + +protected: + void Init() override + { + Receive("ForwardCustomSignalRequest", &MockLocalActor::ForwardCustomSignalRequest); + } +}; + +class BootstrapWrapper : public BootstrapActor { +public: + BootstrapWrapper(const std::shared_ptr &metaClient, + std::shared_ptr globalSched, uint32_t sysFuncRetryPeriod, + bool enableFrontendPool = false) + : BootstrapActor(metaClient, globalSched, sysFuncRetryPeriod, "instanceManagerAddress", enableFrontendPool) + { + } + + litebus::Future LoadBootstrapConfigWrapper(const std::string &customArgs) + { + LoadBootstrapConfig(customArgs); + return true; + } + + litebus::Future UpdateConfigWrapper() + { + UpdateConfigHandler(); + return true; + } + + litebus::Future UpdatePayloadWrapper() + { + UpdatePayloadHandler(); + return true; + } + + litebus::Future UpdateMetaWrapper() + { + UpdateMetaHandler(); + return true; + } +}; + +class BootstrapActorTest : public ::testing::Test { +protected: + std::shared_ptr mockInstanceMgr_ = nullptr; + std::shared_ptr bootstrapActor_ = nullptr; + std::shared_ptr globalSched_ = nullptr; + inline static std::unique_ptr etcdSrvDriver_; + inline static std::shared_ptr metaStoreAccessor_; + inline static std::string metaStoreServerHost_; + inline static std::string localAddress_; + + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + localAddress_ = "127.0.0.1:" + std::to_string(port); + + auto client = std::make_unique( + MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }); + client->Init(); + metaStoreAccessor_ = std::make_shared(std::move(client)); + litebus::os::Rm("/home/sn/function/"); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + metaStoreAccessor_ = nullptr; + etcdSrvDriver_->StopServer(); + etcdSrvDriver_ = nullptr; + } + + void SetUp() override + { + globalSched_ = std::make_shared(); + auto metaClient = + MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost_ }); + bootstrapActor_ = std::make_shared(metaClient, globalSched_, 900, true); + mockInstanceMgr_ = std::make_shared(); + litebus::Spawn(bootstrapActor_); + + litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::BindInstanceManager, mockInstanceMgr_); + litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::UpdateLeaderInfo, + GetLeaderInfo(bootstrapActor_->GetAID())); + + litebus::os::Rm("/home/sn/function/"); + } + + void TearDown() override + { + auto result = metaStoreAccessor_->Delete("/faas/system-function/config", true); + EXPECT_AWAIT_READY(result); + result = metaStoreAccessor_->Delete(INSTANCE_PATH_PREFIX, true); + EXPECT_AWAIT_READY(result); + result = metaStoreAccessor_->Delete(FUNC_META_PATH_PREFIX, true); + EXPECT_AWAIT_READY(result); + + litebus::os::Rm("/home/sn/function/"); + + litebus::Terminate(bootstrapActor_->GetAID()); + litebus::Await(bootstrapActor_->GetAID()); + bootstrapActor_ = nullptr; + globalSched_ = nullptr; + mockInstanceMgr_ = nullptr; + } + + void LoadBootstrapConfig(const std::string &customArgs) + { + auto status = + litebus::Async(bootstrapActor_->GetAID(), &BootstrapWrapper::LoadBootstrapConfigWrapper, customArgs); + EXPECT_AWAIT_READY(status); + } + + bool CheckInstanceExist(const FuncInstanceParams &funcInstanceParams) + { + auto status = + litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::CheckInstanceExist, funcInstanceParams); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status LoadFunctionConfigs() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadFunctionConfigs); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status LoadSysFuncCustomArgs(const std::string &args) + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadSysFuncCustomArgs, args); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + int GetFunctionConfigSize() + { + auto size = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetFunctionConfigSize); + EXPECT_AWAIT_READY(size); + return size.Get(); + } + + litebus::Option GetFunctionConfig(const std::string &funcName) + { + auto config = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetFunctionConfig, funcName); + EXPECT_AWAIT_READY(config); + return config.Get(); + } + + litebus::Option GetSysFuncCustomArgs(const std::string &funcName) + { + auto json = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetSysFuncCustomArgs, funcName); + EXPECT_AWAIT_READY(json); + return json.Get(); + } + + Status LoadCurrentFunctionConfigs() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadCurrentFunctionConfigs); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status LoadSysFuncPayloads() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadSysFuncPayloads); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status LoadSysFuncMetas() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadSysFuncMetas); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status LoadCurrentSysFuncMetas() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::LoadCurrentSysFuncMetas); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + litebus::Option GetCurrFunctionConfig(const std::string &funcName) + { + auto json = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetCurrFunctionConfig, funcName); + EXPECT_AWAIT_READY(json); + return json.Get(); + } + + litebus::Option GetFunctionPayload(const std::string &funcName) + { + auto payload = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetFunctionPayload, funcName); + EXPECT_AWAIT_READY(payload); + return payload.Get(); + } + + litebus::Option>>> + GetFunctionMetaQueue() + { + auto queue = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetFunctionMetaQueue); + EXPECT_AWAIT_READY(queue); + return queue.Get(); + } + void UpdateConfigHandler() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapWrapper::UpdateConfigWrapper); + EXPECT_AWAIT_READY(status); + } + + void UpdatePayloadHandler() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapWrapper::UpdatePayloadWrapper); + EXPECT_AWAIT_READY(status); + } + + void UpdateMetaHandler() + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapWrapper::UpdateMetaWrapper); + EXPECT_AWAIT_READY(status); + } + + Status SendUpdateArgsSignal(const std::string &functionName, const FunctionPayload &functionPayload, int retryTimes) + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::SendUpdateArgsSignal, functionName, + functionPayload, retryTimes); + EXPECT_AWAIT_READY(status); + return status.Get(); + } + + Status SendUpgradeFunctionSignal(const std::string &functionName, const FunctionConfig &newConfig, + const FunctionConfig &currConfig, int retryTimes) + { + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::SendUpgradeFunctionSignal, + functionName, newConfig, currConfig, retryTimes); + EXPECT_AWAIT_READY(status); + return status.Get(); + } +}; + +/** + * Feature: BootstrapActorTest LoadBootstrapConfigInvalidTest + * Description: Try to load invalid config, failed + * Steps: + * 1. File not exist + * 2. Invalid json + * 3. Empty function name + * + * Expectation: + * 1-3. Load failed + */ +TEST_F(BootstrapActorTest, LoadBootstrapConfigInvalidTest) +{ + (void)litebus::os::Rmdir(SYSTEM_FUNC_CONFIG_PATH); + // file not exist + auto status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("bootstrap config file not exist")); + EXPECT_EQ(GetFunctionConfigSize(), 0); + + // invalid json + std::string content = "fake json"; + GenSystemFunctionConfigFile(content); + status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("parse json failed")); + EXPECT_EQ(GetFunctionConfigSize(), 0); + + // empty function name + content = + R"({"":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}},"":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + GenSystemFunctionConfigFile(content); + status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + EXPECT_EQ(GetFunctionConfigSize(), 0); +} + +TEST_F(BootstrapActorTest, LoadBootstrapConfigTest) +{ + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx2":{"xxx":1000}}},"0-system-faasfrontend":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + GenSystemFunctionConfigFile(content); + + auto status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + + auto controllerConfig = GetFunctionConfig("0-system-faascontroller"); + EXPECT_TRUE(controllerConfig.IsSome()); + EXPECT_EQ(controllerConfig.Get().tenantID, "0"); + EXPECT_EQ(controllerConfig.Get().version, "$latest"); + EXPECT_EQ(controllerConfig.Get().cpu, 500); + EXPECT_EQ(controllerConfig.Get().memory, 500); + EXPECT_EQ(controllerConfig.Get().createOptions.size(), 1u); + EXPECT_EQ(controllerConfig.Get().createOptions.find("concurrentNum")->second, "10"); + EXPECT_EQ(controllerConfig.Get().instanceNum, 1u); + EXPECT_EQ(controllerConfig.Get().extension.at("schedule_policy"), "monopoly"); + EXPECT_EQ(controllerConfig.Get().args.dump(), "{\"xxx\":2,\"xxx2\":{\"xxx\":1000}}"); + + auto frontConfig = GetFunctionConfig("0-system-faasfrontend"); + EXPECT_TRUE(frontConfig.IsSome()); + EXPECT_EQ(frontConfig.Get().tenantID, "0"); + EXPECT_EQ(frontConfig.Get().version, "$latest"); + EXPECT_EQ(frontConfig.Get().cpu, 100); + EXPECT_EQ(frontConfig.Get().memory, 100); + EXPECT_EQ(controllerConfig.Get().createOptions.size(), 1u); + EXPECT_EQ(frontConfig.Get().createOptions.find("concurrentNum")->second, "1"); + EXPECT_EQ(frontConfig.Get().instanceNum, 2u); + EXPECT_EQ(frontConfig.Get().extension.at("schedule_policy"), "shared"); + EXPECT_EQ(frontConfig.Get().args.dump(), "{\"xxx\":4,\"xxx2\":{\"xxx\":2000}}"); +} + +/** + * Feature: BootstrapActorTest LoadBootstrapConfigTest + * Description: Load bootstrap config + * Steps: + * 1. Write config file + * 2. LoadFunctionConfigs + * + * Expectation: + * 2. config load success + */ +TEST_F(BootstrapActorTest, LoadBootstrapConfigTestCompatible) +{ + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension":{"schedule_policy":"monopoly"}},"args":{"xxx":2,"xxx2":{"xxx":1000}}},"0-system-faasfrontend":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension":{"schedule_policy":"shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + GenSystemFunctionConfigFile(content); + + auto status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + + auto controllerConfig = GetFunctionConfig("0-system-faascontroller"); + EXPECT_TRUE(controllerConfig.IsSome()); + EXPECT_EQ(controllerConfig.Get().tenantID, "0"); + EXPECT_EQ(controllerConfig.Get().version, "$latest"); + EXPECT_EQ(controllerConfig.Get().cpu, 500); + EXPECT_EQ(controllerConfig.Get().memory, 500); + EXPECT_EQ(controllerConfig.Get().createOptions.size(), 1u); + EXPECT_EQ(controllerConfig.Get().createOptions.find("concurrentNum")->second, "10"); + EXPECT_EQ(controllerConfig.Get().instanceNum, 1u); + EXPECT_EQ(controllerConfig.Get().extension.at("schedule_policy"), "monopoly"); + EXPECT_EQ(controllerConfig.Get().args.dump(), "{\"xxx\":2,\"xxx2\":{\"xxx\":1000}}"); + + auto frontConfig = GetFunctionConfig("0-system-faasfrontend"); + EXPECT_TRUE(frontConfig.IsSome()); + EXPECT_EQ(frontConfig.Get().tenantID, "0"); + EXPECT_EQ(frontConfig.Get().version, "$latest"); + EXPECT_EQ(frontConfig.Get().cpu, 100); + EXPECT_EQ(frontConfig.Get().memory, 100); + EXPECT_EQ(controllerConfig.Get().createOptions.size(), 1u); + EXPECT_EQ(frontConfig.Get().createOptions.find("concurrentNum")->second, "1"); + EXPECT_EQ(frontConfig.Get().instanceNum, 2u); + EXPECT_EQ(frontConfig.Get().extension.at("schedule_policy"), "shared"); + EXPECT_EQ(frontConfig.Get().args.dump(), "{\"xxx\":4,\"xxx2\":{\"xxx\":2000}}"); +} + +/** + * Feature: BootstrapActorTest LoadSysFuncCustomArgsTest + * Description: Load system function custom args from flags + * Steps: + * 1. Empty config + * 2. Invalid config json + * 3. Correct json + * 4. Load bootstrap config + * + * Expectation: + * 1. Load failed + * 2. Load failed + * 3. Load success + * 4. Args are combined + */ +TEST_F(BootstrapActorTest, LoadSysFuncCustomArgsTest) +{ + std::string content = ""; + auto status = LoadSysFuncCustomArgs(content); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("sysFuncCustomArgs is empty")); + + content = "fake json"; + status = LoadSysFuncCustomArgs(content); + EXPECT_TRUE(status.IsError()); + EXPECT_THAT(status.ToString(), testing::HasSubstr("parse arg json failed")); + + content = + R"({"0-system-faascontroller":{"memory":500,"cpu":500},"0-system-faasfrontend":{"memory":100,"cpu":100}})"; + status = LoadSysFuncCustomArgs(content); + EXPECT_TRUE(status.IsOk()); + + auto controllerArgs = GetSysFuncCustomArgs("0-system-faascontroller"); + EXPECT_TRUE(controllerArgs.IsSome()); + EXPECT_EQ(controllerArgs.Get().at("cpu"), 500); + EXPECT_EQ(controllerArgs.Get().at("memory"), 500); + + auto frontEndArgs = GetSysFuncCustomArgs("0-system-faasfrontend"); + EXPECT_TRUE(frontEndArgs.IsSome()); + EXPECT_EQ(frontEndArgs.Get().at("cpu"), 100); + EXPECT_EQ(frontEndArgs.Get().at("memory"), 100); + + // combine args + content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx1":2,"xxx2":{"xxx":1000}}},"0-system-faasfrontend":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx1":4,"xxx2":{"xxx":2000}}}})"; + GenSystemFunctionConfigFile(content); + status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + + auto controllerConfig = GetFunctionConfig("0-system-faascontroller"); + EXPECT_TRUE(controllerConfig.IsSome()); + EXPECT_EQ(controllerConfig.Get().args.dump(), "{\"cpu\":500,\"memory\":500,\"xxx1\":2,\"xxx2\":{\"xxx\":1000}}"); + + auto frontEndConfig = GetFunctionConfig("0-system-faasfrontend"); + EXPECT_TRUE(frontEndConfig.IsSome()); + EXPECT_EQ(frontEndConfig.Get().args.dump(), "{\"cpu\":100,\"memory\":100,\"xxx1\":4,\"xxx2\":{\"xxx\":2000}}"); +} + +/** + * Feature: BootstrapActorTest CheckInstanceExistTest + * Description: Load system function custom args from flags + * Steps: + * 1. Put instance into meta_store + * 2. Check existed instance0 + * 3. Check non-existed instance1 + * 4. Check invalid instance2 + * + * Expectation: + * 2. Error already existed + * 3. Not exist + * 4. Error invalid instance key + */ +TEST_F(BootstrapActorTest, CheckInstanceExistTest) +{ + auto funcKey = "0/0-system-faascontroller/$latest"; + auto instanceID0 = "0-system-faascontroller-0"; + auto instanceID1 = "0-system-faascontroller-1"; + + FuncInstanceParams params0 = { + .traceID = "", .instanceID = instanceID0, .requestID = "requestid0", .functionKey = funcKey + }; + FuncInstanceParams params1 = { .traceID = "", .instanceID = instanceID1, .requestID = "", .functionKey = funcKey }; + FuncInstanceParams params2 = { .traceID = "", .instanceID = "", .requestID = "", .functionKey = "" }; + + litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::BindInstanceManager, nullptr); + EXPECT_TRUE(!CheckInstanceExist(params0)); + + litebus::Promise promise0; + promise0.SetValue({ "", nullptr }); + + litebus::Promise promise1; + promise1.SetValue({ "key", nullptr }); + + auto instance1 = std::make_shared(); + instance1->mutable_instancestatus()->set_code(static_cast(InstanceState::FATAL)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance1 }); + + auto instance2 = std::make_shared(); + instance2->mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + litebus::Promise promise3; + promise3.SetValue({ "key", instance2 }); + + auto instance3 = std::make_shared(); + instance3->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise4; + promise4.SetValue({ "key", instance3 }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(promise0.GetFuture())) + .WillOnce(testing::Return(promise1.GetFuture())) + .WillOnce(testing::Return(promise2.GetFuture())) + .WillOnce(testing::Return(promise3.GetFuture())) + .WillOnce(testing::Return(promise4.GetFuture())); + + litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::BindInstanceManager, mockInstanceMgr_); + // empty instance + EXPECT_FALSE(CheckInstanceExist(params0)); + EXPECT_FALSE(CheckInstanceExist(params0)); + + // unrecoverable instance + EXPECT_TRUE(CheckInstanceExist(params2)); + + // waiting instance + EXPECT_TRUE(CheckInstanceExist(params2)); + + // waiting running + EXPECT_TRUE(CheckInstanceExist(params2)); +} + +TEST_F(BootstrapActorTest, CheckInstanceExistReachMaxWaitTimesTest) +{ + auto funcKey = "0/0-system-faascontroller/$latest"; + auto instanceID0 = "0-system-faascontroller-0"; + auto instanceKey = GenInstanceKey(funcKey, instanceID0, "requestid0").Get(); + FuncInstanceParams params0 = { + .traceID = "", .instanceID = instanceID0, .requestID = "requestid0", .functionKey = funcKey + }; + litebus::Future> address; + address.SetValue(litebus::Option(localAddress_)); + auto mockLocal = std::make_shared(); + litebus::Spawn(mockLocal); + EXPECT_CALL(*globalSched_, GetLocalAddress).WillRepeatedly(testing::Return(address)); + EXPECT_CALL(*mockLocal, ForwardCustomSignalRequest) + .WillRepeatedly(testing::Invoke([aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + })); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + litebus::Promise promise; + promise.SetValue({ instanceKey, instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID).WillRepeatedly(testing::Return(promise.GetFuture())); + + for (int i = 0; i < 59; i++) { + EXPECT_TRUE(CheckInstanceExist(params0)); + } + auto waitMap = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetInstanceWaitingStateTimesMap).Get(); + EXPECT_EQ(static_cast(59), waitMap[instanceKey]); + EXPECT_TRUE(CheckInstanceExist(params0)); + waitMap = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::GetInstanceWaitingStateTimesMap).Get(); + EXPECT_EQ(static_cast(0), waitMap[instanceKey]); + + litebus::Terminate(bootstrapActor_->GetAID()); + litebus::Await(bootstrapActor_->GetAID()); + litebus::Terminate(mockLocal->GetAID()); + litebus::Await(mockLocal->GetAID()); +} + +/** + * Feature: BootstrapActorTest BuildScheduleRequestTest + * Description: Build schedule request + * Steps: + * 1. Build schedule request + * + * Expectation: + * 1. Schedule request has correct parma + */ +TEST_F(BootstrapActorTest, BuildScheduleRequestTest) +{ + nlohmann::json customConfig{}; + customConfig["xxx"] = "1"; + customConfig["xxx2"] = 2; + FunctionConfig config{ .tenantID = "tenantID", + .version = "version", + .memory = 101, + .cpu = 102, + .instanceNum = 100, + .args = customConfig }; + config.extension["schedule_policy"] = "monopoly"; + config.createOptions["concurrentNum"] = "10"; + + FuncInstanceParams params{ + .traceID = "traceID", .instanceID = "instanceID", .requestID = "requestID", .functionKey = "functionKey" + }; + + auto request = bootstrapActor_->BuildScheduleRequest(config, params); + + EXPECT_EQ(request->traceid(), params.traceID); + EXPECT_EQ(request->requestid(), params.requestID); + EXPECT_EQ(request->instance().parentid(), ""); + + runtime::CallRequest callRequestGet; + callRequestGet.ParseFromString(request->initrequest()); + EXPECT_EQ(callRequestGet.traceid(), params.traceID); + EXPECT_EQ(callRequestGet.requestid(), params.requestID); + EXPECT_EQ(callRequestGet.function(), params.functionKey); + EXPECT_EQ(callRequestGet.iscreate(), true); + EXPECT_EQ(callRequestGet.createoptions().at("concurrentNum"), config.createOptions["concurrentNum"]); + EXPECT_EQ(callRequestGet.createoptions().at(RESOURCE_OWNER_KEY), SYSTEM_OWNER_VALUE); + EXPECT_EQ(callRequestGet.args().size(), 1); + + auto instance = request->instance(); + EXPECT_EQ(instance.instanceid(), params.instanceID); + EXPECT_EQ(instance.requestid(), params.requestID); + EXPECT_EQ(instance.function(), params.functionKey); + EXPECT_EQ(instance.createoptions().at("concurrentNum"), config.createOptions["concurrentNum"]); + EXPECT_EQ(instance.createoptions().at(RESOURCE_OWNER_KEY), SYSTEM_OWNER_VALUE); + EXPECT_EQ(instance.scheduleoption().schedpolicyname(), "monopoly"); + EXPECT_EQ(instance.instancestatus().code(), static_cast(InstanceState::SCHEDULING)); + EXPECT_EQ(instance.instancestatus().msg(), "scheduling"); + EXPECT_EQ(instance.tenantid(), "tenantID"); + EXPECT_EQ(instance.resources().resources().find(CPU_RESOURCE_NAME)->second.scalar().value(), config.cpu); + EXPECT_EQ(instance.resources().resources().find(MEMORY_RESOURCE_NAME)->second.scalar().value(), config.memory); +} + +/** + * Feature: BootstrapActorTest LoadSysFuncPayloadsTest + * Description: load system-function payloads files + * Steps: + * 1. Load system-function payloads from files + * Expectation: + * 1. Local cache saved correct payloads + */ +TEST_F(BootstrapActorTest, LoadSysFuncPayloadsTest) +{ + (void)litebus::os::Rmdir(SYSTEM_FUNC_PAYLOAD_PATH); + auto status = LoadSysFuncPayloads(); + EXPECT_EQ(status.StatusCode(), StatusCode::FAILED); + + GenSystemFunctionPayloadFiles(payloadContent); + status = LoadSysFuncPayloads(); + auto payload = GetFunctionPayload("0-system-faasscheduler").Get(); + EXPECT_TRUE(status.IsOk()); + EXPECT_EQ(payload.sysFuncName, "0-system-faascontroller"); + EXPECT_EQ(payload.signal, 65); + EXPECT_EQ(payload.payload.dump(), R"({"version":"v1"})"); + + payload = GetFunctionPayload("0-system-faasfrontend").Get(); + EXPECT_EQ(payload.sysFuncName, "0-system-faascontroller"); + EXPECT_EQ(payload.signal, 66); + EXPECT_EQ(payload.payload.at("instanceNum").get(), 0); + bootstrapActor_->ScaleByFunctionName("0-system-faasfrontend", 3); + payload = GetFunctionPayload("0-system-faasfrontend").Get(); + EXPECT_EQ(payload.payload.at("instanceNum").get(), 3); + // boostrapActor change to slave + explorer::LeaderInfo leaderInfo{.name = "newMaster", .address = "127.0.0.9:8080"}; + bootstrapActor_->UpdateLeaderInfo(leaderInfo); + bootstrapActor_->ScaleByFunctionName("0-system-faasfrontend", 4); + payload = GetFunctionPayload("0-system-faasfrontend").Get(); + EXPECT_EQ(payload.payload.at("instanceNum").get(), 4); +} + +/** + * Feature: BootstrapActorTest LoadCurrentFunctionConfigsTest + * Description: Bootstrap load current system-function-configs from etcd + * Steps: + * 1. Load system-function configs from file + * 2. Put configs into etcd + * 3. Load system-function configs from etcd + * + * Expectation: + * 1. All configs can be loaded from etcd + * 2. All configs are consist with local cache + */ +TEST_F(BootstrapActorTest, LoadCurrentFunctionConfigsTest) +{ + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx2":{"xxx":1000}}},"0-system-faasfrontend":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + GenSystemFunctionConfigFile(content); + + auto status = LoadFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + auto controllerConfig = GetFunctionConfig("0-system-faascontroller").Get(); + auto frontConfig = GetFunctionConfig("0-system-faasfrontend").Get(); + + auto result = + metaStoreAccessor_->Put("/faas/system-function/config/0-system-faascontroller", controllerConfig.jsonStr); + EXPECT_AWAIT_READY(result); + result = metaStoreAccessor_->Put("/faas/system-function/config/0-system-faasfrontend", frontConfig.jsonStr); + EXPECT_AWAIT_READY(result); + + status = LoadCurrentFunctionConfigs(); + EXPECT_TRUE(status.IsOk()); + auto currControllerConfig = GetCurrFunctionConfig("0-system-faascontroller").Get(); + auto currFrontConfig = GetCurrFunctionConfig("0-system-faasfrontend").Get(); + + EXPECT_EQ(controllerConfig.tenantID, currControllerConfig.tenantID); + EXPECT_EQ(controllerConfig.version, currControllerConfig.version); + EXPECT_EQ(controllerConfig.cpu, currControllerConfig.cpu); + EXPECT_EQ(controllerConfig.memory, currControllerConfig.memory); + EXPECT_EQ(controllerConfig.createOptions.size(), currControllerConfig.createOptions.size()); + EXPECT_EQ(controllerConfig.createOptions.find("concurrentNum")->second, + currControllerConfig.createOptions.find("concurrentNum")->second); + EXPECT_EQ(controllerConfig.instanceNum, currControllerConfig.instanceNum); + EXPECT_EQ(controllerConfig.extension.at("schedule_policy"), currControllerConfig.extension.at("schedule_policy")); + EXPECT_EQ(controllerConfig.args.dump(), currControllerConfig.args.dump()); + EXPECT_EQ(controllerConfig.jsonStr, currControllerConfig.jsonStr); + + EXPECT_EQ(frontConfig.tenantID, currFrontConfig.tenantID); + EXPECT_EQ(frontConfig.version, currFrontConfig.version); + EXPECT_EQ(frontConfig.cpu, currFrontConfig.cpu); + EXPECT_EQ(frontConfig.memory, currFrontConfig.memory); + EXPECT_EQ(frontConfig.createOptions.find("concurrentNum")->second, + currFrontConfig.createOptions.find("concurrentNum")->second); + EXPECT_EQ(frontConfig.instanceNum, currFrontConfig.instanceNum); + EXPECT_EQ(frontConfig.extension.at("schedule_policy"), currFrontConfig.extension.at("schedule_policy")); + EXPECT_EQ(frontConfig.args.dump(), currFrontConfig.args.dump()); + EXPECT_EQ(frontConfig.jsonStr, currFrontConfig.jsonStr); +} + +/** + * Feature: BootstrapActorTest UpdateSysFuncMetaTest + * Description: load system-function meta files + * Steps: + * 1. Load system-function metaInfo from files + * 2. reload new system-function metaInfo from files twice + * + * Expectation: + * 1. Local cache saved two versions + * 2. metaStore saved two versions + */ +TEST_F(BootstrapActorTest, UpdateSysFuncMetaTest) +{ + (void)litebus::os::Rmdir(SYSTEM_FUNC_META_PATH); + auto status = LoadSysFuncMetas(); + EXPECT_EQ(status.StatusCode(), StatusCode::FAILED); + + std::unordered_map content = { + { "faas-controller-meta.json", controllerV2 }, + { "faas-scheduler-meta.json", schedulerV1 }, + { "faas-frontend-meta.json", frontendV1 }, + }; + + auto funcMeta = GetFuncMetaFromJson(controllerV1); + auto future = + metaStoreAccessor_->Put(FUNC_META_PATH_PREFIX + "/0/function/faascontroller/version/pre1", controllerV0); + EXPECT_AWAIT_READY(future); + future = metaStoreAccessor_->Put(FUNC_META_PATH_PREFIX + "/0/function/faascontroller/version/pre2", controllerV0); + EXPECT_AWAIT_READY(future); + future = metaStoreAccessor_->Put(FUNC_META_PATH_PREFIX + "/0/function/faascontroller/version/pre3", controllerV0); + EXPECT_AWAIT_READY(future); + future = metaStoreAccessor_->Put(FUNC_META_PATH_PREFIX + "/0/function/faascontroller/version/pre4", controllerV0); + EXPECT_AWAIT_READY(future); + future = metaStoreAccessor_->Put(FUNC_META_PATH_PREFIX + "/0/function/faascontroller/version/pre5", controllerV0); + EXPECT_AWAIT_READY(future); + + LoadCurrentSysFuncMetas(); + auto queue = GetFunctionMetaQueue().Get(); + EXPECT_EQ(queue.size(), 1u); + EXPECT_EQ(queue["faascontroller"].size(), 5u); + + GenSystemFunctionMetaFiles(metaContent); + UpdateMetaHandler(); + queue = GetFunctionMetaQueue().Get(); + EXPECT_EQ(queue.size(), 3u); + EXPECT_EQ(queue["faascontroller"].size(), 6u); + EXPECT_EQ(queue["faasscheduler"].size(), 1u); + EXPECT_EQ(queue["faasfrontend"].size(), 1u); + + GenSystemFunctionMetaFiles(content); + UpdateMetaHandler(); + queue = GetFunctionMetaQueue().Get(); + EXPECT_EQ(queue.size(), 3u); + EXPECT_EQ(queue["faascontroller"].size(), 6u); + EXPECT_EQ(queue["faasscheduler"].size(), 1u); + EXPECT_EQ(queue["faasfrontend"].size(), 1u); + + GenSystemFunctionMetaFiles(metaContent); + UpdateMetaHandler(); + queue = GetFunctionMetaQueue().Get(); + EXPECT_EQ(queue["faascontroller"].at(0).first, + "/yr/functions/business/yrk/tenant/0/function/faascontroller/version/pre2"); + EXPECT_EQ(queue["faascontroller"].at(4).first, + "/yr/functions/business/yrk/tenant/0/function/faascontroller/version/v2"); + EXPECT_EQ(queue["faascontroller"].at(5).first, + "/yr/functions/business/yrk/tenant/0/function/faascontroller/version/v1"); + ASSERT_AWAIT_TRUE([=]() { + return metaStoreAccessor_->GetAllWithPrefix(FUNC_META_PATH_PREFIX + "/0/function/faascontroller").Get().size() == 6; + }); + ASSERT_AWAIT_TRUE([=]() { + return metaStoreAccessor_->GetAllWithPrefix(FUNC_META_PATH_PREFIX + "/0/function/faasscheduler").Get().size() == 1; + }); + ASSERT_AWAIT_TRUE([=]() { + return metaStoreAccessor_->GetAllWithPrefix(FUNC_META_PATH_PREFIX + "/0/function/faasfrontend").Get().size() == 1; + }); +} + +/** + * Feature: BootstrapActorTest ScheduleTest + * Description: Send schedule request to global scheduler + * Steps: + * 1. Put instance into meta_store + * 2. Write config file + * 3. LoadBootstrapConfig + * + * Expectation: + * 3. Send 3 schedule request + */ +TEST_F(BootstrapActorTest, ScheduleTest) +{ + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}},"0-system-actorcontroller":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + bootstrapActor_->retryTimeoutMs_ = 100; + GenSystemFunctionPayloadFiles(payloadContent); + GenSystemFunctionMetaFiles(metaContent); + GenSystemFunctionConfigFile(content); + + auto expectedCall = std::make_shared>(); + EXPECT_CALL(*globalSched_, Schedule) + .WillOnce(testing::Return(Status(StatusCode::FAILED))) + // retry.WillOnce(testing::Return(Status(StatusCode::SUCCESS))) + .WillOnce(testing::Return(Status(StatusCode::SUCCESS))) + .WillOnce(testing::DoAll(testing::Invoke([expectedCall](const std::shared_ptr &) { + expectedCall->SetValue(true); + }), + testing::Return(Status(StatusCode::SUCCESS)))); + + litebus::Promise promise; + promise.SetValue({ "", nullptr }); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillRepeatedly(testing::Return(promise2.GetFuture())); + + LoadBootstrapConfig(""); + EXPECT_AWAIT_READY(expectedCall->GetFuture()); + auto currControllerConfig = GetCurrFunctionConfig("0-system-faascontroller"); + EXPECT_EQ(currControllerConfig.IsSome(), true); +} + +/** + * Feature: BootstrapActorTest KillInstancesTest + * Description: Kill system function instances + * Steps: + * 1. Put function accessor instance into meta_store + * 2. Write config file, load config + * 3. Kill instances + * + * Expectation: + * 2. Call global scheduler 3 times + * 3. Call function accessor 3 times + */ +TEST_F(BootstrapActorTest, KillInstancesTest) +{ + auto mockLocal = std::make_shared(); + litebus::Spawn(mockLocal); + + auto funcKey = "0/0-system-faascontroller/$latest"; + auto funcKey2 = "0/0-system-faasfrontend/$latest"; + auto key = "0-system-faascontroller-0"; + auto key2 = "0-system-faasfrontend-0"; + auto key3 = "0-system-faasfrontend-1"; + auto instanceInfoStr = + R"({"instanceID":"0-system-faascontroller-0","requestID":"0-system-faascontroller-0","runtimeID":"runtime-ca040000-0000-4000-80df-0a1e4dc86433","runtimeAddress":"10.42.2.214:21006","functionAgentID":"function_agent_10.42.2.214-58866","functionProxyID":"mock","function":"0/0-system-faascontroller/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":666}},"CPU":{"name":"CPU","scalar":{"value":666}}}},"scheduleOption":{"schedPolicyName":"monopoly","affinity":{"instanceAffinity":{}},"resourceSelector":{"resource.owner":"0-system-faascontroller-0"}},"createOptions":{"DELEGATE_ENCRYPT":"{\"metaEtcdPwd\":\"\"}","resource.owner":"system","NO_USE_DELEGATE_VOLUME_MOUNTS":"[{\"name\":\"sts-config\",\"mountPath\":\"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/apple/a\",\"subPath\":\"a\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/boy/b\",\"subPath\":\"b\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/cat/c\",\"subPath\":\"c\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/dog/d\",\"subPath\":\"d\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.ini\",\"subPath\":\"HMSCaaSYuanRongWorker.ini\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.sts.p12\",\"subPath\":\"HMSCaaSYuanRongWorker.sts.p12\"},{\"name\":\"certs-volume\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/\"}]","concurrentNum":"10"},"instanceStatus":{"code":3,"msg":"running"},"storageType":"local","scheduleTimes":1,"deployTimes":1,"args":[{"value":"eyJmYWFzZnJvbnRlbmRDb25maWciOnsiYXV0aGVudGljYXRpb25FbmFibGUiOmZhbHNlLCJidXNpbmVzc1R5cGUiOjEsImNsdXN0ZXJJRCI6ImNsdXN0ZXIwMDEiLCJjcHUiOjEyMDAsImRhdGFTeXN0ZW1Db25maWciOnsiZXhlY3V0ZVRUTFNlYyI6MTgwMCwiZXhlY3V0ZVdyaXRlTW9kZSI6Ik5vbmVMMkNhY2hlIiwidGltZW91dE1zIjo2MDAwMCwidXBsb2FkVFRMU2VjIjo4NjQwMCwidXBsb2FkV3JpdGVNb2RlIjoiTm9uZUwyQ2FjaGUifSwiZnVuY3Rpb25DYXBhYmlsaXR5IjoxLCJodHRwIjp7Im1heFJlcXVlc3RCb2R5U2l6ZSI6NiwicmVzcHRpbWVvdXQiOjUsIndvcmtlckluc3RhbmNlUmVhZFRpbWVPdXQiOjV9LCJtZW1vcnkiOjQwOTYsInNsYVF1b3RhIjoxMDAwLCJ0cmFmZmljTGltaXREaXNhYmxlIjp0cnVlLCJ1cnBjQ29uZmlnIjp7ImVuYWJsZWQiOmZhbHNlLCJwb2xsaW5nTnVtIjowLCJwb29sU2l6ZSI6MjAwLCJwb3J0IjoxOTk5Niwid29ya2VyTnVtIjoxMH19LCJmYWFzc2NoZWR1bGVyQ29uZmlnIjp7ImJ1cnN0U2NhbGVOdW0iOjEwMDAsImNwdSI6MTAwMCwiZG9ja2VyUm9vdFBhdGgiOiIvaG9tZS9kaXNrL2RvY2tlciIsImxlYXNlU3BhbiI6MTAwMCwibWVtb3J5Ijo0MDk2LCJzY2FsZURvd25UaW1lIjo2MDAwMCwic2xhUXVvdGEiOjEwMDB9LCJmcm9udGVuZEluc3RhbmNlTnVtIjoxLCJtZXRhRXRjZCI6eyJwYXNzd29yZCI6IiIsInNlcnZlcnMiOlsiNy4yMTguMjEuMjQ6MzIzODAiXSwic3NsRW5hYmxlIjpmYWxzZSwidXNlciI6IiJ9LCJyYXdTdHNDb25maWciOnsic2VydmVyQ29uZmlnIjp7ImRvbWFpbiI6IiR7U1RTX0RPTUFJTl9TRVJWRVJ9IiwicGF0aCI6Ii9vcHQvaHVhd2VpL2NlcnRzL0hNU0NsaWVudENsb3VkQWNjZWxlcmF0ZVNlcnZpY2UvSE1TQ2FhU1l1YW5Sb25nV29ya2VyL0hNU0NhYXNZdWFuUm9uZ1dvcmtlci5pbmkifSwic3RzRW5hYmxlIjpmYWxzZX0sInJvdXRlckV0Y2QiOnsicGFzc3dvcmQiOiIiLCJzZXJ2ZXJzIjpbIjcuMjE4LjIxLjI0OjMyMzc5Il0sInNzbEVuYWJsZSI6ZmFsc2UsInVzZXIiOiIifSwic2NoZWR1bGVySW5zdGFuY2VOdW0iOjEsInRsc0NvbmZpZyI6eyJjYUNvbnRlbnQiOiIke0NBX0NPTlRFTlR9IiwiY2VydENvbnRlbnQiOiIke0NFUlRfQ09OVEVOVH0iLCJrZXlDb250ZW50IjoiJHtLRVlfQ09OVEVOVH0ifX0="}],"version":"3","dataSystemHost":"10.244.158.229"})"; + const std::string content = + R"({"0-system-faascontroller":{"memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}},"0-system-faasfrontend":{"memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + + GenSystemFunctionPayloadFiles({}); + GenSystemFunctionMetaFiles(metaContent); + GenSystemFunctionConfigFile(content); + auto expectedCall = std::make_shared>(); + EXPECT_CALL(*globalSched_, Schedule) + .WillOnce(testing::DoAll( + testing::Invoke([funcKey2, key2, instanceInfoStr](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey2, key2, key2).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + }), + testing::Return(Status(StatusCode::SUCCESS)))) + .WillOnce(testing::DoAll( + testing::Invoke([funcKey2, key3, instanceInfoStr](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey2, key3, key3).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + }), + testing::Return(Status(StatusCode::SUCCESS)))) + .WillOnce(testing::DoAll(testing::Invoke([funcKey, key, instanceInfoStr, + expectedCall](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey, key, key).Get(), + instanceInfoStr); + EXPECT_AWAIT_READY(status); + expectedCall->SetValue(true); + }), + testing::Return(Status(StatusCode::SUCCESS)))); + + litebus::Promise promise; + promise.SetValue({ "", nullptr }); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillRepeatedly(testing::Return(promise2.GetFuture())); + + LoadBootstrapConfig(""); + EXPECT_AWAIT_READY(expectedCall->GetFuture()); + + litebus::Future> address; + address.SetValue(litebus::Option(localAddress_)); + + EXPECT_CALL(*globalSched_, GetLocalAddress).WillRepeatedly(testing::Return(address)); + EXPECT_CALL(*mockLocal, ForwardCustomSignalRequest) + .Times(3) + .WillRepeatedly(testing::Invoke([aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + })); + + auto status = litebus::Async(bootstrapActor_->GetAID(), &BootstrapActor::KillSystemFuncInstances); + EXPECT_AWAIT_READY(status); + EXPECT_TRUE(status.IsOK()); + + litebus::Terminate(mockLocal->GetAID()); + litebus::Await(mockLocal->GetAID()); +} + +/** + * Feature: BootstrapActorTest KeepAliveTest + * Description: Bootstrap keep system function alive + * Steps: + * 1. Start system function + * 2. Delete instance key in meta store + * 3. Wait for check, and start function again + * + * Expectation: + * 1. Invoke schedule once + * 3. Invoke schedule again + */ +TEST_F(BootstrapActorTest, KeepAliveTest) +{ + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + + GenSystemFunctionPayloadFiles(payloadContent); + GenSystemFunctionMetaFiles(metaContent); + GenSystemFunctionConfigFile(content); + + auto expectedCall1 = std::make_shared>(); + auto expectedCall2 = std::make_shared>(); + EXPECT_CALL(*globalSched_, Schedule) + .WillOnce(testing::DoAll( + testing::Invoke([expectedCall1](const std::shared_ptr &) { + expectedCall1->SetValue(true); + }), + testing::Return(Status(StatusCode::SUCCESS)))) + .WillOnce(testing::DoAll( + testing::Invoke([expectedCall2](const std::shared_ptr &) { + expectedCall2->SetValue(true); + }), + testing::Return(Status(StatusCode::SUCCESS)))); + litebus::Promise promise; + promise.SetValue({ "", nullptr }); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise2.GetFuture())) + .WillOnce(testing::Return(promise2.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillRepeatedly(testing::Return(promise2.GetFuture())); + + LoadBootstrapConfig(""); + EXPECT_AWAIT_READY(expectedCall1->GetFuture()); + EXPECT_AWAIT_READY(expectedCall2->GetFuture()); + + litebus::Terminate(bootstrapActor_->GetAID()); + litebus::Await(bootstrapActor_->GetAID()); +} + +TEST_F(BootstrapActorTest, UpdateSysFunctionPayloadTest) +{ + bootstrapActor_->waitKillInstanceMs_ = 300; + bootstrapActor_->waitStartInstanceMs_ = 1000; + bootstrapActor_->waitUpdateConfigMapMs_ = 100; + bootstrapActor_->retryTimeoutMs_ = 100; + + auto mockLocal = std::make_shared(); + litebus::Spawn(mockLocal); + + auto funcKey = "0/0-system-faascontroller/$latest"; + auto key = "0-system-faascontroller-0"; + const std::string content = + R"({"0-system-faascontroller":{"tenantID":"0","version":"$latest","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}},"0-system-faasfrontend":{"tenantID":"0","version":"$latest","memory":100,"cpu":100,"createOptions":{"concurrentNum":"1"},"instanceNum":2,"schedulingOps": {"extension": {"schedule_policy": "shared"}},"args":{"xxx":4,"xxx2":{"xxx":2000}}}})"; + auto instanceInfoStr = + R"({"instanceID":"0-system-faascontroller-0","requestID":"0-system-faascontroller-0","runtimeID":"runtime-ca040000-0000-4000-80df-0a1e4dc86433","runtimeAddress":"10.42.2.214:21006","functionAgentID":"function_agent_10.42.2.214-58866","functionProxyID":"mock","function":"0/0-system-faascontroller/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":666}},"CPU":{"name":"CPU","scalar":{"value":666}}}},"scheduleOption":{"schedPolicyName":"monopoly","affinity":{"instanceAffinity":{}},"resourceSelector":{"resource.owner":"0-system-faascontroller-0"}},"createOptions":{"DELEGATE_ENCRYPT":"{\"metaEtcdPwd\":\"\"}","resource.owner":"system","NO_USE_DELEGATE_VOLUME_MOUNTS":"[{\"name\":\"sts-config\",\"mountPath\":\"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/apple/a\",\"subPath\":\"a\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/boy/b\",\"subPath\":\"b\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/cat/c\",\"subPath\":\"c\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/dog/d\",\"subPath\":\"d\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.ini\",\"subPath\":\"HMSCaaSYuanRongWorker.ini\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.sts.p12\",\"subPath\":\"HMSCaaSYuanRongWorker.sts.p12\"},{\"name\":\"certs-volume\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/\"}]","concurrentNum":"10"},"instanceStatus":{"code":3,"msg":"running"},"storageType":"local","scheduleTimes":1,"deployTimes":1,"args":[{"value":"eyJmYWFzZnJvbnRlbmRDb25maWciOnsiYXV0aGVudGljYXRpb25FbmFibGUiOmZhbHNlLCJidXNpbmVzc1R5cGUiOjEsImNsdXN0ZXJJRCI6ImNsdXN0ZXIwMDEiLCJjcHUiOjEyMDAsImRhdGFTeXN0ZW1Db25maWciOnsiZXhlY3V0ZVRUTFNlYyI6MTgwMCwiZXhlY3V0ZVdyaXRlTW9kZSI6Ik5vbmVMMkNhY2hlIiwidGltZW91dE1zIjo2MDAwMCwidXBsb2FkVFRMU2VjIjo4NjQwMCwidXBsb2FkV3JpdGVNb2RlIjoiTm9uZUwyQ2FjaGUifSwiZnVuY3Rpb25DYXBhYmlsaXR5IjoxLCJodHRwIjp7Im1heFJlcXVlc3RCb2R5U2l6ZSI6NiwicmVzcHRpbWVvdXQiOjUsIndvcmtlckluc3RhbmNlUmVhZFRpbWVPdXQiOjV9LCJtZW1vcnkiOjQwOTYsInNsYVF1b3RhIjoxMDAwLCJ0cmFmZmljTGltaXREaXNhYmxlIjp0cnVlLCJ1cnBjQ29uZmlnIjp7ImVuYWJsZWQiOmZhbHNlLCJwb2xsaW5nTnVtIjowLCJwb29sU2l6ZSI6MjAwLCJwb3J0IjoxOTk5Niwid29ya2VyTnVtIjoxMH19LCJmYWFzc2NoZWR1bGVyQ29uZmlnIjp7ImJ1cnN0U2NhbGVOdW0iOjEwMDAsImNwdSI6MTAwMCwiZG9ja2VyUm9vdFBhdGgiOiIvaG9tZS9kaXNrL2RvY2tlciIsImxlYXNlU3BhbiI6MTAwMCwibWVtb3J5Ijo0MDk2LCJzY2FsZURvd25UaW1lIjo2MDAwMCwic2xhUXVvdGEiOjEwMDB9LCJmcm9udGVuZEluc3RhbmNlTnVtIjoxLCJtZXRhRXRjZCI6eyJwYXNzd29yZCI6IiIsInNlcnZlcnMiOlsiNy4yMTguMjEuMjQ6MzIzODAiXSwic3NsRW5hYmxlIjpmYWxzZSwidXNlciI6IiJ9LCJyYXdTdHNDb25maWciOnsic2VydmVyQ29uZmlnIjp7ImRvbWFpbiI6IiR7U1RTX0RPTUFJTl9TRVJWRVJ9IiwicGF0aCI6Ii9vcHQvaHVhd2VpL2NlcnRzL0hNU0NsaWVudENsb3VkQWNjZWxlcmF0ZVNlcnZpY2UvSE1TQ2FhU1l1YW5Sb25nV29ya2VyL0hNU0NhYXNZdWFuUm9uZ1dvcmtlci5pbmkifSwic3RzRW5hYmxlIjpmYWxzZX0sInJvdXRlckV0Y2QiOnsicGFzc3dvcmQiOiIiLCJzZXJ2ZXJzIjpbIjcuMjE4LjIxLjI0OjMyMzc5Il0sInNzbEVuYWJsZSI6ZmFsc2UsInVzZXIiOiIifSwic2NoZWR1bGVySW5zdGFuY2VOdW0iOjEsInRsc0NvbmZpZyI6eyJjYUNvbnRlbnQiOiIke0NBX0NPTlRFTlR9IiwiY2VydENvbnRlbnQiOiIke0NFUlRfQ09OVEVOVH0iLCJrZXlDb250ZW50IjoiJHtLRVlfQ09OVEVOVH0ifX0="}],"version":"3","dataSystemHost":"10.244.158.229"})"; + + const std::unordered_map newPayloadContent = { + { "faas-scheduler-config.json", + R"({"0-system-faasscheduler":{"systemFunctionName":"0-system-faascontroller","signal":65,"payload":{"version":"v2"}}})" }, + { "faas-frontend-config.json", + R"({"0-system-faasfrontend":{"systemFunctionName":"0-system-faascontroller","signal":66,"payload":{"version":"v1"}}})" }, + }; + + GenSystemFunctionMetaFiles(metaContent); + GenSystemFunctionConfigFile(content); + + EXPECT_CALL(*globalSched_, Schedule) + .WillRepeatedly(testing::DoAll( + testing::Invoke( + [funcKey, key, instanceInfoStr, content](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey, key, key).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + status = metaStoreAccessor_->Put("/faas/system-function/config/0-system-faascontroller", content); + EXPECT_AWAIT_READY(status); + YRLOG_INFO("Success schedule {}", funcKey); + }), + testing::Return(Status(StatusCode::SUCCESS)))); + litebus::Future> address; + address.SetValue(litebus::Option(localAddress_)); + EXPECT_CALL(*globalSched_, GetLocalAddress).WillRepeatedly(testing::Return(address)); + + auto firstCallEnd = std::make_shared>(); + auto secondCallEnd = std::make_shared>(); + EXPECT_CALL(*mockLocal, ForwardCustomSignalRequest) + .WillOnce(testing::Invoke([aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + })) + .WillOnce(testing::Invoke([firstCallEnd, aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + firstCallEnd->SetValue(true); + })) + .WillOnce(testing::Invoke([aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + })) + .WillOnce(testing::Invoke([secondCallEnd, aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + secondCallEnd->SetValue(true); + })); + + litebus::Promise promise; + promise.SetValue({ "", nullptr }); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillOnce(testing::Return(promise.GetFuture())) + .WillRepeatedly(testing::Return(promise2.GetFuture())); + + GenSystemFunctionPayloadFiles(payloadContent); + UpdatePayloadHandler(); + auto config = GetFunctionPayload("0-system-faasscheduler").Get(); + auto status = SendUpdateArgsSignal("0-system-faasscheduler", config, 0); + EXPECT_FALSE(status.IsOk()); + status = SendUpdateArgsSignal("0-system-faasscheduler", config, 6); + EXPECT_FALSE(status.IsOk()); + + YRLOG_INFO("Start first update"); + LoadBootstrapConfig(""); + EXPECT_AWAIT_READY(firstCallEnd->GetFuture()); + + YRLOG_INFO("Start second update"); + GenSystemFunctionPayloadFiles(newPayloadContent); + UpdatePayloadHandler(); + EXPECT_AWAIT_READY(secondCallEnd->GetFuture()); + + litebus::Terminate(mockLocal->GetAID()); + litebus::Await(mockLocal->GetAID()); +} + +TEST_F(BootstrapActorTest, UpdateSysFunctionConfigTest) +{ + bootstrapActor_->waitKillInstanceMs_ = 100; + bootstrapActor_->waitStartInstanceMs_ = 100; + bootstrapActor_->waitUpdateConfigMapMs_ = 100; + bootstrapActor_->retryTimeoutMs_ = 100; + + auto mockLocal = std::make_shared(); + litebus::Spawn(mockLocal); + + auto funcKey = "0/0-system-faascontroller/v1"; + auto funcKey2 = "0/0-system-faascontroller/v2"; + auto key = "0-system-faascontroller-0"; + auto instanceInfoStr = + R"({"instanceID":"0-system-faascontroller-0","requestID":"0-system-faascontroller-0","runtimeID":"runtime-ca040000-0000-4000-80df-0a1e4dc86433","runtimeAddress":"10.42.2.214:21006","functionAgentID":"function_agent_10.42.2.214-58866","functionProxyID":"mock","function":"0/0-system-faascontroller/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":666}},"CPU":{"name":"CPU","scalar":{"value":666}}}},"scheduleOption":{"schedPolicyName":"monopoly","affinity":{"instanceAffinity":{}},"resourceSelector":{"resource.owner":"0-system-faascontroller-0"}},"createOptions":{"DELEGATE_ENCRYPT":"{\"metaEtcdPwd\":\"\"}","resource.owner":"system","NO_USE_DELEGATE_VOLUME_MOUNTS":"[{\"name\":\"sts-config\",\"mountPath\":\"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/apple/a\",\"subPath\":\"a\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/boy/b\",\"subPath\":\"b\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/cat/c\",\"subPath\":\"c\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/dog/d\",\"subPath\":\"d\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.ini\",\"subPath\":\"HMSCaaSYuanRongWorker.ini\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.sts.p12\",\"subPath\":\"HMSCaaSYuanRongWorker.sts.p12\"},{\"name\":\"certs-volume\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/\"}]","concurrentNum":"10"},"instanceStatus":{"code":3,"msg":"running"},"storageType":"local","scheduleTimes":1,"deployTimes":1,"args":[{"value":"eyJmYWFzZnJvbnRlbmRDb25maWciOnsiYXV0aGVudGljYXRpb25FbmFibGUiOmZhbHNlLCJidXNpbmVzc1R5cGUiOjEsImNsdXN0ZXJJRCI6ImNsdXN0ZXIwMDEiLCJjcHUiOjEyMDAsImRhdGFTeXN0ZW1Db25maWciOnsiZXhlY3V0ZVRUTFNlYyI6MTgwMCwiZXhlY3V0ZVdyaXRlTW9kZSI6Ik5vbmVMMkNhY2hlIiwidGltZW91dE1zIjo2MDAwMCwidXBsb2FkVFRMU2VjIjo4NjQwMCwidXBsb2FkV3JpdGVNb2RlIjoiTm9uZUwyQ2FjaGUifSwiZnVuY3Rpb25DYXBhYmlsaXR5IjoxLCJodHRwIjp7Im1heFJlcXVlc3RCb2R5U2l6ZSI6NiwicmVzcHRpbWVvdXQiOjUsIndvcmtlckluc3RhbmNlUmVhZFRpbWVPdXQiOjV9LCJtZW1vcnkiOjQwOTYsInNsYVF1b3RhIjoxMDAwLCJ0cmFmZmljTGltaXREaXNhYmxlIjp0cnVlLCJ1cnBjQ29uZmlnIjp7ImVuYWJsZWQiOmZhbHNlLCJwb2xsaW5nTnVtIjowLCJwb29sU2l6ZSI6MjAwLCJwb3J0IjoxOTk5Niwid29ya2VyTnVtIjoxMH19LCJmYWFzc2NoZWR1bGVyQ29uZmlnIjp7ImJ1cnN0U2NhbGVOdW0iOjEwMDAsImNwdSI6MTAwMCwiZG9ja2VyUm9vdFBhdGgiOiIvaG9tZS9kaXNrL2RvY2tlciIsImxlYXNlU3BhbiI6MTAwMCwibWVtb3J5Ijo0MDk2LCJzY2FsZURvd25UaW1lIjo2MDAwMCwic2xhUXVvdGEiOjEwMDB9LCJmcm9udGVuZEluc3RhbmNlTnVtIjoxLCJtZXRhRXRjZCI6eyJwYXNzd29yZCI6IiIsInNlcnZlcnMiOlsiNy4yMTguMjEuMjQ6MzIzODAiXSwic3NsRW5hYmxlIjpmYWxzZSwidXNlciI6IiJ9LCJyYXdTdHNDb25maWciOnsic2VydmVyQ29uZmlnIjp7ImRvbWFpbiI6IiR7U1RTX0RPTUFJTl9TRVJWRVJ9IiwicGF0aCI6Ii9vcHQvaHVhd2VpL2NlcnRzL0hNU0NsaWVudENsb3VkQWNjZWxlcmF0ZVNlcnZpY2UvSE1TQ2FhU1l1YW5Sb25nV29ya2VyL0hNU0NhYXNZdWFuUm9uZ1dvcmtlci5pbmkifSwic3RzRW5hYmxlIjpmYWxzZX0sInJvdXRlckV0Y2QiOnsicGFzc3dvcmQiOiIiLCJzZXJ2ZXJzIjpbIjcuMjE4LjIxLjI0OjMyMzc5Il0sInNzbEVuYWJsZSI6ZmFsc2UsInVzZXIiOiIifSwic2NoZWR1bGVySW5zdGFuY2VOdW0iOjEsInRsc0NvbmZpZyI6eyJjYUNvbnRlbnQiOiIke0NBX0NPTlRFTlR9IiwiY2VydENvbnRlbnQiOiIke0NFUlRfQ09OVEVOVH0iLCJrZXlDb250ZW50IjoiJHtLRVlfQ09OVEVOVH0ifX0="}],"version":"3","dataSystemHost":"10.244.158.229"})"; + + const std::string content1 = + R"({"0-system-faascontroller":{"version":"v1","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v1\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + + GenSystemFunctionPayloadFiles(payloadContent); + GenSystemFunctionMetaFiles(metaContent); + + EXPECT_CALL(*globalSched_, GetLocalAddress).WillRepeatedly(testing::Return(litebus::Option(localAddress_))); + + // start test + auto expectedCall = std::make_shared>(); + bool isEmpty = true; + EXPECT_CALL(*globalSched_, Schedule) + .WillOnce(testing::DoAll(testing::Invoke([funcKey](const std::shared_ptr &) { + YRLOG_INFO("Failed schedule {}", funcKey); + }), + testing::Return(Status(StatusCode::FAILED)))) + .WillOnce(testing::DoAll(testing::Invoke([expectedCall, funcKey, key, instanceInfoStr, + &isEmpty](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey, key, key).Get(), + instanceInfoStr); + EXPECT_AWAIT_READY(status); + expectedCall->SetValue(true); + isEmpty = false; + }), + testing::Return(Status(StatusCode::SUCCESS)))); + + auto getFuture = [&isEmpty]() { + litebus::Promise promise; + if (isEmpty) { + promise.SetValue({ "", nullptr }); + return promise.GetFuture(); + } + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + promise.SetValue({ "key", instance }); + return promise.GetFuture(); + }; + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID).WillRepeatedly(testing::Return(getFuture())); + + YRLOG_INFO("Start first update"); + GenSystemFunctionConfigFile(content1); + LoadBootstrapConfig(""); + EXPECT_AWAIT_READY(expectedCall->GetFuture()); + + auto updateConfigTestHandler = [&](const std::string &content, const std::string &killKey, + const std::string &putKey) { + auto expectedCall = std::make_shared>(); + EXPECT_CALL(*mockLocal, ForwardCustomSignalRequest) + .WillRepeatedly(testing::Invoke([killKey, key, aid(mockLocal->GetAID()), &isEmpty]( + const litebus::AID &from, std::string &&, std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + EXPECT_FALSE(forwardKillRequest.req().requestid().empty()); + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, + forwardKillRequest.requestid()); + auto status = metaStoreAccessor_->Delete(GenInstanceKey(killKey, key, key).Get()); + EXPECT_AWAIT_READY(status); + isEmpty = true; + })); + EXPECT_CALL(*globalSched_, Schedule) + .WillRepeatedly(testing::DoAll(testing::Invoke([expectedCall, putKey, key, instanceInfoStr, &isEmpty]( + const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put( + GenInstanceKey(putKey, key, key).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + expectedCall->SetValue(true); + isEmpty = false; + }), + testing::Return(Status(StatusCode::SUCCESS)))); + YRLOG_INFO("Start update {}", content); + GenSystemFunctionConfigFile(content); + UpdateConfigHandler(); + EXPECT_AWAIT_READY_FOR(expectedCall->GetFuture(), 20000); + }; + + const std::string content2 = + R"({"0-system-faascontroller":{"version":"v2","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(content2, funcKey, funcKey2); + + const std::string memoryDiff = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":500,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(memoryDiff, funcKey2, funcKey2); + + const std::string cpuDiff = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":600,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(cpuDiff, funcKey2, funcKey2); + + const std::string createOptDiff = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":600,"createOptions":{"concurrentNum":"11","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(createOptDiff, funcKey2, funcKey2); + + const std::string createOptDiff2 = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":600,"createOptions":{"DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(createOptDiff2, funcKey2, funcKey2); + + const std::string argsDiff = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":600,"createOptions":{"DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":3,"xxx":{"xxx":1000}}}})"; + updateConfigTestHandler(argsDiff, funcKey2, funcKey2); + + litebus::Terminate(mockLocal->GetAID()); + litebus::Await(mockLocal->GetAID()); +} + +TEST_F(BootstrapActorTest, UpdateConfigHandlerAtStartTest) +{ + bootstrapActor_->waitKillInstanceMs_ = 100; + bootstrapActor_->waitStartInstanceMs_ = 100; + bootstrapActor_->waitUpdateConfigMapMs_ = 100; + + auto mockLocal = std::make_shared(); + litebus::Spawn(mockLocal); + + auto funcKey = "0/0-system-faascontroller/v2"; + auto key = "0-system-faascontroller-0"; + auto instanceInfoStr = + R"({"instanceID":"0-system-faascontroller-0","requestID":"0-system-faascontroller-0","runtimeID":"runtime-ca040000-0000-4000-80df-0a1e4dc86433","runtimeAddress":"10.42.2.214:21006","functionAgentID":"function_agent_10.42.2.214-58866","functionProxyID":"mock","function":"0/0-system-faascontroller/$latest","resources":{"resources":{"Memory":{"name":"Memory","scalar":{"value":666}},"CPU":{"name":"CPU","scalar":{"value":666}}}},"scheduleOption":{"schedPolicyName":"monopoly","affinity":{"instanceAffinity":{}},"resourceSelector":{"resource.owner":"0-system-faascontroller-0"}},"createOptions":{"DELEGATE_ENCRYPT":"{\"metaEtcdPwd\":\"\"}","resource.owner":"system","NO_USE_DELEGATE_VOLUME_MOUNTS":"[{\"name\":\"sts-config\",\"mountPath\":\"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/apple/a\",\"subPath\":\"a\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/boy/b\",\"subPath\":\"b\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/cat/c\",\"subPath\":\"c\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker/dog/d\",\"subPath\":\"d\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.ini\",\"subPath\":\"HMSCaaSYuanRongWorker.ini\"},{\"name\":\"sts-config\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/HMSCaaSYuanRongWorker.sts.p12\",\"subPath\":\"HMSCaaSYuanRongWorker.sts.p12\"},{\"name\":\"certs-volume\",\"mountPath\": \"/opt/huawei/certs/HMSClientCloudAccelerateService/HMSCaaSYuanRongWorker/\"}]","concurrentNum":"10"},"instanceStatus":{"code":3,"msg":"running"},"storageType":"local","scheduleTimes":1,"deployTimes":1,"args":[{"value":"eyJmYWFzZnJvbnRlbmRDb25maWciOnsiYXV0aGVudGljYXRpb25FbmFibGUiOmZhbHNlLCJidXNpbmVzc1R5cGUiOjEsImNsdXN0ZXJJRCI6ImNsdXN0ZXIwMDEiLCJjcHUiOjEyMDAsImRhdGFTeXN0ZW1Db25maWciOnsiZXhlY3V0ZVRUTFNlYyI6MTgwMCwiZXhlY3V0ZVdyaXRlTW9kZSI6Ik5vbmVMMkNhY2hlIiwidGltZW91dE1zIjo2MDAwMCwidXBsb2FkVFRMU2VjIjo4NjQwMCwidXBsb2FkV3JpdGVNb2RlIjoiTm9uZUwyQ2FjaGUifSwiZnVuY3Rpb25DYXBhYmlsaXR5IjoxLCJodHRwIjp7Im1heFJlcXVlc3RCb2R5U2l6ZSI6NiwicmVzcHRpbWVvdXQiOjUsIndvcmtlckluc3RhbmNlUmVhZFRpbWVPdXQiOjV9LCJtZW1vcnkiOjQwOTYsInNsYVF1b3RhIjoxMDAwLCJ0cmFmZmljTGltaXREaXNhYmxlIjp0cnVlLCJ1cnBjQ29uZmlnIjp7ImVuYWJsZWQiOmZhbHNlLCJwb2xsaW5nTnVtIjowLCJwb29sU2l6ZSI6MjAwLCJwb3J0IjoxOTk5Niwid29ya2VyTnVtIjoxMH19LCJmYWFzc2NoZWR1bGVyQ29uZmlnIjp7ImJ1cnN0U2NhbGVOdW0iOjEwMDAsImNwdSI6MTAwMCwiZG9ja2VyUm9vdFBhdGgiOiIvaG9tZS9kaXNrL2RvY2tlciIsImxlYXNlU3BhbiI6MTAwMCwibWVtb3J5Ijo0MDk2LCJzY2FsZURvd25UaW1lIjo2MDAwMCwic2xhUXVvdGEiOjEwMDB9LCJmcm9udGVuZEluc3RhbmNlTnVtIjoxLCJtZXRhRXRjZCI6eyJwYXNzd29yZCI6IiIsInNlcnZlcnMiOlsiNy4yMTguMjEuMjQ6MzIzODAiXSwic3NsRW5hYmxlIjpmYWxzZSwidXNlciI6IiJ9LCJyYXdTdHNDb25maWciOnsic2VydmVyQ29uZmlnIjp7ImRvbWFpbiI6IiR7U1RTX0RPTUFJTl9TRVJWRVJ9IiwicGF0aCI6Ii9vcHQvaHVhd2VpL2NlcnRzL0hNU0NsaWVudENsb3VkQWNjZWxlcmF0ZVNlcnZpY2UvSE1TQ2FhU1l1YW5Sb25nV29ya2VyL0hNU0NhYXNZdWFuUm9uZ1dvcmtlci5pbmkifSwic3RzRW5hYmxlIjpmYWxzZX0sInJvdXRlckV0Y2QiOnsicGFzc3dvcmQiOiIiLCJzZXJ2ZXJzIjpbIjcuMjE4LjIxLjI0OjMyMzc5Il0sInNzbEVuYWJsZSI6ZmFsc2UsInVzZXIiOiIifSwic2NoZWR1bGVySW5zdGFuY2VOdW0iOjEsInRsc0NvbmZpZyI6eyJjYUNvbnRlbnQiOiIke0NBX0NPTlRFTlR9IiwiY2VydENvbnRlbnQiOiIke0NFUlRfQ09OVEVOVH0iLCJrZXlDb250ZW50IjoiJHtLRVlfQ09OVEVOVH0ifX0="}],"version":"3","dataSystemHost":"10.244.158.229"})"; + + const std::string content1 = + R"({"0-system-faascontroller":{"version":"v1","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v1\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + const std::string content2 = + R"({"0-system-faascontroller":{"version":"v2","memory":600,"cpu":600,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}}})"; + const std::string jsonStr1 = + R"({"version":"v1","memory":500,"cpu":500,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v1\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}})"; + const std::string jsonStr2 = + R"({"version":"v2","memory":600,"cpu":600,"createOptions":{"concurrentNum":"10","DELEGATE_RUNTIME_MANAGER":"{\"image\":\"v2\"}"},"instanceNum":1,"schedulingOps": {"extension": {"schedule_policy": "monopoly"}},"args":{"xxx":2,"xxx":{"xxx":1000}}})"; + + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey, key, key).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + status = metaStoreAccessor_->Put("/faas/system-function/config/0-system-faascontroller", + jsonStr2); // etcd config version == 2 + EXPECT_AWAIT_READY(status); + GenSystemFunctionPayloadFiles(payloadContent); + GenSystemFunctionMetaFiles(metaContent); + GenSystemFunctionConfigFile(content1); // local file version == v1 + + litebus::Future> address; + address.SetValue(litebus::Option(localAddress_)); + EXPECT_CALL(*globalSched_, GetLocalAddress).WillRepeatedly(testing::Return(address)); + + EXPECT_CALL(*mockLocal, ForwardCustomSignalRequest) + .WillRepeatedly(testing::Invoke([aid(mockLocal->GetAID())](const litebus::AID &from, std::string &&, + std::string &&msg) { + internal::ForwardKillRequest forwardKillRequest; + if (msg.empty() || !forwardKillRequest.ParseFromString(msg)) { + YRLOG_WARN("Failed to parse requestID from forwardKillRequest."); + return; + } + litebus::Async(aid, &MockLocalActor::SendForwardCustomSignalResponse, from, forwardKillRequest.requestid()); + })); + + litebus::Promise promise; + EXPECT_CALL(*globalSched_, Schedule) + .WillOnce(testing::Return(Status(StatusCode::SUCCESS))) + .WillRepeatedly(testing::DoAll( + testing::Invoke( + [funcKey, key, instanceInfoStr, jsonStr1, promise](const std::shared_ptr &) { + auto status = metaStoreAccessor_->Put(GenInstanceKey(funcKey, key, key).Get(), instanceInfoStr); + EXPECT_AWAIT_READY(status); + status = metaStoreAccessor_->Put("/faas/system-function/config/0-system-faascontroller", jsonStr1); + EXPECT_AWAIT_READY(status); + YRLOG_INFO("Success schedule {}", funcKey); + promise.SetValue(true); + }), + testing::Return(Status(StatusCode::SUCCESS)))); + + litebus::Promise p; + p.SetValue({ "", nullptr }); + + auto instance = std::make_shared(); + instance->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); + litebus::Promise promise2; + promise2.SetValue({ "key", instance }); + EXPECT_CALL(*mockInstanceMgr_, GetInstanceInfoByInstanceID) + .WillOnce(testing::Return(p.GetFuture())) + .WillRepeatedly(testing::Return(promise2.GetFuture())); + + LoadBootstrapConfig(""); + ASSERT_AWAIT_READY(promise.GetFuture()); + litebus::Terminate(mockLocal->GetAID()); + litebus::Await(mockLocal->GetAID()); +} + +// SlaveBusiness test cases +TEST_F(BootstrapActorTest, SlaveBusinessTest) // NOLINT +{ + auto member = std::make_shared(); + auto bootstrapActor = std::make_shared(nullptr, nullptr, 0, ""); + auto slaveBusiness = std::make_shared(bootstrapActor, member); + slaveBusiness->SystemFunctionKeepAlive(); + slaveBusiness->KillSystemFuncInstances(); + slaveBusiness->ScaleByFunctionName("0-system-faasfrontend", 2, {}); +} + +} // namespace functionsystem::system_function_loader::test diff --git a/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_driver_test.cpp b/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_driver_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..79006bd055e29c569feec59d2afb7bcc4bab72f2 --- /dev/null +++ b/functionsystem/tests/unit/function_master/system_function_loader/bootstrap_driver_test.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "function_master/system_function_loader/bootstrap_driver.h" + +#include + +#include "common/etcd_service/etcd_service_driver.h" +#include "utils/port_helper.h" + +namespace functionsystem::system_function_loader { +const std::string META_STORE_ADDRESS = "127.0.0.1:123"; +class BootstrapDriverTest : public ::testing::Test { +protected: + inline static std::unique_ptr etcdSrvDriver_; + + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + etcdSrvDriver_->StartServer(META_STORE_ADDRESS); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + etcdSrvDriver_->StopServer(); + } +}; + +/** + * Feature: Start Bootstrap with driver + * Description: start Bootstrap + * Steps: + * 1. set up params + * 2. Start driver + * 3. Stop driver + * + * Expectation: + * 2. StatusCode::SUCCESS + * 3. StatusCode::SUCCESS + */ +TEST_F(BootstrapDriverTest, StartSuccess) +{ + system_function_loader::SystemFunctionLoaderStartParam param{ .globalSched = + std::make_shared(), + .sysFuncRetryPeriod = 5000, + .sysFuncCustomArgs = "" }; + + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + std::string metaStoreServerHost = "127.0.0.1:" + std::to_string(metaStoreServerPort); + auto bootstrapDriver = std::make_shared( + param, MetaStoreClient::Create(MetaStoreConfig{ .etcdAddress = metaStoreServerHost }, + GrpcSslConfig(), MetaStoreTimeoutOption{ .operationRetryIntervalLowerBound = 100, + .operationRetryIntervalUpperBound = 100, + .operationRetryTimes = 1 })); + + auto status = bootstrapDriver->Start(); + EXPECT_EQ(status, StatusCode::SUCCESS); + + status = bootstrapDriver->Stop(); + EXPECT_EQ(status, StatusCode::SUCCESS); + bootstrapDriver->Await(); +} + +} // namespace functionsystem::system_function_loader diff --git a/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/instance_proxy_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/instance_proxy_test.cpp index 632c5704971588fe9ca52a01380676bd589f0675..5332af31015b973bf0c550aba0f1a0b60becc5aa 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/instance_proxy_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/instance_proxy_test.cpp @@ -21,15 +21,16 @@ #include -#include "metrics/metrics_adapter.h" -#include "metrics/metrics_constants.h" +#include "common/metrics/metrics_adapter.h" +#include "common/metrics/metrics_constants.h" #include "function_proxy/busproxy/instance_view/instance_view.h" #include "function_proxy/busproxy/invocation_handler/invocation_handler.h" #include "function_proxy/common/observer/data_plane_observer/data_plane_observer.h" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_shared_client.h" #include "mocks/mock_shared_client_manager_proxy.h" #include "utils/future_test_helper.h" -#include "metrics/metrics_adapter.h" +#include "common/metrics/metrics_adapter.h" namespace functionsystem::test { using namespace ::testing; @@ -63,7 +64,7 @@ public: Status AsyncDelete(const std::string &instanceID) { - instanceView_->Delete(instanceID); + instanceView_->Delete(instanceID, -1); return Status::OK(); } @@ -179,10 +180,11 @@ public: litebus::Await(calleeProxy); litebus::Terminate(observer_->GetAID()); litebus::Await(observer_->GetAID()); - instanceView_->Delete("callerIns"); - instanceView_->Delete("calleeIns"); + instanceView_->Delete("callerIns", -1); + instanceView_->Delete("calleeIns", -1); instanceInfo_.clear(); InstanceProxy::BindObserver(nullptr); + RequestDispatcher::BindInternalIAM(nullptr); RequestDispatcher::BindDataInterfaceClientManager(nullptr); instanceView_->BindDataInterfaceClientManager(nullptr); instanceView_ = nullptr; @@ -254,6 +256,7 @@ protected: return msg; })); } + if (!isLowReliability) { observer_->Update(calleeIns, calleeInfo); instanceInfo_[calleeIns] = calleeInfo; @@ -291,6 +294,7 @@ protected: // call result EXPECT_CALL(*mockSharedClient, NotifyResult(_)).WillRepeatedly(Return(runtime::NotifyResponse())); + EXPECT_CALL(*mockSharedClient, IsDone()).WillRepeatedly(Return(false)); auto callResultBeforeCreating = litebus::Async(isCalleeLocal ? callerProxy : calleeProxy, &InstanceProxy::CallResult, calleeIns, callerIns, @@ -352,6 +356,64 @@ TEST_F(InstanceProxyTest, CallLocalTest) CallTest(callerIns, calleeIns, true); } +TEST_F(InstanceProxyTest, CallLocalWithAuthorizeTest) +{ + function_proxy::InternalIAM::Param param_{ true, "" }; + auto mockInternalIam = std::make_shared(param_); + RequestDispatcher::BindInternalIAM(mockInternalIam); + EXPECT_CALL(*mockInternalIam, IsIAMEnabled).WillRepeatedly(Return(true)); + std::string callerIns = "callerIns"; + std::string calleeIns = "calleeIns"; + SetTenantID("tenant123"); + litebus::Future authParam; + litebus::Future authParam1; + EXPECT_CALL(*mockInternalIam, Authorize).WillOnce(testing::DoAll(FutureArg<0>(&authParam), Return(Status::OK()))) + .WillOnce(testing::DoAll(FutureArg<0>(&authParam1), Return(Status::OK()))); + CallTest(callerIns, calleeIns, true); + ASSERT_AWAIT_READY(authParam); + ASSERT_AWAIT_READY(authParam1); + EXPECT_EQ(authParam.Get().callerTenantID, ""); + EXPECT_EQ(authParam.Get().calleeTenantID, "tenant123"); + EXPECT_EQ(authParam1.Get().callerTenantID, "tenant123"); + EXPECT_EQ(authParam1.Get().calleeTenantID, "tenant123"); +} + +TEST_F(InstanceProxyTest, CallAuthorizeFailTest) +{ + function_proxy::InternalIAM::Param param_{ true, "" }; + auto mockInternalIam = std::make_shared(param_); + RequestDispatcher::BindInternalIAM(mockInternalIam); + EXPECT_CALL(*mockInternalIam, IsIAMEnabled).WillRepeatedly(Return(true)); + std::string callerIns = "callerIns"; + std::string calleeIns = "calleeIns"; + auto mockSharedClient = PrepareCaller(callerIns); + litebus::AID callerProxy(callerIns, observer_->GetAID().Url()); + ASSERT_AWAIT_TRUE([&]() { return litebus::GetActor(callerProxy) != nullptr; }); + auto calleeInfo = NewInstance(calleeIns, tenantID_); + observer_->Update(calleeIns, calleeInfo); + litebus::AID calleeProxy(calleeIns, observer_->GetAID().Url()); + + EXPECT_CALL(*mockInternalIam, Authorize).WillOnce(Return(Status(StatusCode::FAILED))); + UpdateInstance(calleeInfo, calleeIns, (int32_t)InstanceState::RUNNING, local_); + auto mockCalleeSharedClient = std::make_shared(); + EXPECT_CALL(*mockSharedClientManagerProxy_, NewDataInterfacePosixClient(calleeIns, _, _)) + .WillOnce(Return(mockCalleeSharedClient)); + EXPECT_CALL(*mockCalleeSharedClient, Call(_)) + .WillRepeatedly(Invoke([](const SharedStreamMsg &request) -> litebus::Future { + auto msg = std::make_shared(); + auto callrsp = msg->mutable_callrsp(); + callrsp->set_code(::common::ErrorCode::ERR_NONE); + return msg; + })); + observer_->Update(calleeIns, calleeInfo); + instanceInfo_[calleeIns] = calleeInfo; + auto firstCall = litebus::Async(callerProxy, &InstanceProxy::Call, + busproxy::CallerInfo{ .instanceID = callerIns, .tenantID = tenantID_ }, calleeIns, + CallRequest(callerIns, calleeIns, "Request-authorize-failed"), nullptr); + ASSERT_AWAIT_SET(firstCall); + EXPECT_TRUE(firstCall.Get()->has_callrsp() && firstCall.Get()->callrsp().code() == common::ERR_AUTHORIZE_FAILED); +} + /** * Feature: invoke test * Description: 模拟 bus proxy invoke 调用 @@ -401,7 +463,6 @@ TEST_F(InstanceProxyTest, CallLowAbilityTest) info->isLocal = true; info->runtimeID = calleeIns; info->proxyID = remote_; - info->isLowReliability = true; auto mockCalleeSharedClient = std::make_shared(); info->localClient = mockCalleeSharedClient; calleeProxyActor->NotifyChanged(calleeIns, info); @@ -417,7 +478,6 @@ TEST_F(InstanceProxyTest, CallLowAbilityTest) CallTest(callerIns, calleeIns, false, true); } - /** * Feature: NotifyChanged test * Description: when instance put event comes, do sth. according to instance info @@ -453,6 +513,27 @@ TEST_F(InstanceProxyTest, NotifyChanged) })); } +TEST_F(InstanceProxyTest, NotifyRemoteChanged) +{ + std::string calleeIns = "calleeIns"; + auto callerProxyActor = std::make_shared("callerIns", ""); + callerProxyActor->InitDispatcher(); + auto info = std::make_shared(); + info->isReady = true; + info->isLocal = false; + info->runtimeID = calleeIns; + info->proxyID = remote_; + litebus::Spawn(callerProxyActor); + + // instance is remote + callerProxyActor->NotifyChanged(calleeIns, info); + EXPECT_EQ(callerProxyActor->remoteDispatchers_[calleeIns]->remoteAid_.Name(), ""); + EXPECT_FALSE(callerProxyActor->remoteDispatchers_[calleeIns]->isReady_); + info->remote = callerProxyActor->GetAID(); + callerProxyActor->NotifyChanged(calleeIns, info); + EXPECT_TRUE(callerProxyActor->remoteDispatchers_[calleeIns]->isReady_); +} + /** * Feature: invoke test * Description: invoke unexist instance diff --git a/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/perf_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/perf_test.cpp index 83199fd7c1f10df4265c818df3c3fab231f66845..0073e366a0b7a1c5490e006cefdd8ae0513dc5e4 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/perf_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/perf_test.cpp @@ -19,7 +19,7 @@ #include #include "busproxy/instance_proxy/perf.h" -#include "status/status.h" +#include "common/status/status.h" namespace functionsystem::test { using namespace ::testing; diff --git a/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/request_router_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/request_router_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..249bf9bc8fda498b2c43e127ed43b74edfbed089 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/busproxy/instance_proxy/request_router_test.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "busproxy/instance_proxy/request_router.h" + +#include +#include + +#include + +#include "function_proxy/busproxy/invocation_handler/invocation_handler.h" +#include "function_proxy/common/observer/data_plane_observer/data_plane_observer.h" +#include "utils/future_test_helper.h" + +namespace functionsystem::test { +using namespace ::testing; +using namespace busproxy; +const std::string CUSTOMS_TAG = "CUSTOMS_TAG"; + +class MockTmpInstanceProxy : public busproxy::InstanceProxy { +public: + MockTmpInstanceProxy(const std::string &instanceID, const std::string &tenantID) + : busproxy::InstanceProxy(instanceID, tenantID) + { + } + + MOCK_METHOD(litebus::Future, DoForwardCall, + (const litebus::AID &, const std::shared_ptr &), (override)); + MOCK_METHOD(void, ResponseForwardCall, (const litebus::AID &, std::string &&, std::string &&msg), (override)); +}; + + +class RequestRouterTest : public ::testing::Test { +public: + void SetUp() override + { + requestRouter_ = std::make_shared(REQUEST_ROUTER_NAME); + litebus::Spawn(requestRouter_); + + toProxy_ = std::make_shared(remote_, ""); + litebus::Spawn(toProxy_); + + fromProxy_ = std::make_shared(local_, ""); + litebus::Spawn(fromProxy_); + } + + void TearDown() override + { + litebus::Terminate(toProxy_->GetAID()); + litebus::Await(toProxy_->GetAID()); + toProxy_ = nullptr; + + litebus::Terminate(fromProxy_->GetAID()); + litebus::Await(fromProxy_->GetAID()); + fromProxy_ = nullptr; + + litebus::Terminate(requestRouter_->GetAID()); + litebus::Await(requestRouter_->GetAID()); + requestRouter_ = nullptr; + } + + ::internal::RouteCallRequest GenRouteCallRequest(const std::string& requestID, const std::string& messageID) { + auto request = std::make_shared(); + auto callreq = request->mutable_callreq(); + callreq->set_senderid(local_); + callreq->set_requestid(requestID); + request->set_messageid(messageID); + + ::internal::RouteCallRequest routeReq; + routeReq.mutable_req()->CopyFrom(*request); + return routeReq; + } + +protected: + std::shared_ptr requestRouter_; + std::shared_ptr toProxy_; + std::shared_ptr fromProxy_; + std::string local_ = "local"; + std::string remote_ = "remote"; +}; + +TEST_F(RequestRouterTest, InstanceNotFound) +{ + auto routeReq = GenRouteCallRequest("requestID", "messageID"); + routeReq.set_instanceid("invalid_instance"); + + EXPECT_CALL(*toProxy_, DoForwardCall(_, _)).Times(0); + + std::string testMsg; + bool flag = false; + EXPECT_CALL(*fromProxy_, ResponseForwardCall(_, _, _)) + .WillOnce([&](const litebus::AID &, std::string &&, std::string &&msg) { + testMsg = std::move(msg); + flag = true; + }); + + requestRouter_->ForwardCall(fromProxy_->GetAID(), std::string(), routeReq.SerializeAsString()); + + EXPECT_AWAIT_TRUE([&]() -> bool { return flag; }); + auto response = std::make_shared(); + (void)response->ParseFromString(testMsg); + EXPECT_EQ(response->messageid(), "messageID"); + ASSERT_TRUE(response->has_callrsp()); + EXPECT_EQ(response->callrsp().code(), ERR_INSTANCE_NOT_FOUND); +} + +TEST_F(RequestRouterTest, InstanceFound) +{ + auto routeReq = GenRouteCallRequest("requestID", "messageID"); + routeReq.set_instanceid(remote_); + + std::string testMsg; + bool flag = false; + EXPECT_CALL(*toProxy_, DoForwardCall(_, _)).WillOnce([&](const litebus::AID &, const auto &) { + flag = true; + return litebus::Future(Status::OK()); + }); + requestRouter_->ForwardCall(fromProxy_->GetAID(), std::string(), routeReq.SerializeAsString()); + + EXPECT_AWAIT_TRUE([&]() -> bool { return flag; }); +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/busproxy/instance_view/instance_view_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/instance_view/instance_view_test.cpp index ffc3f2f158330b76859cce4c3b6dcee8f0a7a68f..95aa28f6f730ae8e0350e173f7075e196c821d28 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/instance_view/instance_view_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/instance_view/instance_view_test.cpp @@ -20,8 +20,8 @@ #include -#include "proto/pb/posix/resource.pb.h" -#include "resource_type.h" +#include "common/proto/pb/posix/resource.pb.h" +#include "common/resource_view/resource_type.h" #include "common/types/instance_state.h" #include "mocks/mock_shared_client.h" #include "mocks/mock_shared_client_manager_proxy.h" @@ -156,9 +156,9 @@ TEST_F(InstanceViewTest, InstanceStateChange) UpdateInstance(childA, parent, nodeID, "remote2", 1); aid.SetName(childA); ASSERT_AWAIT_TRUE([=]() -> bool { return litebus::GetActor(aid) == nullptr; }); - instanceView_->Delete(childA); - instanceView_->Delete(childB); - instanceView_->Delete(parent); + instanceView_->Delete(childA, -1); + instanceView_->Delete(childB, -1); + instanceView_->Delete(parent, -1); aid.SetName(parent); ASSERT_AWAIT_TRUE([=]() -> bool { return litebus::GetActor(aid) == nullptr; }); aid.SetName(childA); diff --git a/functionsystem/tests/unit/function_proxy/busproxy/invocation_handler/invocation_handler_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/invocation_handler/invocation_handler_test.cpp index b0d202a412b3e5ecb0039f1d6b4c387cf08a1286..363f6703db666e5e9fff72c006adfd345d2fc190 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/invocation_handler/invocation_handler_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/invocation_handler/invocation_handler_test.cpp @@ -26,6 +26,7 @@ #include "common/trace/trace_actor.h" #endif #include "function_proxy/common/observer/observer_actor.h" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_data_observer.h" #include "mocks/mock_instance_proxy_wrapper.h" #include "mocks/mock_shared_client.h" @@ -60,6 +61,7 @@ public: MemoryControlConfig config; memoryMonitor_ = std::make_shared(config); InvocationHandler::BindMemoryMonitor(memoryMonitor_); + InternalIAM::Param param_{ false, "" }; } void TearDown() override @@ -74,6 +76,7 @@ public: mockObserver_ = nullptr; etcdSrvDriver_->StopServer(); etcdSrvDriver_ = nullptr; + InvocationHandler::BindInternalIAM(nullptr); } protected: @@ -150,4 +153,45 @@ TEST_F(InvocationHandlerTest, CallResultAdapter) EXPECT_EQ(responseFuture.Get()->callresultack().code(), common::ERR_INNER_COMMUNICATION); } +TEST_F(InvocationHandlerTest, AuthorizeTest) +{ + InternalIAM::Param param_{ true, "" }; + auto mockInternalIAM = std::make_shared(param_); + InvocationHandler::BindInternalIAM(mockInternalIAM); + EXPECT_CALL(*mockInternalIAM, IsIAMEnabled).WillRepeatedly(Return(true)); + expectedAid_.SetName("fromIns"); + // instance actor is null + std::unique_ptr request = std::make_unique(); + request->mutable_invokereq()->set_instanceid("to"); + std::shared_ptr response = std::make_shared(); + response->mutable_callrsp()->set_code(common::ErrorCode::ERR_NONE); + litebus::Future callerInfoFuture; + EXPECT_CALL(*instanceProxy_, Call(expectedAid_, _, _, _, _)) + .WillOnce(testing::DoAll(FutureArg<1>(&callerInfoFuture), Return(response))); + auto responseFuture = InvocationHandler::Invoke("fromIns", std::move(request)); + ASSERT_AWAIT_READY(responseFuture); + EXPECT_EQ(responseFuture.Get()->invokersp().code(), common::ErrorCode::ERR_NONE); + ASSERT_AWAIT_READY(callerInfoFuture); + EXPECT_TRUE(callerInfoFuture.Get().tenantID.empty()); + EXPECT_EQ(callerInfoFuture.Get().instanceID, "fromIns"); + + // instance actor is not null + auto instanceProxy = std::make_shared("fromIns", "tenant123"); + instanceProxy->InitDispatcher(); + (void)litebus::Spawn(instanceProxy, true); + ASSERT_AWAIT_TRUE([&]() { return litebus::GetActor(expectedAid_) != nullptr; }); + + litebus::Future callerInfoFuture1; + EXPECT_CALL(*instanceProxy_, Call(expectedAid_, _, _, _, _)).WillOnce(testing::DoAll(FutureArg<1>(&callerInfoFuture1), Return(response))); + EXPECT_CALL(*instanceProxy_, GetTenantID).WillOnce(Return("tenant123")); + std::unique_ptr request1 = std::make_unique(); + request1->mutable_invokereq()->set_instanceid("to"); + InvocationHandler::Invoke("fromIns", std::move(request1)); + ASSERT_AWAIT_READY(callerInfoFuture1); + EXPECT_EQ(callerInfoFuture1.Get().tenantID, "tenant123"); + EXPECT_EQ(callerInfoFuture1.Get().instanceID, "fromIns"); + litebus::Terminate(expectedAid_); + litebus::Await(expectedAid_); +} + } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/function_proxy/busproxy/memory_monitor/memory_monitor_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/memory_monitor/memory_monitor_test.cpp index a1f14696034608650a719f9e2fede3312ac365dd..01ca35a0b06b30bc993af91a6a09cf4b31d51762 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/memory_monitor/memory_monitor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/memory_monitor/memory_monitor_test.cpp @@ -20,7 +20,7 @@ #include #include "utils/future_test_helper.h" #include "function_proxy/common/flags/flags.h" -#include "files.h" +#include "common/utils/files.h" #include "busproxy/memory_monitor/memory_monitor.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/function_proxy/busproxy/startup/startup_test.cpp b/functionsystem/tests/unit/function_proxy/busproxy/startup/startup_test.cpp index bd7afdbfd8a3c3271dad8b7f229cd79d4dbd457d..2b74fc5c75382bff94b712f016d5f5abf35daf36 100644 --- a/functionsystem/tests/unit/function_proxy/busproxy/startup/startup_test.cpp +++ b/functionsystem/tests/unit/function_proxy/busproxy/startup/startup_test.cpp @@ -20,7 +20,7 @@ #include "busproxy/startup/busproxy_startup.h" #include "common/etcd_service/etcd_service_driver.h" -#include "status/status.h" +#include "common/status/status.h" #include "utils/port_helper.h" namespace functionsystem::test { @@ -31,7 +31,7 @@ protected: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -39,7 +39,7 @@ protected: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } diff --git a/functionsystem/tests/unit/function_proxy/common/CMakeLists.txt b/functionsystem/tests/unit/function_proxy/common/CMakeLists.txt index 39000dd51e6c5087769ca69698438a8b166abf79..25239880d4faa3f5650f8c3da360324ea9148aa8 100644 --- a/functionsystem/tests/unit/function_proxy/common/CMakeLists.txt +++ b/functionsystem/tests/unit/function_proxy/common/CMakeLists.txt @@ -21,5 +21,6 @@ add_subdirectory(state_handler) add_subdirectory(state_machine) add_subdirectory(rate_limiter) add_subdirectory(common_driver) +add_subdirectory(iam) target_include_directories(${UNIT_TEST_MODULE} PRIVATE ${SRC_BUSPROXY_INCLUDE_DIR}) diff --git a/functionsystem/tests/unit/function_proxy/common/common_driver/common_driver_test.cpp b/functionsystem/tests/unit/function_proxy/common/common_driver/common_driver_test.cpp index 26b6c5096b5fcf1461bd6ff652f8780e03610983..1058abd111d6a871af15f1002387cfc882a78842 100644 --- a/functionsystem/tests/unit/function_proxy/common/common_driver/common_driver_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/common_driver/common_driver_test.cpp @@ -30,7 +30,7 @@ public: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -38,7 +38,7 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } diff --git a/functionsystem/tests/unit/function_proxy/common/distribute_cache_client/distribute_cache_client_test.cpp b/functionsystem/tests/unit/function_proxy/common/distribute_cache_client/distribute_cache_client_test.cpp index 3cfa0c71a0c3f3a32219d2c4723b616675c1a0c1..807884f88c0575a2d387abc63878d31074a3efa8 100644 --- a/functionsystem/tests/unit/function_proxy/common/distribute_cache_client/distribute_cache_client_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/distribute_cache_client/distribute_cache_client_test.cpp @@ -17,7 +17,7 @@ #include -#include "hex/hex.h" +#include "common/hex/hex.h" #include "function_proxy/common/distribute_cache_client/ds_cache_client_impl.h" #include "utils/os_utils.hpp" @@ -25,11 +25,11 @@ namespace functionsystem::test { class DistributeCacheClientTest : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } void SetUp() override diff --git a/functionsystem/tests/unit/function_proxy/common/iam/CMakeLists.txt b/functionsystem/tests/unit/function_proxy/common/iam/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c481f5c17d026193faf6b0a79bcdb2183b05d99 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/common/iam/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} IAM_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${IAM_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/iam/internal_credential_mgr_test.cpp b/functionsystem/tests/unit/function_proxy/common/iam/internal_credential_mgr_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3d13379e1fe4a3a95a72ac0f758aefe33b9d7503 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/common/iam/internal_credential_mgr_test.cpp @@ -0,0 +1,300 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/aksk/aksk_util.h" +#include "common/constants/constants.h" +#include "common/etcd_service/etcd_service_driver.h" +#include "common/utils/aksk_content.h" +#include "common/utils/sensitive_value.h" +#include "function_proxy/common/iam/internal_credential_manager_actor.h" +#include "utils/future_test_helper.h" +#include "utils/os_utils.hpp" +#include "utils/port_helper.h" + +namespace functionsystem::test { +using namespace functionsystem::function_proxy; + +class BuildInCredentialTest : public ::testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); + metaStorageAccessor_ = std::make_shared(metaStoreClient_); + } + + [[maybe_unused]] static void TearDownTestSuite() + { + metaStoreClient_ = nullptr; + metaStorageAccessor_ = nullptr; + etcdSrvDriver_->StopServer(); + etcdSrvDriver_ = nullptr; + } + +protected: + void SetUp() override + { + litebus::os::SetEnv("LITEBUS_AKSK_ENABLED", "1"); + litebus::os::SetEnv("LITEBUS_ACCESS_KEY", "c2b4887bb63e71b88773503708d625d8f0662ca18e513972bfcc8ffcfc7e3ed2"); + litebus::os::SetEnv( + "LITEBUS_SECRET_KEY", + SensitiveValue("16bf66680df37fd9d5ff6f781b50c53c463f4af0bb87c72c102aa7dde71da7cf").GetData()); + litebus::os::SetEnv( + LITEBUS_DATA_KEY, + SensitiveValue("0D2F1C6AEA8AFF3204128DFFD7CEC95471D1149DB628DF5267FDA5233C1D2F05").GetData()); + } + + void TearDown() override + { + litebus::os::UnSetEnv("LITEBUS_AKSK_ENABLED"); + litebus::os::UnSetEnv("LITEBUS_ACCESS_KEY"); + litebus::os::UnSetEnv("LITEBUS_SECRET_KEY"); + litebus::os::UnSetEnv(LITEBUS_DATA_KEY); + } + + inline static std::shared_ptr metaStoreClient_; + inline static std::shared_ptr metaStorageAccessor_; + inline static std::unique_ptr etcdSrvDriver_; + inline static std::string metaStoreServerHost_; +}; + +inline void ToUpper(std::string *source) +{ + (void)std::transform(source->begin(), source->end(), source->begin(), + [](unsigned char c) { return std::toupper(c); }); +} + +TEST_F(BuildInCredentialTest, InitCredentialIAM_ValidEnv_Test) // NOLINT +{ + const SensitiveValue dataKey = GetComponentDataKey(); + ASSERT_FALSE(dataKey.Empty()); + + std::string cipher; + SensitiveValue plain(R"({"tenantID":"t","accessKey":"a","secretKey":"s","dataKey":"d"})"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + auto actorName = "InternalCredentialManagerActor_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + const auto actor = std::make_shared(actorName, ""); + // actor->BindIAMClient(IAMClient::CreateIAMClient(TEST_IAM_BASE_PATH)); + actor->BindMetaStorageAccessor(metaStorageAccessor_); + const auto aid = litebus::Spawn(actor); + + const auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + EXPECT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 2); + + EXPECT_NE(actor->newCredMap_.find("t"), actor->newCredMap_.end()); + EXPECT_EQ(actor->newCredMap_.find("t")->second->accessKey, "a"); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + + auto result = litebus::Async(aid, &InternalCredentialManagerActor::AbandonCredential, "t"); + EXPECT_AWAIT_READY(result); // wait async action + EXPECT_TRUE(result.IsOK()); + + const auto count_2 = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + EXPECT_AWAIT_READY(count_2); // wait async action + EXPECT_EQ(count_2.Get(), 2); // 永久凭æ®ä¸æ”¯æŒåˆ é™¤ + + litebus::Terminate(aid); + litebus::Await(aid); +} + +TEST_F(BuildInCredentialTest, InitCredentialIAM_InvalidEnv_Test) // NOLINT +{ + SensitiveValue dataKey = GetComponentDataKey(); + ASSERT_FALSE(dataKey.Empty()); + + auto actorName = "InternalCredential_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto actor = std::make_shared(actorName, ""); + // actor->BindIAMClient(IAMClient::CreateIAMClient(TEST_IAM_BASE_PATH)); + actor->BindMetaStorageAccessor(metaStorageAccessor_); + auto aid = litebus::Spawn(actor); + // Do set env action after load credential from env within Init + auto ready = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(ready); // for ready + EXPECT_EQ(ready.Get(), 1); + + { // invalid json + SensitiveValue plain(R"({])"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + litebus::Async(aid, &InternalCredentialManagerActor::LoadBuiltInCredential); + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + } + + { // invalid access key + SensitiveValue plain(R"({"tenantID":"t"})"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + litebus::Async(aid, &InternalCredentialManagerActor::LoadBuiltInCredential); + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + } + + { // invalid secret key + SensitiveValue plain(R"({"tenantID":"t","accessKey":"a","secretKey":""})"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + litebus::Async(aid, &InternalCredentialManagerActor::LoadBuiltInCredential); + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + } + + { // invalid data key + SensitiveValue plain(R"({"tenantID":"t","accessKey":"a","secretKey":"s","dataKey":""})"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + litebus::Async(aid, &InternalCredentialManagerActor::LoadBuiltInCredential); + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + } + + { // no component data key + litebus::os::UnSetEnv(LITEBUS_DATA_KEY); + + SensitiveValue plain(R"({"tenantID":"t","accessKey":"a","secretKey":"s","dataKey":""})"); + litebus::os::SetEnv(YR_BUILD_IN_CREDENTIAL, plain.GetData()); + + litebus::Async(aid, &InternalCredentialManagerActor::LoadBuiltInCredential); + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::os::UnSetEnv(YR_BUILD_IN_CREDENTIAL); + } + + litebus::Terminate(aid); + litebus::Await(aid); +} + +class AES_GCM_Test : public ::testing::Test {}; + +TEST_F(AES_GCM_Test, UpperHexKeyDataKey) // NOLINT +{ + std::string upperHexKey = "E390F226ACF4F78E883F52147293BF25"; + std::string lowerHexKey = "e390f226acf4f78e883f52147293bf25"; + + EXPECT_EQ(FromHexString(lowerHexKey), FromHexString(upperHexKey)); + + litebus::os::SetEnv(LITEBUS_DATA_KEY, upperHexKey); + SensitiveValue udk = GetComponentDataKey(); + litebus::os::UnSetEnv(LITEBUS_DATA_KEY); + + SensitiveValue ldk(FromHexString(lowerHexKey)); + + EXPECT_EQ(udk, ldk); +} + +class SystemCredentialTest : public ::testing::Test { +protected: + void SetUp() override + { + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); + metaStorageAccessor_ = std::make_shared(metaStoreClient_); + } + + void TearDown() override + { + litebus::os::UnSetEnv("LITEBUS_AKSK_ENABLED"); + litebus::os::UnSetEnv("LITEBUS_ACCESS_KEY"); + litebus::os::UnSetEnv("LITEBUS_SECRET_KEY"); + litebus::os::UnSetEnv(LITEBUS_DATA_KEY); + + metaStorageAccessor_ = nullptr; + metaStoreClient_ = nullptr; + + etcdSrvDriver_->StopServer(); + etcdSrvDriver_ = nullptr; + } + +private: + std::shared_ptr metaStoreClient_; + std::shared_ptr metaStorageAccessor_; + std::unique_ptr etcdSrvDriver_; + std::string metaStoreServerHost_; +}; + +TEST_F(SystemCredentialTest, LoadSystemCredential_Test) // NOLINT +{ + litebus::os::SetEnv("LITEBUS_AKSK_ENABLED", "1"); + litebus::os::SetEnv("LITEBUS_ACCESS_KEY", "c2b4887bb63e71b88773503708d625d8f0662ca18e513972bfcc8ffcfc7e3ed2"); + litebus::os::SetEnv("LITEBUS_SECRET_KEY", + SensitiveValue("16bf66680df37fd9d5ff6f781b50c53c463f4af0bb87c72c102aa7dde71da7cf").GetData()); + litebus::os::SetEnv(LITEBUS_DATA_KEY, + SensitiveValue("0D2F1C6AEA8AFF3204128DFFD7CEC95471D1149DB628DF5267FDA5233C1D2F05").GetData()); + + auto actorName = "InternalCredential_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto actor = std::make_shared(actorName, "default-cluster"); + // actor->BindIAMClient(IAMClient::CreateIAMClient(TEST_IAM_BASE_PATH)); + actor->BindMetaStorageAccessor(metaStorageAccessor_); + auto aid = litebus::Spawn(actor); + + // Do set env action after load credential from env within Init + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 1); + + litebus::Terminate(aid); + litebus::Await(aid); +} + +TEST_F(SystemCredentialTest, LoadSystemCredential_Error_Test) // NOLINT +{ + auto actorName = "InternalCredential_" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto actor = std::make_shared(actorName, "default-cluster"); + // actor->BindIAMClient(IAMClient::CreateIAMClient(TEST_IAM_BASE_PATH)); + actor->BindMetaStorageAccessor(metaStorageAccessor_); + auto aid = litebus::Spawn(actor); + + // Do set env action after load credential from env within Init + auto count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 0); + + litebus::os::SetEnv("LITEBUS_AKSK_ENABLED", "1"); + litebus::Async(aid, &InternalCredentialManagerActor::LoadSystemCredential); + count = litebus::Async(aid, &InternalCredentialManagerActor::GetCredentialCount); + ASSERT_AWAIT_READY(count); // wait async action + EXPECT_EQ(count.Get(), 0); + + litebus::Terminate(aid); + litebus::Await(aid); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/iam/internal_iam_test.cpp b/functionsystem/tests/unit/function_proxy/common/iam/internal_iam_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bb1143e302d3189f6c646985aae85ce0e94b25e6 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/common/iam/internal_iam_test.cpp @@ -0,0 +1,1059 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "httpd/http_connect.hpp" + +#include "common/etcd_service/etcd_service_driver.h" +#include "common/utils/files.h" +#include "function_proxy/common/iam/internal_iam.h" +#include "mocks/mock_meta_store_client.h" +#include "utils/future_test_helper.h" +#include "utils/port_helper.h" + +namespace functionsystem::test { +using namespace functionsystem::function_proxy; + +namespace { +const std::string DEFAULT_POLICY_PATH = "/home/sn/stupid_internal_iam_secret"; +const std::string POLICY_FILE_NAME = "policy"; +const std::string RESOURCE_DIR = "/home/sn/resource"; +const std::string A_TXT_CONTENT = "f48f9d5a9706088947ac438ebe005aa26c9370579f2231c538b28894a315562182da0eb18002c86728c4cdc0df5efb19e1c2060e93370fd891d4f3d9e5b2b61376643f86d0210ce996446a985759b15112037a5a2f6463cf5fd6afc7ff30fe814bf960eb0c16c5059407c74d6a93a8b3110405cbc935dff672da3b648d62e0d5cecd91bc7063211e6b33210afb6899e8322eabffe167318a5ac5d591aa7579efd37e9e4c7fcf390e97c1151b7c1bf00b4a18764a1a0cac1fda1ea6389b39d755127f0e5bc072e6d5936738be1585535dc63b71ad58686f71c821325009de36bdbac31c1c044845bd1bb41230ec9815695ef3f9e7143a16410113ff3286147a76"; +const std::string B_TXT_CONTENT = "5d3da9f432be72b34951c737053eb2c816aaccae2b390d092046288aa5ce2cc5b16529f8197de316303735fbc0c041ccc3885b9be5fef4933b6806febb940b6bb609b3bf1d1501110e3ba62c6d8b2cf4388a08a8e123a3cea96daec619fbca177bdf092461f5701b02e5af83ddf0f6ce40deb279cda3ec7d6805237d229e26e30555f3dd890b7306b42bdef0ca1f963dbe25cd00d75018ab3216fcd3b7002b8a493d015306bf264cca12718890ef11c8d9e54721ebd6bdecab6c7084442f45611f249d9b5d703414770a46380d0b97c0187185241e9b6187c8168414370649fe6e7afef83a0df645424c4b6c0631dc3ef50c30af37eda905a1886ca12474c68a"; +const std::string D_TXT_CONTENT = "37a1b37efbb9bb6beadb4446f40aa2c4bcaeb298192fa390ed03ee65bfcd54e55da39bae9961b9fa0d4b89591e41eed835ed01cca315eab75ebaf8a9e7b02287a468ec6d0c61f9f8e4d58dad90fb8a6a13bee7fe4685dbb535bfdb7e76b328d66b4d4bc7aa48791b205d1d2f2ef176f2b5b80a8ddc34ed9514372130eb896bc18745facf059a7fa37ef5e2ef413d0030f5bca581055eb3b3565dca642651cb802530e2e4964ab3c8a37370adfd65c80483398a1a8668caed455deabae0dbae7fb2bcdeeee4c2a2d9431ed93c6527985ef684127691904c799e13f37daeb1cb7ebfb0904d61796362514e521ac0fed682fd952ca3e9ce9a7a4407aaaa44f8aab6"; +const std::string E_TXT_CONTENT = "43b0d158d9dcf4ffd416eb4e6a89d1b7a66d595c43329bb5c1c66d5befe33c37f31da53aaf539e43238457c46e1f28339cb9dda461c71c0ea2dba3dc8006684ff0d8d59ee2192582983c155e400d5b7cadcb65bbe682e61d175af54549796e447f3174b95f1f50998ae7785b5c0c359746e1ee6eeb989284fbe9e0f801ce5a7267285afbab7694c0e8434d6b86991298a46039de4d1fbfd824b8337b11c2d0b2f30ed4d46312e315cd9042abddc09ea73169f9e1f5baa496d44ed5cac9659cab076212499ef09a56db69e7444d665195a0562a7c82d176d027b0ecc7f4a26215e003fd463bf3911633baf85ee98f9187357a65ee2869b3d93a3871d830b4034e"; + +const std::string TEST_TENANT_ID = "TEST_TENANT_ID"; +const std::string TEST_INSTANCE_ID_1 = "TEST_INSTANCE_ID_1"; +const std::string TEST_INSTANCE_ID_2 = "TEST_INSTANCE_ID_2"; + +// encrypt once +std::string EncryptToken(std::string &token) +{ + return token; +} +// encrypt twice +std::string EncryptTokenForStorage(std::string &token) +{ + auto now = static_cast(std::time(nullptr)); + std::string tokenForStorage = token + "+" + std::to_string(now + 720); + return tokenForStorage; +} + +std::shared_ptr GenAkSkContent(const std::string &tenantID, const std::string &ak, const std::string &sk, + const std::string &dk) +{ + auto credential = std::make_shared(); + auto now = static_cast(std::time(nullptr)); + credential->tenantID = tenantID; + credential->expiredTimeStamp = now + 720; + credential->accessKey = ak; + credential->secretKey = sk; + credential->dataKey = dk; + return credential; +} +} + +class MockIAMClient : public IAMClient { +public: + static std::shared_ptr CreateMockIAMClient(const std::string &iamAddress) + { + auto client = std::make_shared(); + client->SetIAMAddress(iamAddress); + return client; + } + + MOCK_METHOD(litebus::Future, LaunchRequest, (const litebus::http::Request &request), + (override)); + +private: + std::string iamAddress_; +}; + +class InternalIAMTest : public ::testing::Test { +public: + [[maybe_unused]] static void SetUpTestSuite() + { + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + testIamBasePath_ = "http://127.0.0.1:" + std::to_string(port); + } + + void SetUp() override + { + auto applePath = RESOURCE_DIR + "/" + RDO + "/" + ROOT_KEY_VERSION + "/" + APPLE; + auto boyPath = RESOURCE_DIR + "/" + RDO + "/" + ROOT_KEY_VERSION + "/" + BOY; + auto dogPath = RESOURCE_DIR + "/" + RDO + "/" + ROOT_KEY_VERSION + "/" + DOG; + auto eggPath = RESOURCE_DIR + "/" + RDO + "/" + ROOT_KEY_VERSION + "/" + EGG; + if (FileExists(applePath) && FileExists(boyPath) && FileExists(dogPath) && FileExists(eggPath)) { + isResourceExisted_ = true; + } + if (!isResourceExisted_) { + EXPECT_TRUE(litebus::os::Mkdir(applePath).IsNone()); + TouchFile(litebus::os::Join(applePath, A_TXT)); + EXPECT_TRUE(Write(litebus::os::Join(applePath, A_TXT), A_TXT_CONTENT)); + + EXPECT_TRUE(litebus::os::Mkdir(boyPath).IsNone()); + TouchFile(litebus::os::Join(boyPath, B_TXT)); + EXPECT_TRUE(Write(litebus::os::Join(boyPath, B_TXT), B_TXT_CONTENT)); + + EXPECT_TRUE(litebus::os::Mkdir(dogPath).IsNone()); + TouchFile(litebus::os::Join(dogPath, D_TXT)); + EXPECT_TRUE(Write(litebus::os::Join(dogPath, D_TXT), D_TXT_CONTENT)); + + EXPECT_TRUE(litebus::os::Mkdir(eggPath).IsNone()); + TouchFile(litebus::os::Join(eggPath, E_TXT)); + EXPECT_TRUE(Write(litebus::os::Join(eggPath, E_TXT), E_TXT_CONTENT)); + } + + if (!FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Mkdir(DEFAULT_POLICY_PATH).IsNone()); + TouchFile(policyPath_); + } + litebus::os::SetEnv("LITEBUS_AKSK_ENABLED", "1"); + litebus::os::SetEnv("LITEBUS_ACCESS_KEY", "c2b4887bb63e71b88773503708d625d8f0662ca18e513972bfcc8ffcfc7e3ed2"); + litebus::os::SetEnv("LITEBUS_SECRET_KEY", SensitiveValue("16bf66680df37fd9d5ff6f781b50c53c463f4af0bb87c72c102aa7dde71da7cf").GetData()); + litebus::os::SetEnv(LITEBUS_DATA_KEY, SensitiveValue("0D2F1C6AEA8AFF3204128DFFD7CEC95471D1149DB628DF5267FDA5233C1D2F05").GetData()); + etcdSrvDriver_ = std::make_unique(); + int metaStoreServerPort = functionsystem::test::FindAvailablePort(); + metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); + etcdSrvDriver_->StartServer(metaStoreServerHost_); + metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); + metaStorageAccessor_ = std::make_shared(metaStoreClient_); + auto iamClient = IAMClient::CreateIAMClient(testIamBasePath_); + internalIAM_ = std::make_shared(param_); + internalIAM_->BindMetaStorageAccessor(metaStorageAccessor_); + internalIAM_->BindIAMClient(iamClient); + EXPECT_TRUE(internalIAM_->Init()); + internalIAMCred_ = std::make_shared(credParam_); + internalIAMCred_->BindMetaStorageAccessor(metaStorageAccessor_); + internalIAMCred_->BindIAMClient(iamClient); + EXPECT_TRUE(internalIAMCred_->Init()); + } + + void TearDown() override + { + YRLOG_DEBUG("In TearDown"); + if (FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Rmdir(DEFAULT_POLICY_PATH).IsNone()); + } + metaStorageAccessor_->metaClient_ = metaStoreClient_; + internalIAM_ = nullptr; + internalIAMCred_ = nullptr; + metaStoreClient_ = nullptr; + metaStorageAccessor_ = nullptr; + etcdSrvDriver_->StopServer(); + etcdSrvDriver_ = nullptr; + litebus::os::UnSetEnv("LITEBUS_AKSK_ENABLED"); + litebus::os::UnSetEnv("LITEBUS_ACCESS_KEY"); + litebus::os::UnSetEnv("LITEBUS_SECRET_KEY"); + litebus::os::UnSetEnv(LITEBUS_DATA_KEY); + YRLOG_DEBUG("TearDown finish"); + } + + bool isResourceExisted_{ false }; + std::string policyPath_ = litebus::os::Join(DEFAULT_POLICY_PATH, POLICY_FILE_NAME); + InternalIAM::Param param_{ true, policyPath_ }; + InternalIAM::Param credParam_{ true, policyPath_, "", IAMCredType::AK_SK}; + std::shared_ptr internalIAM_; + std::shared_ptr internalIAMCred_; + std::unique_ptr etcdSrvDriver_; + std::shared_ptr metaStoreClient_; + std::shared_ptr metaStorageAccessor_; + std::string metaStoreServerHost_; + inline static std::string testIamBasePath_; +}; + +using ::testing::Return; + +/** + * Feature: InternalIAMTest--InitTest + * Description: test InternalIAM Init interface + * Steps: + * 1. test Init interface without enable iam, failed + * 2. test Init interface with sync token from metastore, check token counts in token map + */ +TEST_F(InternalIAMTest, InitTest) // NOLINT +{ + std::string initToken = "init_token"; + std::string encryptToken = EncryptToken(initToken); + std::string encryptTokenForStorage = EncryptTokenForStorage(encryptToken); + TokenSalt tokenSalt{.token = encryptTokenForStorage, .salt = ""}; + EXPECT_TRUE(metaStoreClient_->Put("/yr/iam/token/new//" + TEST_TENANT_ID, TransToJsonFromTokenSalt(tokenSalt), {}) + .Get() + ->status.IsOk()); + + auto testIAM = std::make_shared(param_); + auto iamClient = IAMClient::CreateIAMClient(testIamBasePath_); + testIAM->BindMetaStorageAccessor(metaStorageAccessor_); + testIAM->BindIAMClient(iamClient); + EXPECT_TRUE(testIAM->Init()); + sleep(1); + EXPECT_TRUE(testIAM->IsIAMEnabled()); + EXPECT_EQ(testIAM->syncTokenActor_->newTokenMap_.size(), static_cast(1)); + YRLOG_DEBUG("InitTest finish"); +} + +/** + * Feature: InternalIAMTest--RequireVerifyAbandonTokenTest + * Description: test InternalIAM VerifyToken RequireEncryptToken AbandonTokenByTenantID interface + * Steps: + * 1. RequireEncryptToken, no token cache, ret code error, failed + * 2. RequireEncryptToken, no token cache, no token key, failed + * 3. RequireEncryptToken, no token cache, success + * 4. RequireEncryptToken, in token cache, success + * 5. VerifyToken, in token cache, success + * 5. VerifyToken, no token cache, failed + * 6. AbandonToken, in token cache, success + */ +TEST_F(InternalIAMTest, RequireVerifyAbandonTokenTest) // NOLINT +{ + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAM_->BindIAMClient(testIamClient); + internalIAM_->syncTokenActor_->BindIAMClient(testIamClient); + + auto now = static_cast(std::time(nullptr)); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response2.headers["X-Expired-Time-Span"] = "aaaa"; + auto response3 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response3.headers["X-Auth"] = "encrypt_token"; + response3.headers["X-Expired-Time-Span"] = std::to_string(now - 400); + auto response4 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response4.headers["X-Auth"] = "encrypt_token"; + response4.headers["X-Expired-Time-Span"] = std::to_string(now + 400); + auto response5 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + auto response6 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + + EXPECT_CALL(*testIamClient, LaunchRequest).Times(6).WillOnce(Return(response1)).WillOnce(Return(response2)) + .WillOnce(Return(response3)).WillOnce(Return(response4)).WillOnce(Return(response5)).WillOnce(Return(response6)); + + EXPECT_TRUE(internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get()->token.empty()); + EXPECT_TRUE(internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get()->token.empty()); + auto tokenSalt = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_STREQ(tokenSalt->token.c_str(), "encrypt_token"); + EXPECT_TRUE(tokenSalt->expiredTimeStamp < now); + + auto tokenSalt1 = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_STREQ(tokenSalt1->token.c_str(), "encrypt_token"); + EXPECT_TRUE(tokenSalt1->expiredTimeStamp > now); + EXPECT_EQ(internalIAM_->syncTokenActor_->newTokenMap_.size(), static_cast(1)); + + EXPECT_TRUE(internalIAM_->VerifyToken("encrypt_token").Get().IsOk()); + EXPECT_FALSE(internalIAM_->VerifyToken("err_token").Get().IsOk()); + + EXPECT_TRUE(internalIAM_->AbandonTokenByTenantID(TEST_TENANT_ID).Get().IsOk()); + EXPECT_FALSE(internalIAM_->VerifyToken("encrypt_token").Get().IsOk()); +} + +TEST_F(InternalIAMTest, RequireTokenWhenGoingToExpiredTest) +{ + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAM_->BindIAMClient(testIamClient); + internalIAM_->syncTokenActor_->BindIAMClient(testIamClient); + auto now = static_cast(std::time(nullptr)); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response1.headers["X-Auth"] = "encrypt_token1"; + response1.headers["X-Expired-Time-Span"] = std::to_string(now + 15); + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response2.headers["X-Auth"] = "encrypt_token2"; + response2.headers["X-Expired-Time-Span"] = std::to_string(now + 400); + + EXPECT_CALL(*testIamClient, LaunchRequest).Times(2).WillOnce(Return(response1)).WillOnce(Return(response2)); + auto tokenSalt1 = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_STREQ(tokenSalt1->token.c_str(), "encrypt_token1"); + auto tokenSalt2 = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_STREQ(tokenSalt2->token.c_str(), "encrypt_token2"); +} + +/** + * Feature: InternalIAMTest--UpdatePolicyTest + * Description: test AuthorizeProxy UpdatePolicy interface + * Steps: + * 1. call UpdatePolicy with no policy file exists, success + * 2. call UpdatePolicy with invalid json of policy file, failed + * 3. call UpdatePolicy with valid format of policy file, success + **/ +TEST_F(InternalIAMTest, UpdatePolicyTest) +{ + if (FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Rmdir(DEFAULT_POLICY_PATH).IsNone()); + } + AuthorizeProxy authorizeProxy(policyPath_); + EXPECT_FALSE(authorizeProxy.UpdatePolicy().IsOk()); + + EXPECT_TRUE(litebus::os::Mkdir(DEFAULT_POLICY_PATH).IsNone()); + TouchFile(policyPath_); + std::string emptyStr = R"()"; + EXPECT_TRUE(Write(policyPath_, emptyStr)); + EXPECT_FALSE(authorizeProxy.UpdatePolicy().IsOk()); + + std::string invalidPolicyStr = R"({"invalidPolicyStr##"})"; + EXPECT_TRUE(Write(policyPath_, invalidPolicyStr)); + EXPECT_FALSE(authorizeProxy.UpdatePolicy().IsOk()); + + if (FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Rm(policyPath_).IsNone()); + } + TouchFile(policyPath_); + std::string validPolicyStr = R"({"tenant_group":{"system":{"tenant0":["func1"]}}, "white_list": {}, "policy":{"allow":{"invoke":{"system":{"system": ["*"]}}, "create":{}, "kill":{}}, "deny":{}}})"; + EXPECT_TRUE(Write(policyPath_, validPolicyStr)); + EXPECT_TRUE(authorizeProxy.UpdatePolicy().IsOk()); + if (FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Rmdir(DEFAULT_POLICY_PATH).IsNone()); + } +} + +/** +* Feature InternalIAMTest--UpdatePolicyContentTest +* Description: test policyContent UpdatePolicyContent interface +* Steps: +* 1. call UpdatePolicy with empty policy string, success +* 2. call UpdatePolicy with invalid json of policy string, failed +* 3. call UpdatePolicy with valid json but invalid format of policy string, failed +* 4. call UpdatePolicy with valid json and valid format of policy string, success +**/ +TEST_F(InternalIAMTest, UpdatePolicyContentTest) +{ + PolicyContent policyContent; + // empty policy string + std::string policyStr1 = R"()"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr1).IsOk()); + // invalid json string + std::string policyStr2 = R"({"invalidPolicyStr##"})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr2).IsOk()); + // valid json string, invalid key + std::string policyStr3 = R"({"group##":{}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr3).IsOk()); + + std::string policyStr4 = R"({"tenant_group":{}, "white_list##":{}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr4).IsOk()); + + std::string policyStr5 = R"({"tenant_group":{}, "white_list":{}, "policy##":{}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr5).IsOk()); + + std::string policyStr6 = R"({"tenant_group":{}, "white_list":{}, "policy":{"allow##":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr6).IsOk()); + + std::string policyStr7 = R"({"tenant_group":{}, "white_list":{}, "policy":{"allow":{}, "deny##":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr7).IsOk()); + + std::string policyStr8 = R"({"tenant_group":{}, "white_list":{"": []}, "policy":{"allow":{}, "deny":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr8).IsOk()); + + std::string policyStr9 = R"({"tenant_group":{}, "white_list":{}, "policy":{"allow":{"":{}}, "deny":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr9).IsOk()); + + std::string policyStr10 = R"({"tenant_group":{}, "white_list":{"func1": []}, "policy":{"allow":{"invalid_method":{}}, "deny":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr10).IsOk()); + + std::string policyStr11 = R"({"tenant_group":{}, "white_list":{"func1": []}, "policy":{"allow":{"invoke":{"invalid_group":{"system": ["*"]}}}, "deny":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr11).IsOk()); + + std::string policyStr12 = R"({"tenant_group":{"system":{"tenant0":["func1"]}}, "white_list":{"func1": []}, "policy":{"allow":{"invoke":{"system":{"invalid_group": ["*"]}}}, "deny":{}}})"; + EXPECT_FALSE(policyContent.UpdatePolicyContent(policyStr12).IsOk()); + + // valid json string + std::string policyStr13 = R"({"tenant_group":{"system":{"tenant0":["func1"]}}, "white_list":{"func1":["tenant0"], "func2": ["tenant0"]}, "policy":{"allow":{"invoke":{"system":{"system": ["*"]}}, "create":{}, "kill":{}}, "deny":{}}})"; + EXPECT_TRUE(policyContent.UpdatePolicyContent(policyStr13).IsOk()); + std::string policyStr14 = R"({"tenant_group":{"system":{"tenant0":["func1"]}}, "white_list": {"func1":[], "func2":[""]}, "policy":{"allow":{"invoke":{"system":{"system": ["*"]}}, "create":{"system": {"system": ["func1"]}}, "kill":{"system": {"system": [""]}}}, "deny":{"tenant_list":["TEST_TENANT_ID"]}}})"; + EXPECT_TRUE(policyContent.UpdatePolicyContent(policyStr14).IsOk()); + std::string policyStr15 = R"({"tenant_group":{"system":{}, "external": {}}, "white_list": {"func1":[], "func2":[""]}, "policy":{"allow":{"invoke":{"system":{"system": ["*"]}, "external":{"external":["="]}}, "create":{"system": {"system": []}}, "kill":{"system": {"system": [""]}}}, "deny":{"tenant_list":["TEST_TENANT_ID"]}}})"; + Status status = policyContent.UpdatePolicyContent(policyStr15); + EXPECT_TRUE(status.IsOk()); +} + +/** + * Feature InternalIAMTest--AuthorizeTest + * Description: test AuthorizeProxy Authorize interface + * Steps: + * 1. satisfy allow rule, success + * 2. satisfy deny rule, failed + */ +TEST_F(InternalIAMTest, AuthorizeTest) +{ + if (FileExists(policyPath_)) { + EXPECT_TRUE(litebus::os::Rmdir(DEFAULT_POLICY_PATH).IsNone()); + } + AuthorizeProxy authorizeProxy(policyPath_); + EXPECT_TRUE(litebus::os::Mkdir(DEFAULT_POLICY_PATH).IsNone()); + TouchFile(policyPath_); + std::string policyStr = R"({"tenant_group":{"system":{"tenant0":["func1"]}}, "white_list":{"func1":["tenant0"], "func2": ["tenant0"]}, "policy":{"allow":{"invoke":{"system":{"system": ["*"]}}, "create":{}, "kill":{}}, "deny":{}}})"; + EXPECT_TRUE(Write(policyPath_, policyStr)); + EXPECT_TRUE(authorizeProxy.UpdatePolicy().IsOk()); + + AuthorizeParam authorizeParam1{ + .callerTenantID = "1", + .calleeTenantID = "1", + .callMethod = "invoke", + .funcName = "func1" + }; + EXPECT_FALSE(authorizeProxy.Authorize(authorizeParam1).IsOk()); + authorizeParam1 = { + .callerTenantID = "tenant0", + .calleeTenantID = "tenant0", + .callMethod = "invoke", + .funcName = "func1" + }; + EXPECT_TRUE(authorizeProxy.Authorize(authorizeParam1).IsOk()); +} + +/** +* Feature InternalIAMTest--HandleAuthorizeRequestTest +* Description: test PolicyHandler HandleAuthorizeRequest interface +* Steps: +* 1. empty policy, success +* 2. deny rules satisfied, failed +* 3. deny rules not satisfied, allow rules not satisfied, failed +* 4. deny rules not satisfied, allow rules satisfied, success +**/ +TEST_F(InternalIAMTest, HandleAuthorizeRequestTest) +{ + PolicyHandler policyHandler; + auto policyContent = std::make_shared(); + AuthorizeParam authorizeParam1{ + .callerTenantID = TEST_TENANT_ID, + .calleeTenantID = TEST_TENANT_ID, + .callMethod = "invoke", + .funcName = "func1" + }; + // empty policy + std::string policyStr1 = R"()"; + EXPECT_FALSE(policyContent->UpdatePolicyContent(policyStr1).IsOk()); + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam1, policyContent).IsOk()); + + std::string policyStr2 = R"({ + "tenant_group": { + "system": { + "0": ["func0"] + }, + "external": {} + }, + "white_list": { + "func0": ["1"] + }, + "policy": { + "allow": { + "invoke": { + "system": { + "system": ["*"], + "external": ["*"] + }, + "external": { + "external": ["white_list","=","func1"] + } + }, + "create": { + "system": { + "system": ["*"], + "external": ["*"] + } + }, + "kill": { + "system": { + "system": ["*"], + "external": ["*"] + }, + "external": { + "external": ["white_list","func1"] + } + } + }, + "deny": { + "tenant_list": ["4"] + } + } + })"; + EXPECT_TRUE(policyContent->UpdatePolicyContent(policyStr2).IsOk()); + + AuthorizeParam authorizeParam5; + + authorizeParam5 = { + .callerTenantID = "", + .calleeTenantID = "0", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "0", + .calleeTenantID = "", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "0", + .calleeTenantID = "0", + .callMethod = "", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "0", + .calleeTenantID = "0", + .callMethod = "invoke", + .funcName = "" + }; + EXPECT_TRUE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "0", + .calleeTenantID = "0", + .callMethod = "invalid_method", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "4", + .calleeTenantID = "0", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "0", + .calleeTenantID = "0", + .callMethod = "create", + .funcName = "func0" + }; + EXPECT_TRUE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + + authorizeParam5 = { + .callerTenantID = "1", + .calleeTenantID = "0", + .callMethod = "create", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "1", + .calleeTenantID = "0", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "2", + .calleeTenantID = "1", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "1", + .calleeTenantID = "1", + .callMethod = "invoke", + .funcName = "func0" + }; + EXPECT_TRUE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "2", + .calleeTenantID = "2", + .callMethod = "invoke", + .funcName = "func2" + }; + EXPECT_TRUE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "2", + .calleeTenantID = "3", + .callMethod = "invoke", + .funcName = "func1" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "2", + .calleeTenantID = "3", + .callMethod = "kill", + .funcName = "func1" + }; + EXPECT_TRUE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); + authorizeParam5 = { + .callerTenantID = "2", + .calleeTenantID = "3", + .callMethod = "kill", + .funcName = "func2" + }; + EXPECT_FALSE(policyHandler.HandleAuthorizeRequest(authorizeParam5, policyContent).IsOk()); +} + +/** + * Feature: InternalIAMTest--VerifiedInstanceTest + * Description: test InternalIAM verifiedInstance member + * Steps: + * 1. Insert Delete instance, check verifiedInstance size + * 2. Insert Delete instance, check instance verified or not + */ +TEST_F(InternalIAMTest, VerifiedInstanceTest) +{ + EXPECT_EQ(internalIAM_->GetVerifiedInstanceSize(), static_cast(0)); + internalIAM_->Insert(TEST_INSTANCE_ID_1); + internalIAM_->Insert(TEST_INSTANCE_ID_2); + EXPECT_EQ(internalIAM_->GetVerifiedInstanceSize(), static_cast(2)); + EXPECT_TRUE(internalIAM_->VerifyInstance(TEST_INSTANCE_ID_1)); + EXPECT_TRUE(internalIAM_->VerifyInstance(TEST_INSTANCE_ID_2)); + internalIAM_->Delete(TEST_INSTANCE_ID_1); + internalIAM_->Delete(TEST_INSTANCE_ID_2); + EXPECT_EQ(internalIAM_->GetVerifiedInstanceSize(), static_cast(0)); + EXPECT_FALSE(internalIAM_->VerifyInstance(TEST_INSTANCE_ID_1)); + EXPECT_FALSE(internalIAM_->VerifyInstance(TEST_INSTANCE_ID_2)); +} + +/** + * Feature: InternalIAMTest--SyncTokenTest + * Description: test SyncPutToken interface + * 1. no token in cache, verify token failed + * 2. put token into metastore, verify token success + * 3. delete token from metastore, verify token failed + */ +TEST_F(InternalIAMTest, SyncTokenTest) +{ + metaStoreClient_->Delete("/yr/iam/token/", { false, true }).Get(); + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAM_->BindIAMClient(testIamClient); + internalIAM_->syncTokenActor_->BindIAMClient(testIamClient); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + + EXPECT_CALL(*testIamClient, LaunchRequest).Times(1).WillOnce(Return(response1)); + + EXPECT_FALSE(internalIAM_->VerifyToken("encrypt_token").Get().IsOk()); + + std::string initToken = "test_token"; + std::string encryptToken = EncryptToken(initToken); + YRLOG_DEBUG("encryptToken is: {}", encryptToken); + std::string encryptTokenForStorage = EncryptTokenForStorage(encryptToken); + + TokenSalt tokenSalt{.token = encryptTokenForStorage, .salt = ""}; + EXPECT_TRUE(metaStoreClient_->Put("/yr/iam/token/new//" + TEST_TENANT_ID, TransToJsonFromTokenSalt(tokenSalt), {}) + .Get() + ->status.IsOk()); + EXPECT_AWAIT_TRUE([&]() -> bool { return internalIAM_->syncTokenActor_->newTokenMap_.size() == 1;}); + EXPECT_EQ(internalIAM_->syncTokenActor_->newTokenMap_.size(), static_cast(1)); + auto tokenSaltPtr = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_TRUE(internalIAM_->VerifyToken(tokenSaltPtr->token).Get().IsOk()); + metaStoreClient_->Delete("/yr/iam/token/", { false, true }).Get(); + ASSERT_AWAIT_TRUE([&]() -> bool { return internalIAM_->syncTokenActor_->newTokenMap_.size() == 0;}); +} + +TEST_F(InternalIAMTest, IAMTokenSyncerTest) +{ + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAM_->BindIAMClient(testIamClient); + internalIAM_->syncTokenActor_->BindIAMClient(testIamClient); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + + EXPECT_CALL(*testIamClient, LaunchRequest).Times(1).WillOnce(Return(response1)); + + EXPECT_FALSE(internalIAM_->VerifyToken("encrypt_token").Get().IsOk()); + + std::string initToken = "test_token"; + std::string encryptToken = EncryptToken(initToken); + YRLOG_DEBUG("encryptToken is: {}", encryptToken); + std::string encryptTokenForStorage = EncryptTokenForStorage(encryptToken); + + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + + { + std::shared_ptr rep = std::make_shared(); + rep->status = Status(StatusCode::FAILED, ""); + auto future = internalIAM_->syncTokenActor_->IAMTokenSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_FALSE(future.Get().status.IsOk()); + } + + { + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + auto future = internalIAM_->syncTokenActor_->IAMTokenSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + } + + { + TokenSalt tokenSalt{.token = encryptTokenForStorage, .salt = ""}; + auto iamKey = "/yr/iam/token/new//" + TEST_TENANT_ID; + auto iamValue = TransToJsonFromTokenSalt(tokenSalt); + + KeyValue getKeyValue; + getKeyValue.set_key(iamKey) ; + getKeyValue.set_value(iamValue); + + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + rep->kvs.emplace_back(getKeyValue); + + EXPECT_EQ(internalIAM_->syncTokenActor_->newTokenMap_.size(), static_cast(0)); + + auto future = internalIAM_->syncTokenActor_->IAMTokenSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + + EXPECT_EQ(internalIAM_->syncTokenActor_->newTokenMap_.size(), static_cast(1)); + auto tokenSaltPtr = internalIAM_->RequireEncryptToken(TEST_TENANT_ID).Get(); + EXPECT_TRUE(internalIAM_->VerifyToken(tokenSaltPtr->token).Get().IsOk()); + } +} + +TEST_F(InternalIAMTest, CheckTokenExpiredInAdvanceTest) +{ + std::string initToken = "test_token"; + std::string encryptToken = EncryptToken(initToken); + auto tokenSalt = std::make_shared(); + tokenSalt->token = encryptToken; + tokenSalt->salt = "salt"; + tokenSalt->expiredTimeStamp = static_cast(std::time(nullptr)) + 120; + + auto tokenSalt1 = std::make_shared(); + tokenSalt1->token = encryptToken; + tokenSalt1->salt = "salt"; + auto expire1 = static_cast(std::time(nullptr)) + 300; + tokenSalt1->expiredTimeStamp = expire1; + + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAM_->BindIAMClient(testIamClient); + internalIAM_->syncTokenActor_->BindIAMClient(testIamClient); + auto response0 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response1.headers["X-Auth"] = "encrypt_token"; + auto newExpired = static_cast(std::time(nullptr)) + 600; + response1.headers["X-Expired-Time-Span"] = std::to_string(newExpired); + response1.headers["X-Salt"] = "salt1"; + EXPECT_CALL(*testIamClient, LaunchRequest).Times(2).WillOnce(Return(response0)).WillOnce(Return(response1)); + internalIAM_->syncTokenActor_->newTokenMap_["tenant001"] = tokenSalt; + internalIAM_->syncTokenActor_->newTokenMap_["tenant002"] = tokenSalt1; + internalIAM_->syncTokenActor_->CheckTokenExpiredInAdvance(); + internalIAM_->syncTokenActor_->CheckTokenExpiredInAdvance(); + ASSERT_AWAIT_TRUE( + [&]() -> bool { return internalIAM_->syncTokenActor_->newTokenMap_["tenant001"]->expiredTimeStamp == newExpired;}); + EXPECT_TRUE( internalIAM_->syncTokenActor_->newTokenMap_["tenant002"]->expiredTimeStamp == expire1); +} + +TEST_F(InternalIAMTest, InitWithCredentialTest) // NOLINT +{ + auto credential = GenAkSkContent("tenant1", "access1", "secret1", "data1"); + auto encCredential = EncryptAKSKContentForStorage(credential); + EXPECT_TRUE(metaStoreClient_->Put("/yr/iam/aksk/new//tenant1", TransToJsonFromEncAKSKContent(encCredential), {}) + .Get() + ->status.IsOk()); + + auto testIAM = std::make_shared(credParam_); + auto iamClient = IAMClient::CreateIAMClient(testIamBasePath_); + testIAM->BindMetaStorageAccessor(metaStorageAccessor_); + testIAM->BindIAMClient(iamClient); + EXPECT_TRUE(testIAM->Init()); + ASSERT_AWAIT_TRUE( + [&]() -> bool { return testIAM->credentialActor_->newCredMap_.size() == 2;}); + EXPECT_TRUE(testIAM->IsIAMEnabled()); + YRLOG_DEBUG("InitTest finish"); +} + +TEST_F(InternalIAMTest, AbandonToken) // NOLINT +{ + auto client = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + auto response = litebus::http::Response{ litebus::http::ResponseCode::OK }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + auto future = client->AbandonToken("x-tenant-id"); + EXPECT_AWAIT_READY(future); + EXPECT_TRUE(future.Get().IsOk()); +} + +TEST_F(InternalIAMTest, RequireCredential_Unreachable) // NOLINT +{ + auto client = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + client->useAKSK_ = true; + + litebus::Status response{ litebus::http::HttpErrorCode::CONNECTION_REFUSED }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + + auto credential = client->RequireEncryptCredential("x-tenant-id").Get(); + EXPECT_EQ(credential->status.StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); +} + +TEST_F(InternalIAMTest, AbandonCredential) // NOLINT +{ + auto client = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + { + auto response = litebus::http::Response{ litebus::http::ResponseCode::OK }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + auto future = client->AbandonCredential("x-tenant-id"); + EXPECT_AWAIT_READY(future); + EXPECT_TRUE(future.Get().IsOk()); + } + + { + auto response = litebus::http::Response{ litebus::http::ResponseCode(500) }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + auto future = client->AbandonCredential("x-tenant-id"); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get().StatusCode(), ERR_INNER_SYSTEM_ERROR); + } + + { + auto response = litebus::http::Response{ litebus::http::ResponseCode(110) }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + auto future = client->AbandonCredential("x-tenant-id"); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get().StatusCode(), ERR_INNER_SYSTEM_ERROR); + } + + { + auto response = litebus::http::Response{ litebus::http::ResponseCode(400) }; + EXPECT_CALL(*client, LaunchRequest).WillOnce(Return(response)); + auto future = client->AbandonCredential("x-tenant-id"); + EXPECT_AWAIT_READY(future); + EXPECT_EQ(future.Get().StatusCode(), FAILED); + } +} + +TEST_F(InternalIAMTest, RequireVerifyAbandonCredentialWithCredentialTest) // NOLINT +{ + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAMCred_->BindIAMClient(testIamClient); + internalIAMCred_->credentialActor_->BindIAMClient(testIamClient); + internalIAMCred_->credentialActor_->requestRetryInterval_ = 1; + internalIAMCred_->credentialActor_->requestMaxRetryTimes_ = 1; + auto persistentCounter = std::make_shared>(0); + internalIAMCred_->RegisterUpdateCredentialCallback([cnt(persistentCounter)](const std::string &tenantID, const std::shared_ptr &content){ + (*cnt)++; + }); + + { + // require credential err response + litebus::Promise promise1; + promise1.SetFailed(-1); + litebus::Promise promise2; + promise2.SetFailed(-1); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::INTERNAL_SERVER_ERROR }; + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + EXPECT_CALL(*testIamClient, LaunchRequest).Times(5).WillOnce(Return(promise1.GetFuture())).WillOnce(Return(promise2.GetFuture())) + .WillOnce(Return(response1)).WillOnce(Return(response1)) + .WillOnce(Return(response2)); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::FAILED); + } + { + // verify credential err response + litebus::Promise promise1; + promise1.SetFailed(-1); + litebus::Promise promise2; + promise2.SetFailed(-1); + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::INTERNAL_SERVER_ERROR }; + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + EXPECT_CALL(*testIamClient, LaunchRequest).Times(5).WillOnce(Return(promise1.GetFuture())).WillOnce(Return(promise2.GetFuture())) + .WillOnce(Return(response1)).WillOnce(Return(response1)) + .WillOnce(Return(response2)); + EXPECT_EQ(internalIAMCred_->RequireCredentialByAK("ak1").Get()->status.StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(internalIAMCred_->RequireCredentialByAK("ak1").Get()->status.StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(internalIAMCred_->RequireCredentialByAK("ak1").Get()->status.StatusCode(), StatusCode::FAILED); + } + { + // request but body is error + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response1.body = R"({)"; + // expireStamp is error + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response2.body = R"({"tenantID":"tenant1", "accessKey":"ak", "secretKey":"sk", "dataKey":"", "expiredTimeStamp": "aaa" })"; + // expired + auto credential = GenAkSkContent("tenant1", "access1", "secret1", "data1"); + auto encCredential = EncryptAKSKContentForStorage(credential); + encCredential->expiredTimeStamp = static_cast(std::time(nullptr)) - 10; + auto response5 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response5.body = TransToJsonFromEncAKSKContent(encCredential); + + EXPECT_CALL(*testIamClient, LaunchRequest).Times(3).WillOnce(Return(response1)).WillOnce(Return(response2)).WillOnce(Return(response5)); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::PARAMETER_ERROR); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::PARAMETER_ERROR); + + EXPECT_TRUE(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.ToString().find("aksk expired time stamp is earlier") != std::string::npos); + } + { + // require success + auto credential = GenAkSkContent("tenant1", "access1", "secret1", "data1"); + auto encCredential = EncryptAKSKContentForStorage(credential); + auto response = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response.body = TransToJsonFromEncAKSKContent(encCredential); + EXPECT_CALL(*testIamClient, LaunchRequest).Times(1).WillOnce(Return(response)); + auto akSk = internalIAMCred_->RequireCredentialByTenantID("tenant1").Get(); + EXPECT_TRUE(akSk->status.IsOk()); + EXPECT_EQ(akSk->tenantID, "tenant1"); + EXPECT_EQ(akSk->accessKey, "access1"); + EXPECT_EQ(std::string(akSk->secretKey.GetData()), "secret1"); + EXPECT_EQ(std::string(akSk->dataKey.GetData()), "data1"); + EXPECT_EQ(*persistentCounter, 0); + // return from local + auto akSk1 = internalIAMCred_->RequireCredentialByTenantID("tenant1").Get(); + EXPECT_EQ(std::string(akSk1->secretKey.GetData()), "secret1"); + // request by ak + auto akSk2 = internalIAMCred_->RequireCredentialByAK("access1").Get(); + EXPECT_EQ(std::string(akSk2->secretKey.GetData()), "secret1"); + + auto isSystemTenant = internalIAMCred_->IsSystemTenant("tenant1"); + ASSERT_AWAIT_READY(isSystemTenant); + EXPECT_FALSE(isSystemTenant.Get()); + + auto status = internalIAMCred_->AbandonCredentialByTenantID("tenant1").Get(); + EXPECT_TRUE(status.IsOk()); + EXPECT_TRUE(internalIAMCred_->credentialActor_->newCredMap_.find("tenant1") + == internalIAMCred_->credentialActor_->newCredMap_.end()); + EXPECT_TRUE(internalIAMCred_->credentialActor_->accessKey2TenantIDMap_.find("access1") + == internalIAMCred_->credentialActor_->accessKey2TenantIDMap_.end()); + } + { + // verify success + auto credential = GenAkSkContent("tenant1", "access1", "secret1", "data1"); + auto encCredential = EncryptAKSKContentForStorage(credential); + auto response = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response.body = TransToJsonFromEncAKSKContent(encCredential); + EXPECT_CALL(*testIamClient, LaunchRequest).Times(1).WillOnce(Return(response)); + auto akSk = internalIAMCred_->RequireCredentialByAK("access1").Get(); + EXPECT_TRUE(akSk->status.IsOk()); + EXPECT_EQ(akSk->tenantID, "tenant1"); + EXPECT_EQ(akSk->accessKey, "access1"); + EXPECT_EQ(std::string(akSk->secretKey.GetData()), "secret1"); + EXPECT_EQ(std::string(akSk->dataKey.GetData()), "data1"); + } + { + // require credential, and going to be expired, need to require new + auto credential1 = GenAkSkContent("tenant11", "access1", "secret1", "data1"); + auto encCredential1 = EncryptAKSKContentForStorage(credential1); + encCredential1->expiredTimeStamp = static_cast(std::time(nullptr)) + 20; + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response1.body = TransToJsonFromEncAKSKContent(encCredential1); + + auto credential2 = GenAkSkContent("tenant11", "access2", "secret1", "data1"); + auto encCredential2 = EncryptAKSKContentForStorage(credential2); + encCredential2->expiredTimeStamp = static_cast(std::time(nullptr)) + 100; + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + response2.body = TransToJsonFromEncAKSKContent(encCredential2); + EXPECT_CALL(*testIamClient, LaunchRequest).Times(2).WillOnce(Return(response1)).WillOnce(Return(response2)); + auto akSk1 = internalIAMCred_->RequireCredentialByTenantID("tenant11").Get(); + auto akSk2 = internalIAMCred_->RequireCredentialByTenantID("tenant11").Get(); + EXPECT_TRUE(akSk1->accessKey != akSk2->accessKey); + EXPECT_EQ(*persistentCounter, 1); + } +} + +TEST_F(InternalIAMTest, SyncCredentialTest) +{ + metaStoreClient_->Delete("/yr/iam/aksk/", { false, true }).Get(); + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAMCred_->BindIAMClient(testIamClient); + internalIAMCred_->credentialActor_->BindIAMClient(testIamClient); + auto credential1 = GenAkSkContent(TEST_TENANT_ID, "access1", "secret1", "data1"); + auto encCredential1 = EncryptAKSKContentForStorage(credential1); + encCredential1->expiredTimeStamp = static_cast(std::time(nullptr)) + 100; + EXPECT_TRUE(metaStoreClient_->Put("/yr/iam/aksk/new//" + TEST_TENANT_ID, TransToJsonFromEncAKSKContent(encCredential1), {}) + .Get() + ->status.IsOk()); + ASSERT_AWAIT_TRUE([&]() -> bool { return internalIAMCred_->credentialActor_->newCredMap_.size() == 2;}); + auto akSk = internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get(); + EXPECT_EQ(std::string(akSk->secretKey.GetData()), "secret1"); + auto akSk1 = internalIAMCred_->RequireCredentialByAK("access1").Get(); + EXPECT_EQ(std::string(akSk1->secretKey.GetData()), "secret1"); + metaStoreClient_->Delete("/yr/iam/aksk/", { false, true }).Get(); + ASSERT_AWAIT_TRUE([&]() -> bool { return internalIAMCred_->credentialActor_->newCredMap_.size() == 1;}); +} + +#if 0 +TEST_F(InternalIAMTest, CheckCredentialExpiredInAdvanceTest) +{ + auto credential1 = GenAkSkContent("tenant001", "access1", "secret1", "data1"); + credential1->expiredTimeStamp = static_cast(std::time(nullptr)) + 120; + auto credential2 = GenAkSkContent("tenant002", "access2", "secret2", "data2"); + credential2->expiredTimeStamp = static_cast(std::time(nullptr)) + 300; + internalIAMCred_->credentialActor_->SaveCredentialAndTriggerUpdate("tenant001", credential1); + internalIAMCred_->credentialActor_->SaveCredentialAndTriggerUpdate("tenant002", credential2); + + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + auto response2 = litebus::http::Response{ litebus::http::ResponseCode::OK }; + auto credential11 = GenAkSkContent("tenant001", "access11", "secret11", "data11"); + credential11->expiredTimeStamp = static_cast(std::time(nullptr)) + 720; + auto encCredential11 = EncryptAKSKContentForStorage(credential11); + response2.body = TransToJsonFromEncAKSKContent(encCredential11); + + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAMCred_->BindIAMClient(testIamClient); + internalIAMCred_->credentialActor_->BindIAMClient(testIamClient); + EXPECT_CALL(*testIamClient, CallApi).Times(2).WillOnce(Return(response1)).WillOnce(Return(response2)); + internalIAMCred_->credentialActor_->CheckCredentialExpiredInAdvance(); + internalIAMCred_->credentialActor_->CheckCredentialExpiredInAdvance(); + + ASSERT_AWAIT_TRUE( + [&]() -> bool { return internalIAMCred_->credentialActor_->newCredMap_["tenant001"]->expiredTimeStamp == credential11->expiredTimeStamp;}); + EXPECT_TRUE( internalIAMCred_->credentialActor_->newCredMap_["tenant002"]->expiredTimeStamp == credential2->expiredTimeStamp); +} +#endif + +TEST_F(InternalIAMTest, IAMCredentialSyncerTest) +{ + auto response1 = litebus::http::Response{ litebus::http::ResponseCode::FORBIDDEN }; + auto testIamClient = MockIAMClient::CreateMockIAMClient(testIamBasePath_); + internalIAMCred_->BindIAMClient(testIamClient); + internalIAMCred_->credentialActor_->BindIAMClient(testIamClient); + EXPECT_CALL(*testIamClient, LaunchRequest).Times(1).WillOnce(Return(response1)); + EXPECT_EQ(internalIAMCred_->RequireCredentialByTenantID(TEST_TENANT_ID).Get()->status.StatusCode(), StatusCode::FAILED); + + auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); + metaStorageAccessor_->metaClient_ = mockMetaStoreClient; + { + std::shared_ptr rep = std::make_shared(); + rep->status = Status(StatusCode::FAILED, ""); + auto future = internalIAMCred_->credentialActor_->IAMCredentialSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_FALSE(future.Get().status.IsOk()); + } + + { + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + auto future = internalIAMCred_->credentialActor_->IAMCredentialSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + } + + { + auto credential = GenAkSkContent("tenant1", "access1", "secret1", "data1"); + auto encCredential = EncryptAKSKContentForStorage(credential); + auto iamKey = "/yr/iam/aksk/new//tenant1"; + auto iamValue = TransToJsonFromEncAKSKContent(encCredential); + + KeyValue getKeyValue; + getKeyValue.set_key(iamKey) ; + getKeyValue.set_value(iamValue); + + std::shared_ptr rep = std::make_shared(); + rep->status = Status::OK(); + rep->kvs.emplace_back(getKeyValue); + + EXPECT_EQ(internalIAMCred_->credentialActor_->newCredMap_.size(), static_cast(1)); + + auto future = internalIAMCred_->credentialActor_->IAMCredentialSyncer(rep); + ASSERT_AWAIT_READY(future); + ASSERT_TRUE(future.Get().status.IsOk()); + + EXPECT_EQ(internalIAMCred_->credentialActor_->newCredMap_.size(), static_cast(2)); + EXPECT_TRUE(internalIAMCred_->RequireCredentialByTenantID("tenant1").Get()->status.IsOk()); + } +} + +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/observer/control_plane_observer_test.cpp b/functionsystem/tests/unit/function_proxy/common/observer/control_plane_observer_test.cpp index f29f97ab6a063feed7b68fcbdb3fc2db2efda3f5..b4d4855c204d321e34cae42a51a7514ff11d5011 100644 --- a/functionsystem/tests/unit/function_proxy/common/observer/control_plane_observer_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/observer/control_plane_observer_test.cpp @@ -24,13 +24,16 @@ #include "common/etcd_service/etcd_service_driver.h" #include "meta_storage_accessor/meta_storage_accessor.h" #include "meta_store_client/meta_store_client.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/types/instance_state.h" #include "common/utils/struct_transfer.h" +#define private public #include "function_proxy/common/observer/observer_actor.h" +#undef private #include "function_proxy/common/posix_client/shared_client/shared_client_manager.h" #include "function_proxy/common/posix_client/shared_client/posix_stream_manager_proxy.h" #include "litebus.hpp" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_meta_store_client.h" #include "utils/future_test_helper.h" #include "utils/generate_info.h" @@ -45,7 +48,7 @@ protected: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -67,18 +70,24 @@ protected: metaStorageAccessor_, std::move(param)); observerActor_->BindDataInterfaceClientManager(sharedPosixClientManager); + function_proxy::InternalIAM::Param iamParam; + iamParam.isEnableIAM = true; + iamParam.credType = IAMCredType::AK_SK; + internalIAM_ = std::make_shared(iamParam); + observerActor_->BindInternalIAM(internalIAM_); + EXPECT_CALL(*internalIAM_, IsSystemTenant).WillRepeatedly(testing::Return(false)); litebus::Spawn(observerActor_); controlPlaneObserver_ = std::make_shared(observerActor_); controlPlaneObserver_->Register(); // one from meta json, three from services.yaml - ASSERT_AWAIT_TRUE([&]() -> bool { return observerActor_->funcMetaMap_.size() == 4; }); + ASSERT_AWAIT_TRUE([&]() -> bool { return observerActor_->funcMetaMap_.size() == 5; }); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { - YRLOG_INFO("TearDownTestCase......"); + YRLOG_INFO("TearDownTestSuite......"); metaStorageAccessor_->metaClient_ = metaStoreClient_; - YRLOG_INFO("TearDownTestCase......Finish"); + YRLOG_INFO("TearDownTestSuite......Finish"); litebus::Terminate(observerActor_->GetAID()); litebus::Await(observerActor_); @@ -91,6 +100,7 @@ protected: controlPlaneObserver_ = nullptr; metaStorageAccessor_ = nullptr; metaStoreClient_ = nullptr; + internalIAM_ = nullptr; etcdSrvDriver_->StopServer(); } @@ -100,6 +110,8 @@ protected: void TearDown() override { + metaStoreClient_->Delete("/yr/route", { .prefix = true }).Get(); + metaStoreClient_->Delete("/sn/ins", { .prefix = true }).Get(); YRLOG_INFO("TearDown......"); metaStorageAccessor_->metaClient_ = metaStoreClient_; YRLOG_INFO("TearDown......Finish"); @@ -112,6 +124,7 @@ protected: inline static std::shared_ptr sharedClientMgr_; inline static std::shared_ptr observerActor_; inline static std::shared_ptr metaStoreClient_; + inline static std::shared_ptr internalIAM_; }; void CheckInstanceInfo(const InstanceInfo &l, const InstanceInfo &r) @@ -293,6 +306,7 @@ TEST_F(ObserverTest, ErrMetaStorageAccessor) auto observerActor1_ = std::make_shared("err_observer", "node", metaStorageAccessor1_, std::move(param)); observerActor1_->BindDataInterfaceClientManager(sharedPosixClientManager1); + observerActor1_->BindInternalIAM(internalIAM_); litebus::Spawn(observerActor1_); std::string funcAgentID = "funcAgent"; @@ -343,7 +357,7 @@ TEST_F(ObserverTest, DriverCallBack) (*instanceInfoA.mutable_extensions())["source"] = "driver"; controlPlaneObserver_->PutInstanceEvent(instanceInfoA, false, 1); EXPECT_TRUE(driverPromise->GetFuture().Get().IsOk()); - controlPlaneObserver_->DelInstanceEvent(instanceIDA); + controlPlaneObserver_->DelInstanceEvent(instanceIDA, -1); } /** @@ -553,37 +567,37 @@ TEST_F(ObserverTest, SetUpdateFuncMetasFunc) .WillOnce(testing::Return()) .WillOnce(testing::DoAll(testing::Assign(&isFinished, true), testing::Return())); controlPlaneObserver_->SetUpdateFuncMetasFunc( - [&](bool isAdd, const std::unordered_map &funcMetas) { - mockUpdateFuncMetasFunc->UpdateFuncMetas(isAdd, funcMetas); + [&](bool isAdd, const std::unordered_map &meta) { + mockUpdateFuncMetasFunc->UpdateFuncMetas(isAdd, meta); }); ASSERT_AWAIT_TRUE([&]() { return isFinished; }); - litebus::Future isAdd; + litebus::Future added; litebus::Future> funcMetas; EXPECT_CALL(*mockUpdateFuncMetasFunc, UpdateFuncMetas) - .WillOnce(testing::DoAll(FutureArg<0>(&isAdd), FutureArg<1>(&funcMetas), testing::Return())); + .WillOnce(testing::DoAll(FutureArg<0>(&added), FutureArg<1>(&funcMetas), testing::Return())); // put function meta to meta store auto status = metaStorageAccessor_->Put(path, funcMetaJson).Get(); EXPECT_TRUE(status.IsOk()); - ASSERT_AWAIT_READY(isAdd); - EXPECT_TRUE(isAdd.Get()); + ASSERT_AWAIT_READY(added); + EXPECT_TRUE(added.Get()); ASSERT_AWAIT_READY(funcMetas); EXPECT_EQ(funcMetas.Get().size(), static_cast(1)); EXPECT_FALSE(funcMetas.Get().at(funcKey).funcMetaData.isSystemFunc); - litebus::Future isAdd2; + litebus::Future added2; litebus::Future> funcMetas2; EXPECT_CALL(*mockUpdateFuncMetasFunc, UpdateFuncMetas) - .WillOnce(testing::DoAll(FutureArg<0>(&isAdd2), FutureArg<1>(&funcMetas2), testing::Return())); + .WillOnce(testing::DoAll(FutureArg<0>(&added2), FutureArg<1>(&funcMetas2), testing::Return())); // delete function meta in meta store metaStorageAccessor_->Delete(path).Get(); status = metaStorageAccessor_->Delete(path).Get(); EXPECT_TRUE(status.IsOk()); - ASSERT_AWAIT_READY(isAdd2); - EXPECT_FALSE(isAdd2.Get()); + ASSERT_AWAIT_READY(added2); + EXPECT_FALSE(added2.Get()); ASSERT_AWAIT_READY(funcMetas2); EXPECT_EQ(funcMetas2.Get().size(), static_cast(1)); EXPECT_TRUE(funcMetas2.Get().find(funcKey) != funcMetas2.Get().end()); @@ -595,6 +609,13 @@ TEST_F(ObserverTest, SetUpdateFuncMetasFunc) TEST_F(ObserverTest, SetUpdateSysFuncMetasFunc) { + function_proxy::InternalIAM::Param iamParam; + iamParam.isEnableIAM = true; + iamParam.credType = IAMCredType::AK_SK; + auto internalIAM = std::make_shared(iamParam); + observerActor_->BindInternalIAM(internalIAM); + EXPECT_CALL(*internalIAM, IsSystemTenant).WillRepeatedly(testing::Return(true)); + std::string funcMetaJson = R"({"funcMetaData":{"layers":[{"appId":"appA","bucketId":"bucketA","objectId":"objectA","bucketUrl":"bucketUrlA","sha256":"1a2b3c"}],"name":"0-yrjava-yr-smoke","description":"","functionUrn":"sn:cn:yrk:12345678901234561234567890123456:function:0-yrjava-yr-smoke","functionVersionUrn":"sn:cn:yrk:12345678901234561234567890123456:function:0-yrjava-yr-smoke:$latest","codeSize":22029378,"codeSha256":"1211a06","handler":"fusion_computation_handler.fusion_computation_handler","runtime":"java1.8","timeout":900,"tenantId":"0","hookHandler":{"call":"com.actorTaskCallHandler"}},"codeMetaData":{"storage_type":"s3","appId":"61022","bucketId":"bucket-test-log1","objectId":"yr-smoke-1667888605803","bucketUrl":"http://bucket-test-log1.hwcloudtest.cn:18085"},"envMetaData":{"envKey":"1d34ef","environment":"e819e3","encrypted_user_data":""},"resourceMetaData":{"cpu":500,"memory":500,"customResources":""}})"; std::string path = @@ -625,6 +646,7 @@ TEST_F(ObserverTest, SetUpdateSysFuncMetasFunc) EXPECT_TRUE(isAdd.Get()); ASSERT_AWAIT_READY(funcMetas); EXPECT_EQ(funcMetas.Get().size(), static_cast(1)); + EXPECT_TRUE(funcMetas.Get().at(funcKey).funcMetaData.isSystemFunc); litebus::Future isAdd2; litebus::Future> funcMetas2; @@ -642,6 +664,7 @@ TEST_F(ObserverTest, SetUpdateSysFuncMetasFunc) EXPECT_TRUE(funcMetas2.Get().find(funcKey) != funcMetas2.Get().end()); controlPlaneObserver_->SetUpdateFuncMetasFunc(nullptr); + observerActor_->BindInternalIAM(internalIAM_); } TEST_F(ObserverTest, GetLocalSchedulerAID) @@ -664,6 +687,26 @@ TEST_F(ObserverTest, GetLocalSchedulerAID) ASSERT_TRUE(AIDOption.IsNone()); } +TEST_F(ObserverTest, GetLocalSchedulerAIDWithoutCache) +{ + auto future = controlPlaneObserver_->GetLocalSchedulerAID("proxyID_A"); + EXPECT_TRUE(future.Get().IsNone()); + std::string proxyID = "proxyID_B"; + auto key = BUSPROXY_PATH_PREFIX + "/0/node/" + proxyID; + auto jsonStr = R"({"node":")" + proxyID + R"(","aid":")" + proxyID + R"("})"; + metaStorageAccessor_->Put(key, jsonStr).Get(); + ASSERT_AWAIT_TRUE([&]() -> bool { return observerActor_->localSchedulerView_->Get("proxyID_B") != nullptr; }); + future = controlPlaneObserver_->GetLocalSchedulerAID("proxyID_B"); + EXPECT_TRUE(future.Get().IsSome()); + // delete cache + auto proxyDelRsp = GetProxyEventRsp(EVENT_TYPE_DELETE, proxyID); + // get it from metastore + litebus::Async(observerActor_->GetAID(), &function_proxy::ObserverActor::UpdateProxyEvent, proxyDelRsp); + ASSERT_AWAIT_TRUE([&]() -> bool { return observerActor_->localSchedulerView_->Get("proxyID_B") == nullptr; }); + auto future1 = controlPlaneObserver_->GetLocalSchedulerAID("proxyID_B"); + EXPECT_TRUE(future1.Get().IsSome()); +} + class TestTenantListener : public TenantListener { public: TestTenantListener() : updateCount(0), deleteCount(0) @@ -754,39 +797,33 @@ TEST_F(ObserverTest, FailedOrEmptySyncerTest) auto mockMetaStoreClient = std::make_shared(metaStoreServerHost_); metaStorageAccessor_->metaClient_ = mockMetaStoreClient; { - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status(StatusCode::FAILED, ""); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillRepeatedly(testing::Return(getResponseFuture)); - auto future = observerActor_->FunctionMetaSyncer(); + auto future = observerActor_->FunctionMetaSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); - future = observerActor_->InstanceInfoSyncer(); + future = observerActor_->InstanceInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); - future = observerActor_->BusProxySyncer(); + future = observerActor_->BusProxySyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); } { - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillRepeatedly(testing::Return(getResponseFuture)); - auto future = observerActor_->FunctionMetaSyncer(); + auto future = observerActor_->FunctionMetaSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); - future = observerActor_->InstanceInfoSyncer(); + future = observerActor_->InstanceInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); - future = observerActor_->BusProxySyncer(); + future = observerActor_->BusProxySyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } @@ -805,14 +842,11 @@ TEST_F(ObserverTest, BusProxySyncerTest) getKeyValue.set_key(key) ; getKeyValue.set_value(value); - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); rep->kvs.emplace_back(getKeyValue); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = observerActor_->BusProxySyncer(); + auto future = observerActor_->BusProxySyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); @@ -858,12 +892,9 @@ TEST_F(ObserverTest, FunctionMetaSyncerTest) getKeyValue.set_key(key) ; getKeyValue.set_value(meta); - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); rep->kvs.emplace_back(getKeyValue); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); auto counter = std::make_shared>(0); observerActor_->SetUpdateFuncMetasFunc( [cnt(counter)](bool isAdd, const std::unordered_map &funcMetas) { @@ -871,18 +902,19 @@ TEST_F(ObserverTest, FunctionMetaSyncerTest) cnt->fetch_add(funcMetas.size()); } }); - auto future = observerActor_->FunctionMetaSyncer(); + + auto future = observerActor_->FunctionMetaSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); auto funcKey = GetFuncKeyFromFuncMetaPath(key); - EXPECT_TRUE(observerActor_->funcMetaMap_.count(funcKey) == 1); - EXPECT_TRUE(observerActor_->funcMetaMap_.count(funcKey2.Get()) == 0); - EXPECT_TRUE(observerActor_->funcMetaMap_.count(funcKey3.Get()) == 0); - EXPECT_TRUE(observerActor_->funcMetaMap_.count("deleteKey") == 0); // not local cache key, need to delete - EXPECT_TRUE(observerActor_->localFuncMetaSet_.count(funcKey1.Get()) == 1); - EXPECT_TRUE(observerActor_->systemFuncMetaMap_.count(funcKey3.Get()) == 0); - EXPECT_EQ(*counter, 3); + ASSERT_AWAIT_TRUE([&]() { return observerActor_->funcMetaMap_.count(funcKey) == 1; }); + EXPECT_EQ(observerActor_->funcMetaMap_.count(funcKey), 1); + EXPECT_EQ(observerActor_->funcMetaMap_.count(funcKey2.Get()), 0); + EXPECT_EQ(observerActor_->funcMetaMap_.count(funcKey3.Get()), 0); + EXPECT_EQ(observerActor_->funcMetaMap_.count("deleteKey"), 0); // not local cache key, need to delete + EXPECT_EQ(observerActor_->localFuncMetaSet_.count(funcKey1.Get()), 1); + EXPECT_EQ(observerActor_->systemFuncMetaMap_.count(funcKey3.Get()), 0); observerActor_->SetUpdateFuncMetasFunc(nullptr); } @@ -948,15 +980,11 @@ TEST_F(ObserverTest, InstanceInfoSyncerTest) EXPECT_TRUE(observerActor_->instanceInfoMap_.count("InstanceID4") != 0); // get key1(status 3), key2, key4 in etcd - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->header.revision = 4; rep->status = Status::OK(); rep->kvs.emplace_back(events[1].kv); rep->kvs.emplace_back(events[2].kv); - getResponseFuture.SetValue(rep); - - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); std::string cbFuncInstanceID; controlPlaneObserver_->SetInstanceInfoSyncerCbFunc([&cbFuncInstanceID](const resource_view::RouteInfo &routeInfo) { YRLOG_DEBUG("{}|{}execute instance info sync callback function, create client for instance({}), job({})", @@ -964,7 +992,7 @@ TEST_F(ObserverTest, InstanceInfoSyncerTest) cbFuncInstanceID = routeInfo.instanceid(); return Status::OK(); }); - auto future = observerActor_->InstanceInfoSyncer(); + auto future = observerActor_->InstanceInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); @@ -987,11 +1015,10 @@ TEST_F(ObserverTest, InstanceInfoSyncerTest) rep->header.revision = 2; rep->kvs.emplace_back(events[2].kv); rep->kvs.emplace_back(events[4].kv); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(rep)); cbFuncInstanceID = ""; EXPECT_EQ(cbFuncInstanceID, ""); - future = observerActor_->InstanceInfoSyncer(); + future = observerActor_->InstanceInfoSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); EXPECT_TRUE(observerActor_->instanceInfoMap_.count("InstanceID1") == 0); // rep don't have key1, so need to delete @@ -1064,6 +1091,42 @@ TEST_F(ObserverTest, GetAndWatchInstanceTest) observerActor_->isPartialWatchInstances_ = false; } +TEST_F(ObserverTest, GetOrWatchInstanceTest) +{ + auto future = controlPlaneObserver_->GetOrWatchInstance("InstanceID1"); + ASSERT_AWAIT_TRUE([&]() { return future.IsError(); }); + + auto events = GenerateResponseRouteEvent(observerActor_->nodeID_); + metaStorageAccessor_->Put(GenInstanceRouteKey("InstanceID1"), events[0].kv.value()); + ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 1;}); + + future = controlPlaneObserver_->GetOrWatchInstance("InstanceID1"); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().instanceid(), "InstanceID1"); + + metaStorageAccessor_->Delete(GenInstanceRouteKey("InstanceID1")); + ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 0;}); + + observerActor_->isPartialWatchInstances_ = true; + future = controlPlaneObserver_->GetOrWatchInstance("InstanceID1"); + ASSERT_AWAIT_TRUE([&]() { return future.IsError(); }); + + metaStorageAccessor_->Put(GenInstanceRouteKey("InstanceID1"), events[0].kv.value()); + ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 1;}); + + future = controlPlaneObserver_->GetOrWatchInstance("InstanceID1"); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().instanceid(), "InstanceID1"); + + metaStorageAccessor_->Delete(GenInstanceRouteKey("InstanceID1")); + ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 0;}); + + future = controlPlaneObserver_->GetOrWatchInstance("InstanceID02"); + ASSERT_AWAIT_TRUE([&]() { return future.IsError(); }); + + observerActor_->isPartialWatchInstances_ = false; +} + TEST_F(ObserverTest, SubscribeInstanceEventTest) { @@ -1073,7 +1136,7 @@ TEST_F(ObserverTest, SubscribeInstanceEventTest) auto events = GenerateResponseRouteEvent(observerActor_->nodeID_); metaStorageAccessor_->Put(GenInstanceRouteKey("InstanceID1"), events[0].kv.value()); ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 1;}); - controlPlaneObserver_->DelInstanceEvent("InstanceID1"); + controlPlaneObserver_->DelInstanceEvent("InstanceID1", 100); ASSERT_AWAIT_TRUE([&](){return observerActor_->instanceInfoMap_.count("InstanceID1") == 0;}); observerActor_->isPartialWatchInstances_ = false; auto future1 = observerActor_->TrySubscribeInstanceEvent("InstanceID-NotExist", "InstanceID1", false); @@ -1105,18 +1168,15 @@ TEST_F(ObserverTest, PartialInstanceInfoSyncerTest) EXPECT_TRUE(observerActor_->instanceInfoMap_.count("InstanceID4") != 0); // get key1(status 3) in etcd - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->header.revision = 5; rep->status = Status::OK(); rep->kvs.emplace_back(events[1].kv); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); controlPlaneObserver_->SetInstanceInfoSyncerCbFunc( [](const resource_view::RouteInfo &routeInfo) { return Status::OK(); }); - auto future = observerActor_->PartialInstanceInfoSyncer("InstanceID1"); + auto future = observerActor_->PartialInstanceInfoSyncer(rep, "InstanceID1"); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); @@ -1126,12 +1186,9 @@ TEST_F(ObserverTest, PartialInstanceInfoSyncerTest) == static_cast(InstanceState::RUNNING)); // get key2 in etcd - getResponseFuture = litebus::Future>(); rep->kvs.clear(); rep->kvs.emplace_back(events[2].kv); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - future = observerActor_->PartialInstanceInfoSyncer("InstanceID2"); + future = observerActor_->PartialInstanceInfoSyncer(rep, "InstanceID2"); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); @@ -1139,19 +1196,15 @@ TEST_F(ObserverTest, PartialInstanceInfoSyncerTest) EXPECT_TRUE(observerActor_->instanceInfoMap_.count("InstanceID2") != 0); // get key3 not in etcd - getResponseFuture = litebus::Future>(); rep->kvs.clear(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - future = observerActor_->PartialInstanceInfoSyncer("InstanceID3"); + future = observerActor_->PartialInstanceInfoSyncer(rep, "InstanceID3"); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); // test not in etcd but in cache, need delete EXPECT_TRUE(observerActor_->instanceInfoMap_.count("InstanceID3") == 0); // get key4 not in etcd - EXPECT_CALL(*mockMetaStoreClient, Get).WillOnce(testing::Return(getResponseFuture)); - future = observerActor_->PartialInstanceInfoSyncer("InstanceID4"); + future = observerActor_->PartialInstanceInfoSyncer(rep, "InstanceID4"); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); // test belong to self, not found in remote, don't delete diff --git a/functionsystem/tests/unit/function_proxy/common/observer/data_plane_observer_test.cpp b/functionsystem/tests/unit/function_proxy/common/observer/data_plane_observer_test.cpp index c3c628e3b788e599682e8ea3df8fd9ea69438ca9..8b3aeb446c74c856c74764ba5d5f4cbacb88e7f5 100644 --- a/functionsystem/tests/unit/function_proxy/common/observer/data_plane_observer_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/observer/data_plane_observer_test.cpp @@ -25,13 +25,14 @@ #include "common/etcd_service/etcd_service_driver.h" #include "meta_store_client/key_value/watcher.h" #include "meta_store_client/watch_client.h" -#include "metrics/metrics_adapter.h" -#include "metrics/metrics_constants.h" -#include "status/status.h" +#include "common/metrics/metrics_adapter.h" +#include "common/metrics/metrics_constants.h" +#include "common/status/status.h" #include "common/types/instance_state.h" -#include "meta_store_kv_operation.h" +#include "common/utils/meta_store_kv_operation.h" #include "function_proxy/common/observer/observer_actor.h" #include "function_proxy/common/state_machine/instance_context.h" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_shared_client.h" #include "mocks/mock_shared_client_manager_proxy.h" #include "utils/port_helper.h" @@ -44,7 +45,7 @@ public: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -52,7 +53,7 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -68,6 +69,12 @@ public: FUNCTION_PROXY_OBSERVER_ACTOR_NAME, "nodeA", metaStorageAccessor, function_proxy::ObserverParam{}); mockSharedClientManagerProxy_ = std::make_shared(); observerActor_->BindDataInterfaceClientManager(mockSharedClientManagerProxy_); + function_proxy::InternalIAM::Param iamParam; + iamParam.isEnableIAM = true; + iamParam.credType = IAMCredType::AK_SK; + internalIAM_ = std::make_shared(iamParam); + observerActor_->BindInternalIAM(internalIAM_); + EXPECT_CALL(*internalIAM_, IsSystemTenant).WillRepeatedly(testing::Return(false)); litebus::Spawn(observerActor_); dataPlaneObserver_ = std::make_shared(observerActor_); @@ -82,6 +89,7 @@ public: observerActor_ = nullptr; dataPlaneObserver_ = nullptr; + internalIAM_ = nullptr; unsetenv("HOST_IP"); unsetenv("HOSTNAME"); @@ -91,6 +99,7 @@ protected: std::shared_ptr observerActor_; std::shared_ptr dataPlaneObserver_; std::shared_ptr mockSharedClientManagerProxy_; + std::shared_ptr internalIAM_; }; inline RouteInfo GenRouteInfo(const std::string &instanceID, const std::string &funcAgentID, diff --git a/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/CMakeLists.txt b/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b65444615b7324c37fc95054a8273efec92c3b91 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} POSIX_AUTH_INTERCEPTOR_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${POSIX_AUTH_INTERCEPTOR_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/posix_auth_interceptor_test.cpp b/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/posix_auth_interceptor_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..25dc6350890742115727bec7e4c12a75ab6ec609 --- /dev/null +++ b/functionsystem/tests/unit/function_proxy/common/posix_auth_interceptor/posix_auth_interceptor_test.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "function_proxy/common/posix_auth_interceptor/posix_auth_interceptor.h" + +#include +#include + +#include "common/posix_service/posix_service.h" +#include "common/rpc/server/common_grpc_server.h" +#include "common/utils/files.h" +#include "function_proxy/common/iam/internal_iam.h" +#include "mocks/mock_internal_iam.h" +#include "mocks/mock_runtime_client.h" +#include "utils/future_test_helper.h" +#include "utils/port_helper.h" + +namespace functionsystem::test { +using namespace ::testing; +using namespace functionsystem::grpc; +using namespace runtime_rpc; +using namespace function_proxy; + +namespace { +const std::string GRPC_SERVER_IP = "127.0.0.1"; +const std::string TEST_INSTANCE_ID = "TEST_INSTANCE_ID"; +const std::string TEST_RUNTIME_ID = "TEST_RUNTIME_ID"; +} // namespace + +class MockClientProxy { +public: + MOCK_METHOD(void, MockUpdatePosixClient, + (const std::string &instanceID, const std::string &runtimeID, + const std::shared_ptr &posixClient)); + + void FakeUpdatePosixClient(const std::string &instanceID, const std::string &runtimeID, + const std::shared_ptr &posixClient) + { + std::cout << "instance id = " << instanceID << std::endl; + std::cout << "runtime id = " << runtimeID << std::endl; + clients.emplace(instanceID, posixClient); + MockUpdatePosixClient(instanceID, runtimeID, posixClient); + } + + std::unordered_map> clients; +}; + +class PosixAuthInterceptorTest : public ::testing::Test { +public: + inline static uint16_t grpcServerPort_; + [[maybe_unused]] static void SetUpTestSuite() + { + grpcServerPort_ = functionsystem::test::FindAvailablePort(); + } + + void SetUp() override + { + InternalIAM::Param param; + param.isEnableIAM = true; + param.credType = IAMCredType::AK_SK; + internalIAM_ = std::make_shared(param); + std::shared_ptr<::grpc::ServerCredentials> creds = ::grpc::InsecureServerCredentials(); + + CommonGrpcServerConfig serverConfig; + serverConfig.ip = GRPC_SERVER_IP; + serverConfig.listenPort = std::to_string(grpcServerPort_); + serverConfig.creds = creds; + server_ = std::make_shared(serverConfig); + posixService_ = std::make_shared(); + posixService_->BindInternalIAM(internalIAM_); + server_->RegisterService(posixService_); + server_->Start(); + ASSERT_TRUE(server_->WaitServerReady()); + + mockProxy_ = std::make_shared(); + posixService_->RegisterUpdatePosixClientCallback(std::bind(&MockClientProxy::FakeUpdatePosixClient, mockProxy_, + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3)); + } + + void TearDown() override + { + server_ = nullptr; + posixService_ = nullptr; + internalIAM_ = nullptr; + PosixService::DeleteClient(TEST_INSTANCE_ID); + } + + std::shared_ptr CreateRuntimeClient(const std::string &instanceID, const std::string &runtimeID) + { + RuntimeClientConfig config; + config.serverAddress = GRPC_SERVER_IP + ":" + std::to_string(grpcServerPort_); + config.serverName = "server"; + config.runtimeID = runtimeID; + config.instanceID = instanceID; + auto client = std::make_shared(config); + client->Start(); + return client; + } + +protected: + std::shared_ptr server_{ nullptr }; + std::shared_ptr posixService_{ nullptr }; + std::shared_ptr internalIAM_{ nullptr }; + std::shared_ptr mockProxy_{ nullptr }; +}; + +TEST_F(PosixAuthInterceptorTest, VerifyAKSKTest) +{ + KillRequest request; + auto killMsg = std::make_shared(); + *killMsg->mutable_killreq() = request; + killMsg->set_messageid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); + + internalIAM_->Insert(TEST_INSTANCE_ID); + EXPECT_CALL(*internalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); + EXPECT_CALL(*internalIAM_, GetCredType).WillRepeatedly(Return(IAMCredType::AK_SK)); + + EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID, testing::_)).Times(1); + litebus::Future> akSkFuture; + auto akSkContent = std::make_shared(); + akSkContent->tenantID = "tenantID"; + akSkContent->accessKey = "ak"; + akSkContent->secretKey = "sk"; + akSkContent->dataKey = "dk"; + auto now = static_cast(std::time(nullptr)); + akSkContent->expiredTimeStamp = now + 10000; + akSkFuture.SetValue(akSkContent); + EXPECT_CALL(*internalIAM_, RequireCredentialByAK) + .WillOnce(Return(akSkFuture)) + .WillOnce(Return(akSkFuture)) + .WillOnce(Return(akSkFuture)); + + auto client = CreateRuntimeClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID); + EXPECT_CALL(*client, MockClientClosedCallback).Times(1); + + auto recvFuture = litebus::Future>(); + EXPECT_CALL(*client, MockReceiver).WillOnce(DoAll(FutureArg<0>(&recvFuture))); + ASSERT_TRUE(client->Send(killMsg).Get(3000).IsSome()); + + ASSERT_AWAIT_READY(recvFuture); + ASSERT_TRUE(recvFuture.IsOK()); + EXPECT_EQ(recvFuture.Get()->body_case(), StreamingMessage::kKillRsp); + EXPECT_EQ(recvFuture.Get()->killrsp().code(), common::ERR_LOCAL_SCHEDULER_ABNORMAL); + + EXPECT_EQ(mockProxy_->clients.size(), static_cast(1)); + + client->Stop(); +} + +TEST_F(PosixAuthInterceptorTest, VerifyAKSKFailedTest) +{ + KillRequest request; + auto killMsg = std::make_shared(); + *killMsg->mutable_killreq() = request; + killMsg->set_messageid(litebus::uuid_generator::UUID::GetRandomUUID().ToString()); + + internalIAM_->Insert(TEST_INSTANCE_ID); + EXPECT_CALL(*internalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); + EXPECT_CALL(*internalIAM_, GetCredType).WillRepeatedly(Return(IAMCredType::AK_SK)); + + EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID, testing::_)).Times(1); + litebus::Future> akSkFuture; + auto akSkContent = std::make_shared(); + akSkContent->tenantID = "tenantID"; + akSkContent->accessKey = "ak"; + akSkContent->secretKey = "sk"; + akSkContent->dataKey = "dk"; + auto now = static_cast(std::time(nullptr)); + akSkContent->expiredTimeStamp = now + 10000; + akSkFuture.SetValue(akSkContent); + + litebus::Future> fakeAkSkFuture; + auto fakeAkSkContent = std::make_shared(); + fakeAkSkContent->tenantID = "tenantID"; + fakeAkSkContent->accessKey = "ak"; + fakeAkSkContent->secretKey = "fake"; + fakeAkSkContent->dataKey = "dk"; + fakeAkSkContent->expiredTimeStamp = now + 10000; + fakeAkSkFuture.SetValue(fakeAkSkContent); + EXPECT_CALL(*internalIAM_, RequireCredentialByAK) + .WillOnce(Return(akSkFuture)) + .WillOnce(Return(akSkFuture)) + .WillOnce(Return(akSkFuture)) + .WillOnce(Return(fakeAkSkFuture)); + + auto client = CreateRuntimeClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID); + EXPECT_CALL(*client, MockClientClosedCallback).Times(1); + + auto recvFuture = litebus::Future>(); + EXPECT_CALL(*client, MockReceiver).WillOnce(DoAll(FutureArg<0>(&recvFuture))); + ASSERT_TRUE(client->Send(killMsg).Get(3000).IsSome()); + + ASSERT_AWAIT_READY(recvFuture); + ASSERT_TRUE(recvFuture.IsOK()); + EXPECT_EQ(recvFuture.Get()->body_case(), StreamingMessage::kKillRsp); + EXPECT_EQ(recvFuture.Get()->killrsp().code(), common::ERR_LOCAL_SCHEDULER_ABNORMAL); + EXPECT_EQ(mockProxy_->clients.size(), static_cast(1)); + + auto status = client->Send(killMsg); + ASSERT_AWAIT_READY(status); + + client->Stop(); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/posix_client/shared_posix_test.cpp b/functionsystem/tests/unit/function_proxy/common/posix_client/shared_posix_test.cpp index 4b05ef8fe11c744d5b9f030377979e25b3695fb3..872358ca058a4b3b6629c70e358fb3025b6b5ffc 100644 --- a/functionsystem/tests/unit/function_proxy/common/posix_client/shared_posix_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/posix_client/shared_posix_test.cpp @@ -15,7 +15,7 @@ */ #include "async/async.hpp" -#include "actor_worker.h" +#include "common/utils/actor_worker.h" #include "function_proxy/common/posix_client/shared_client/shared_client.h" #include "function_proxy/common/posix_client/shared_client/shared_client_manager.h" #include "function_proxy/common/posix_client/shared_client/posix_stream_manager_proxy.h" diff --git a/functionsystem/tests/unit/function_proxy/common/posix_service/posix_service_test.cpp b/functionsystem/tests/unit/function_proxy/common/posix_service/posix_service_test.cpp index 17c4f0fdacfdc20ad5c06a2addfc9679bdbeabe4..e27a3a27ccaecb256ee02d39ac7f15f0b821c218 100644 --- a/functionsystem/tests/unit/function_proxy/common/posix_service/posix_service_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/posix_service/posix_service_test.cpp @@ -18,8 +18,9 @@ #include #include -#include "rpc/server/common_grpc_server.h" -#include "files.h" +#include "common/rpc/server/common_grpc_server.h" +#include "common/utils/files.h" +#include "function_proxy/common/iam/internal_iam.h" #include "mocks/mock_runtime_client.h" #include "utils/future_test_helper.h" #include "utils/port_helper.h" @@ -28,6 +29,7 @@ namespace functionsystem::test { using namespace ::testing; using namespace functionsystem::grpc; using namespace runtime_rpc; +using namespace function_proxy; namespace { const std::string GRPC_SERVER_IP = "127.0.0.1"; @@ -56,13 +58,16 @@ public: class PosixServiceTest : public ::testing::Test { public: inline static uint16_t grpcServerPort_; - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { grpcServerPort_ = functionsystem::test::FindAvailablePort(); } void SetUp() override { + InternalIAM::Param param; + param.isEnableIAM = false; + internalIAM_ = std::make_shared(param); std::shared_ptr<::grpc::ServerCredentials> creds = ::grpc::InsecureServerCredentials(); CommonGrpcServerConfig serverConfig; @@ -71,6 +76,7 @@ public: serverConfig.creds = creds; server_ = std::make_shared(serverConfig); posixService_ = std::make_shared(); + posixService_->BindInternalIAM(internalIAM_); server_->RegisterService(posixService_); server_->Start(); ASSERT_TRUE(server_->WaitServerReady()); @@ -102,6 +108,7 @@ public: protected: std::shared_ptr server_{ nullptr }; std::shared_ptr posixService_{ nullptr }; + std::shared_ptr internalIAM_{ nullptr }; std::shared_ptr mockProxy_{ nullptr }; }; @@ -124,7 +131,13 @@ TEST_F(PosixServiceTest, ClientConnectTest) EXPECT_EQ(mockProxy_->clients.size(), static_cast(0)); EXPECT_CALL(*client1, MockClientClosedCallback).Times(1); + auto client11 = CreateRuntimeClient(TEST_INSTANCE_ID + "1", TEST_RUNTIME_ID); + EXPECT_EQ(mockProxy_->clients.size(), static_cast(0)); + litebus::Future isCalled11; + EXPECT_CALL(*client11, MockClientClosedCallback).Times(1).WillOnce(Assign(&isCalled11, true)); + EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID, testing::_)).Times(1); + internalIAM_->Insert(TEST_INSTANCE_ID); auto client2 = CreateRuntimeClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID); litebus::Future isCalled2; EXPECT_CALL(*client2, MockClientClosedCallback).Times(1).WillOnce(Assign(&isCalled2, true)); @@ -142,6 +155,7 @@ TEST_F(PosixServiceTest, ClientConnectTest) client1->Stop(); client2->Stop(); + ASSERT_AWAIT_READY(isCalled11); ASSERT_AWAIT_READY(isCalled2); } @@ -154,6 +168,7 @@ TEST_F(PosixServiceTest, ClientConnectTest) TEST_F(PosixServiceTest, UpdatePosixClientTest) { EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID, testing::_)).Times(1); + internalIAM_->Insert(TEST_INSTANCE_ID); auto client = CreateRuntimeClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID); EXPECT_CALL(*client, MockClientClosedCallback).Times(1); @@ -190,6 +205,7 @@ TEST_F(PosixServiceTest, DuplicateClientConnect) { EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID, testing::_)).Times(1); EXPECT_CALL(*mockProxy_, MockUpdatePosixClient(TEST_INSTANCE_ID, "TEST_RUNTIME_ID_ACCEPT", testing::_)).Times(1); + internalIAM_->Insert(TEST_INSTANCE_ID); auto client = CreateRuntimeClient(TEST_INSTANCE_ID, TEST_RUNTIME_ID); litebus::Future isCalled1; EXPECT_CALL(*client, MockClientClosedCallback).Times(1).WillOnce(Assign(&isCalled1, true)); diff --git a/functionsystem/tests/unit/function_proxy/common/state_handler/state_handler_test.cpp b/functionsystem/tests/unit/function_proxy/common/state_handler/state_handler_test.cpp index cbe1765203b3ff7c99abf13c76b864ea380be1b6..6f95149bc81101fc858f37ee0bbf658bb7eb5892 100644 --- a/functionsystem/tests/unit/function_proxy/common/state_handler/state_handler_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/state_handler/state_handler_test.cpp @@ -16,8 +16,8 @@ #include "function_proxy/common/state_handler/state_handler.h" -#include "hex/hex.h" -#include "files.h" +#include "common/hex/hex.h" +#include "common/utils/files.h" #include "gtest/gtest.h" #include "mocks/mock_distributed_cache_client.h" #include "state_handler_helper.h" @@ -262,4 +262,69 @@ TEST_F(StateHandlerTest, LoadStateFailedWithInValidAID) EXPECT_EQ(response.Get()->saversp().message(), "save state failed: don't init state actor"); } +/** + * Feature: SaveStateSuccess + * Description: save state success + * Steps: + * 1. save state + * + * Expectation: + * 1. StatusCode::ERR_NONE + */ +TEST_F(StateHandlerTest, DeleteStateSuccess) +{ + EXPECT_CALL(*distributedCacheClient_, Del(Matcher("DeleteStateSuccess_instance_id"))) + .WillOnce(Return(Status::OK())); + + auto status = StateHandler::DeleteState("DeleteStateSuccess_instance_id"); + EXPECT_TRUE(status.Get().IsOk()); +} + +/** + * Feature: DeleteStateFailed + * Description: delete state failed + * Steps: + * 1. delete state with empty instance id + * 2. delete state from cache failed + * + * Expectation: + * 1. StatusCode::ERR_INSTANCE_INFO_INVALID + * 2. StatusCode::FAILED + */ +TEST_F(StateHandlerTest, DeleteStateFailed) +{ + auto status = StateHandler::DeleteState(""); + EXPECT_AWAIT_READY(status); + EXPECT_FALSE(status.Get()); + + EXPECT_CALL(*distributedCacheClient_, Del(Matcher("DeleteStateFailed_instance_id"))) + .WillOnce(Return(Status(StatusCode::FAILED, "something was wrong"))); + + status = StateHandler::DeleteState("DeleteStateFailed_instance_id"); + EXPECT_AWAIT_READY(status); + EXPECT_EQ(status.Get().StatusCode(), StatusCode::FAILED); + +} + + +/** + * Feature: DeleteStateFailed + * Description: delete state failed + * Steps: + * 1. clear aid + * 2. delete state + * + * Expectation: + * 1. StatusCode::ERR_INNER_SYSTEM_ERROR + */ + TEST_F(StateHandlerTest, DeleteStateFailedWithInValidAID) + { + StateHandlerHelper::ClearStateActorHelper(); + + auto status = StateHandler::DeleteState("instance_id"); + + EXPECT_AWAIT_READY(status); + EXPECT_EQ(status.Get().StatusCode(), StatusCode::ERR_INNER_SYSTEM_ERROR); + } + } // namespace functionsystem::function_proxy::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/state_machine/instance_control_view_test.cpp b/functionsystem/tests/unit/function_proxy/common/state_machine/instance_control_view_test.cpp index af9f18a0998eac43cdacba416143dc548ae53625..1ceead8dfaf5e72bd05a8b47067ce086f6e05c13 100644 --- a/functionsystem/tests/unit/function_proxy/common/state_machine/instance_control_view_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/state_machine/instance_control_view_test.cpp @@ -19,8 +19,8 @@ #include #include "common/etcd_service/etcd_service_driver.h" -#include "logs/logging.h" -#include "proto/pb/message_pb.h" +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" #include "mocks/mock_meta_store_client.h" #include "mocks/mock_observer.h" #include "mocks/mock_txn_transaction.h" @@ -38,7 +38,7 @@ public: inline static std::unique_ptr etcdSrvDriver_; inline static std::string metaStoreServerHost_; - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -46,7 +46,7 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -283,7 +283,7 @@ TEST_F(InstanceControlViewTest, DelInstanceTest) auto observer = std::make_shared(); InstanceStateMachine::BindControlPlaneObserver(observer); - EXPECT_CALL(*observer, DelInstanceEvent(generatedID)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*observer, DelInstanceEvent(generatedID, _)).WillOnce(testing::Return(Status::OK())); status = instanceControlView.DelInstance(generatedID); EXPECT_TRUE(status.Get().IsOk()); @@ -370,8 +370,9 @@ TEST_F(InstanceControlViewTest, DeleteInstance) resources::InstanceInfo instanceInfo; instanceInfo.set_functionproxyid("proxyid"); instanceInfo.mutable_instancestatus()->set_code(static_cast(InstanceState::EXITING)); + (*instanceInfo.mutable_extensions())[INSTANCE_MOD_REVISION] = "1000"; instanceControlView.Update(instanceID, instanceInfo, false); - instanceControlView.Delete("instanceIDA"); + instanceControlView.Delete("instanceIDA", 100); instanceControlView.Delete("instanceIDA"); instanceControlView.Delete(instanceID); @@ -425,6 +426,36 @@ TEST_F(InstanceControlViewTest, IsDuplicateScheduling) res = instanceControlView.NewInstance(scheduleReq); ASSERT_AWAIT_READY(res); EXPECT_TRUE(res.Get().preState == InstanceState::SCHEDULING); + + instanceControlView.ReleaseOwner(instanceID); + res = instanceControlView.NewInstance(scheduleReq); + ASSERT_AWAIT_READY(res); + EXPECT_TRUE(res.Get().preState == InstanceState::SCHEDULING); +} + +TEST_F(InstanceControlViewTest, OldUpdateInstanceEvent) +{ + InstanceControlView instanceControlView(TEST_NODE_ID, false); + std::string instanceID = "instanceID"; + std::string requestID = "req"; + instanceControlView.SetOwner(instanceID); + resources::InstanceInfo instanceInfo; + instanceInfo.set_functionproxyid("proxyid"); + instanceInfo.set_requestid(requestID); + instanceInfo.mutable_instancestatus()->set_code(static_cast(InstanceState::SCHEDULING)); + instanceControlView.Update(instanceID, instanceInfo, false); + + instanceInfo.mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + (*instanceInfo.mutable_extensions())["modRevision"] = "10"; + instanceControlView.Update(instanceID, instanceInfo, false); + + instanceInfo.mutable_instancestatus()->set_code(static_cast(InstanceState::SCHEDULING)); + (*instanceInfo.mutable_extensions())["modRevision"] = "1"; + instanceControlView.Update(instanceID, instanceInfo, false); + + auto stateMachine = instanceControlView.GetInstance(instanceID); + EXPECT_EQ(stateMachine->GetModRevision(), 10); + EXPECT_EQ(stateMachine->GetInstanceState(), InstanceState::CREATING); } } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/common/state_machine/state_machine_test.cpp b/functionsystem/tests/unit/function_proxy/common/state_machine/state_machine_test.cpp index d8eeabbec5f37b6442ffbaba762e3b5f5d1a2902..d3d629828cd977f2b108c406c4ea14ca57343130 100644 --- a/functionsystem/tests/unit/function_proxy/common/state_machine/state_machine_test.cpp +++ b/functionsystem/tests/unit/function_proxy/common/state_machine/state_machine_test.cpp @@ -18,10 +18,10 @@ #define private public #include "common/etcd_service/etcd_service_driver.h" -#include "logs/logging.h" -#include "metrics/metrics_adapter.h" -#include "proto/pb/message_pb.h" -#include "actor_worker.h" +#include "common/logs/logging.h" +#include "common/metrics/metrics_adapter.h" +#include "common/proto/pb/message_pb.h" +#include "common/utils/actor_worker.h" #include "function_proxy/common/state_machine/instance_state_machine.h" #include "function_proxy/local_scheduler/instance_control/instance_ctrl_actor.h" #include "mocks/mock_instance_operator.h" @@ -38,7 +38,8 @@ const std::string TEST_NODE_ID = "test node id"; class InstanceStateMachineTest : public ::testing::Test { protected: - static void SetUpTestCase() { + [[maybe_unused]] static void SetUpTestSuite() + { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); @@ -47,9 +48,10 @@ protected: { functionsystem::metrics::YRInstrument::YR_INSTANCE_RUNNING_DURATION }); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); + InstanceStateMachine::UnBindControlPlaneObserver(); metrics::MetricsAdapter::GetInstance().GetMetricsContext().SetEnabledInstruments({}); } @@ -89,13 +91,14 @@ TEST_F(InstanceStateMachineTest, LowReliabilityTypeTransitionStateToRunning) scheduleReq->mutable_instance()->mutable_instancestatus()->set_code(0); scheduleReq->mutable_instance()->set_function(function); scheduleReq->mutable_instance()->mutable_createoptions()->operator[](functionsystem::RELIABILITY_TYPE) = "low"; + scheduleReq->mutable_instance()->set_lowreliability(true); auto context = std::make_shared(scheduleReq); auto instanceStateMachine = std::make_shared(TEST_NODE_ID, context, false); auto mockControlPlaneObserver = std::make_shared(); instanceStateMachine->BindControlPlaneObserver(mockControlPlaneObserver); EXPECT_CALL(*mockControlPlaneObserver, WatchInstance).Times(1).WillRepeatedly(Return()); - EXPECT_CALL(*mockControlPlaneObserver, PutInstanceEvent).Times(1).WillRepeatedly(Return()); + EXPECT_CALL(*mockControlPlaneObserver, PutInstanceEvent).Times(2).WillRepeatedly(Return()); auto mockInstanceOpt = std::make_shared(); instanceStateMachine->instanceOpt_ = mockInstanceOpt; @@ -529,7 +532,7 @@ TEST_F(InstanceStateMachineTest, DelInstanceFailed) auto res = instanceStateMachine->DelInstance(instanceID); EXPECT_TRUE(res.Get().IsError()); - EXPECT_EQ(instanceStateMachine->lastSaveFailedState_, static_cast(InstanceState::EXITED)); + EXPECT_EQ(instanceStateMachine->lastSaveFailedState_.first, static_cast(InstanceState::EXITED)); context = nullptr; instanceStateMachine->UpdateInstanceContext(context); @@ -623,7 +626,8 @@ TEST_F(InstanceStateMachineTest, ScheduleMutableSetters) EXPECT_EQ(resources.at(name).vectors().values().at(resource_view::HETEROGENEOUS_MEM_KEY) .vectors().at("uuid").values().at(0), 1010); - ASSERT_TRUE(instanceStateMachine.GetScheduleRequest()->instance().createoptions().at("func-NPU-DEVICE-IDS") == "0,2,5"); + ASSERT_TRUE(instanceStateMachine.GetScheduleRequest()->instance().createoptions().at("func-NPU-DEVICE-IDS") + == "0,2,5"); instanceStateMachine.SetRuntimeAddress("runtime-address-0"); ASSERT_TRUE(instanceStateMachine.GetScheduleRequest()->mutable_instance()->runtimeaddress() == "runtime-address-0"); @@ -833,7 +837,7 @@ TEST_F(InstanceStateMachineTest, TransitionStateFatalFromRunning) { const std::string instanceId = "instanceID"; std::map createOptions = {}; - metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceId, createOptions); + metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceId, "", createOptions); auto scheduleReq = std::make_shared(); scheduleReq->mutable_instance()->mutable_instancestatus()->set_code(0); @@ -856,7 +860,7 @@ TEST_F(InstanceStateMachineTest, TransitionStateFailedFromRunning) { const std::string instanceId = "instanceID"; std::map createOptions = {}; - metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceId, createOptions); + metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(instanceId, "", createOptions); auto scheduleReq = std::make_shared(); scheduleReq->mutable_instance()->mutable_instancestatus()->set_code(0); @@ -906,7 +910,7 @@ TEST_F(InstanceStateMachineTest, ForceDelInstance) res = instanceStateMachine->ForceDelInstance(); EXPECT_TRUE(res.Get().IsError()); - EXPECT_EQ(instanceStateMachine->lastSaveFailedState_, static_cast(InstanceState::EXITED)); + EXPECT_EQ(instanceStateMachine->lastSaveFailedState_.first, static_cast(InstanceState::EXITED)); } TEST_F(InstanceStateMachineTest, TransitionStateFailedAfterForceDelInstance) @@ -969,4 +973,19 @@ TEST_F(InstanceStateMachineTest, TestTagStop) stop = instanceStateMachine->IsStopped(); EXPECT_EQ(stop, true); } + +TEST_F(InstanceStateMachineTest, TestTagSaved) +{ + const std::string instanceId = "instanceID"; + auto scheduleReq = std::make_shared(); + auto context = std::make_shared(scheduleReq); + auto instanceStateMachine = std::make_shared(TEST_NODE_ID, context, false); + instanceStateMachine->NewSavingPomise(); + EXPECT_EQ(instanceStateMachine->IsSaving(), true); + auto saving = instanceStateMachine->GetSavingFuture(); + EXPECT_EQ(saving.IsInit(), true); + instanceStateMachine->TagSaved(); + EXPECT_EQ(saving.IsInit(), false); + EXPECT_EQ(instanceStateMachine->IsSaving(), false); +} } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/abnormal_processor/abnormal_processor_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/abnormal_processor/abnormal_processor_test.cpp index 95915982fcd6cf0570231583b842f9428e046e22..272e0302a76c6fc9d7f364909c0fd5b525167c63 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/abnormal_processor/abnormal_processor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/abnormal_processor/abnormal_processor_test.cpp @@ -22,8 +22,8 @@ #include #include -#include "logs/logging.h" -#include "status/status.h" +#include "common/logs/logging.h" +#include "common/status/status.h" #include "common/utils/exec_utils.h" #include "gmock/gmock-actions.h" #include "gmock/gmock-spec-builders.h" @@ -32,6 +32,7 @@ #include "mocks/mock_observer.h" #include "mocks/mock_function_agent_mgr.h" #include "utils/future_test_helper.h" +#include "utils/port_helper.h" namespace functionsystem::test { using namespace local_scheduler; @@ -46,14 +47,18 @@ public: MOCK_METHOD(void, Raise, (int sig), (override)); }; -const std::string TEST_META_STORE_ADDRESS = "127.0.0.1:32279"; class AbnormalProcessorTest : public ::testing::Test { public: + [[maybe_unused]] static void SetUpTestSuite() + { + metaStoreAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + } + void SetUp() override { mockObserver_ = std::make_shared(); - mockMetaStoreClient_ = std::make_shared(TEST_META_STORE_ADDRESS); - mockInstanceCtrl_ = std::make_shared(nullptr); + mockMetaStoreClient_ = std::make_shared(metaStoreAddress_); + mockInstanceCtrl_ = std::make_shared(); mockFuntionAgentMgr_ = std::make_shared("funcAgentMgr", nullptr); mockRaiseWrapper_ = std::make_shared(); abnormalProcessor_ = std::make_shared("nodeID"); @@ -78,6 +83,7 @@ protected: std::shared_ptr mockMetaStoreClient_; std::shared_ptr mockRaiseWrapper_; std::shared_ptr mockFuntionAgentMgr_; + inline static std::string metaStoreAddress_; }; /** @@ -148,7 +154,7 @@ TEST_F(AbnormalProcessorTest, RegisterWatchAbnormal) TEST_F(AbnormalProcessorTest, StartWithAbnormal) { std::string key = "/yr/abnormal/localscheduler/nodeID"; - auto jsonStr = R"({"isAbnormal":"true"})"; + auto jsonStr = R"({"nodeName":"nodeName-1","instanceManagerActorAid":"instanceManagerActorAid-1"})"; KeyValue kv; kv.set_key(key); kv.set_value(jsonStr); @@ -171,28 +177,50 @@ TEST_F(AbnormalProcessorTest, StartWithAbnormal) ASSERT_AWAIT_READY(sig); EXPECT_EQ(sig.IsOK(), true); EXPECT_EQ(sig.Get(), 2); -}; +} + +TEST_F(AbnormalProcessorTest, StartWithAbnormalWithWrongJson) +{ + std::string key = "/yr/abnormal/localscheduler/nodeID"; + auto notJsonStr = "not a json"; + KeyValue kv; + kv.set_key(key); + kv.set_value(notJsonStr); + auto getResponse = std::make_shared(); + getResponse->kvs.push_back(std::move(kv)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockInstanceCtrl_, SetAbnormal).WillOnce(Return()); + EXPECT_CALL(*mockFuntionAgentMgr_, SetAbnormal).WillOnce(Return()); + EXPECT_CALL(*mockObserver_, GetLocalInstances).WillOnce(Return(std::vector())); + auto deleteResponse = std::make_shared(); + EXPECT_CALL(*mockMetaStoreClient_, Delete).WillOnce(Return(deleteResponse)); + litebus::Future sig; + EXPECT_CALL(*mockRaiseWrapper_, Raise).WillOnce(DoAll(FutureArg<0>(&sig), Return())); + + auto future = litebus::Async(abnormalProcessor_->GetAID(), &AbnormalProcessorActor::CheckLocalSchedulerIsLegal); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.IsOK(), true); + EXPECT_EQ(future.Get(), false); + + ASSERT_AWAIT_READY(sig); + EXPECT_EQ(sig.IsOK(), true); + EXPECT_EQ(sig.Get(), 2); +} TEST_F(AbnormalProcessorTest, AbnormalSyncerTest) { { - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status(StatusCode::FAILED, ""); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillRepeatedly(testing::Return(getResponseFuture)); - auto future = abnormalProcessor_->AbnormalSyncer(); + auto future = abnormalProcessor_->AbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_FALSE(future.Get().status.IsOk()); } { - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillRepeatedly(testing::Return(getResponseFuture)); - auto future = abnormalProcessor_->AbnormalSyncer(); + auto future = abnormalProcessor_->AbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); } @@ -215,14 +243,11 @@ TEST_F(AbnormalProcessorTest, AbnormalSyncerTest) getKeyValue.set_key(key); getKeyValue.set_value(jsonStr); - litebus::Future> getResponseFuture; std::shared_ptr rep = std::make_shared(); rep->status = Status::OK(); rep->kvs.emplace_back(getKeyValue); - getResponseFuture.SetValue(rep); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(testing::Return(getResponseFuture)); - auto future = abnormalProcessor_->AbnormalSyncer(); + auto future = abnormalProcessor_->AbnormalSyncer(rep); ASSERT_AWAIT_READY(future); ASSERT_TRUE(future.Get().status.IsOk()); diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/bundle_manager/bundle_mgr_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/bundle_manager/bundle_mgr_test.cpp index 3d4feee9a807ce042653851b8125d564ec84214f..9fafa748931527cae0e50ce4d88f9a5e9a09438a 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/bundle_manager/bundle_mgr_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/bundle_manager/bundle_mgr_test.cpp @@ -161,7 +161,7 @@ public: clientManager_ = std::make_shared(); mockScheduler_ = std::make_shared(); mockLocalSchedSrv_ = std::make_shared(); - mockInstanceCtrl_ = std::make_shared(nullptr); + mockInstanceCtrl_ = std::make_shared(); EXPECT_CALL(*mockInstanceCtrl_, RegisterClearGroupInstanceCallBack).WillRepeatedly(Return()); mockMetaStoreClient_ = std::make_shared(""); auto resourceViewMgr = std::make_shared(); @@ -373,7 +373,7 @@ TEST_F(BundleMgrTest, ReserveAndUnReserveSuccessful) } EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {}, {}, "", {}, allocatedPromise })) + .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {}, {}, "", {}, {}, allocatedPromise })) .WillOnce(Return(scheduleResult)); auto changes = std::make_shared(); EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(changes)); @@ -721,7 +721,6 @@ TEST_F(BundleMgrTest, SyncFailedBundlesTest) EXPECT_EQ(bundleMgrActor_->bundles_.size(), static_cast(3)); explorer::LeaderInfo leaderInfo{ - .name = "", .address = mockResourceGroupActor->GetAID().UnfixUrl(), }; bundleMgrActor_->UpdateMasterInfo(leaderInfo); @@ -786,7 +785,6 @@ TEST_F(BundleMgrTest, NotifyFailedAgentTest) litebus::Spawn(mockResourceGroupActor); explorer::LeaderInfo leaderInfo{ - .name = "", .address = mockResourceGroupActor->GetAID().UnfixUrl(), }; bundleMgrActor_->UpdateMasterInfo(leaderInfo); diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker_test.cpp index 0dffe7cb659c9bbec57e407938b3abac9f35d248..ef2565c92da9e2e5cc4d1bc6ac103037168aefd2 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/ds_healthy_checker/ds_healthy_checker_test.cpp @@ -21,11 +21,13 @@ #include #include -#include -#include "logs/logging.h" +#include "async/async.hpp" +#include "common/constants/constants.h" +#include "common/logs/logging.h" #include "mocks/mock_distributed_cache_client.h" #include "utils/future_test_helper.h" +#include "utils/os_utils.hpp" namespace functionsystem::test { using namespace local_scheduler; @@ -33,13 +35,13 @@ using namespace ::testing; class DsHealthyCheckerTest : public ::testing::Test { public: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { mockDistributedCacheClient_ = std::make_shared(); - dsHealthyChecker_ = std::make_shared(1000, 5, mockDistributedCacheClient_); + dsHealthyChecker_ = std::make_shared(0, 5, mockDistributedCacheClient_); } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { dsHealthyChecker_ = nullptr; mockDistributedCacheClient_ = nullptr; @@ -74,16 +76,47 @@ TEST_F(DsHealthyCheckerTest, CheckHealthy) dsHealthyChecker_->SubscribeDsHealthy(cb); EXPECT_FALSE(dsHealthyChecker_->GetIsUnhealthy()); - (void)litebus::Spawn(dsHealthyChecker_); + litebus::os::SetEnv("DATA_SYSTEM_FEATURE_USED", "stream"); + const auto aid = litebus::Spawn(dsHealthyChecker_); YRLOG_WARN("checker->Start"); ASSERT_AWAIT_READY(promise->GetFuture()); EXPECT_TRUE(promise->GetFuture().Get()); EXPECT_TRUE(dsHealthyChecker_->GetIsUnhealthy()); + EXPECT_TRUE(litebus::Async(aid, &DsHealthyChecker::IsDataSystemFeatureUsed).Get()); litebus::Terminate(dsHealthyChecker_->GetAID()); litebus::Await(dsHealthyChecker_->GetAID()); } +TEST_F(DsHealthyCheckerTest, CheckHealthyWithKScaleDown) +{ + auto mockDistributedCacheClient = std::make_shared(); + auto dsHealthyChecker = std::make_shared(0, 5, mockDistributedCacheClient); + + EXPECT_CALL(*mockDistributedCacheClient, GetHealthStatus) + .WillOnce(Return(Status(SUCCESS))) + .WillRepeatedly(Return(Status(DS_SCALE_DOWN))); + + auto promise = std::make_shared>(); + auto cb = [promise](const bool healthy) { + if (!healthy) + promise->SetValue(true); + }; + dsHealthyChecker->SubscribeDsHealthy(cb); + EXPECT_FALSE(dsHealthyChecker->GetIsUnhealthy()); + + (void)litebus::Spawn(dsHealthyChecker); + YRLOG_WARN("checker->Start"); + + ASSERT_AWAIT_READY(promise->GetFuture()); + EXPECT_TRUE(promise->GetFuture().Get()); + + EXPECT_TRUE(dsHealthyChecker->GetIsUnhealthy()); + + litebus::Terminate(dsHealthyChecker->GetAID()); + litebus::Await(dsHealthyChecker->GetAID()); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_helper.h b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_helper.h index dba4907a4cd6a4dcf67cfb53423a3bf70e3e1ddc..f178c67cf74fcc8f801316d76acc942ed39335a6 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_helper.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_helper.h @@ -21,8 +21,8 @@ #include -#include "heartbeat/ping_pong_driver.h" -#include "logs/logging.h" +#include "common/heartbeat/heartbeat_client.h" +#include "common/logs/logging.h" #include "common/utils/generate_message.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_actor_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_actor_test.cpp index bcef11045eb58a7fbb93a4426f5a9e50222640e4..fe4f9bd5d82a7af3b7722c34a3e339b3968c6b34 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_actor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_actor_test.cpp @@ -17,7 +17,7 @@ #include #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" #include "common/types/instance_state.h" #include "function_agent/code_deployer/s3_deployer.h" #include "kv_service_accessor_actor.h" @@ -36,7 +36,6 @@ using std::string; using namespace local_scheduler; using namespace ::testing; -const string TEST_META_STORE_ADDRESS = "127.0.0.1:32279"; const local_scheduler::FunctionAgentMgrActor::Param PARAM = { .retryTimes = 3, .retryCycleMs = 100, @@ -44,7 +43,7 @@ const local_scheduler::FunctionAgentMgrActor::Param PARAM = { .pingCycleMs = 500, .enableTenantAffinity = true, .tenantPodReuseTimeWindow = 3, - .enableForceDeletePod = true, + .enableIpv4TenantIsolation = true, .getAgentInfoRetryMs = 100, .invalidAgentGCInterval = 100, }; @@ -75,9 +74,9 @@ protected: class FuncAgentMgrActorHelper : public FunctionAgentMgrActor { public: - FuncAgentMgrActorHelper() + FuncAgentMgrActorHelper(const std::string &metaStoreAddress) : FunctionAgentMgrActor("funcAgentMgr", PARAM, "nodeID", - std::make_shared(TEST_META_STORE_ADDRESS)) + std::make_shared(metaStoreAddress)) { } @@ -101,10 +100,17 @@ public: class FuncAgentMgrActorTest : public ::testing::Test { protected: + [[maybe_unused]] static void SetUpTestSuite() + { + metaStoreAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + } + void SetUp() override { - agentMgrActorHelper_ = make_shared(); + agentMgrActorHelper_ = make_shared(metaStoreAddress_); } + + inline static std::string metaStoreAddress_; shared_ptr agentMgrActorHelper_; }; @@ -142,7 +148,6 @@ TEST_F(FuncAgentMgrActorTest, TimeoutEventTest) auto mockMetaStoreClient_ = std::make_shared("111111"); auto actor = make_shared("RecoverHeartBeatSuccessActor", PARAM, "nodeID", mockMetaStoreClient_); - actor->heartBeatObserverCtrl_ = std::make_shared(100, 100); actor->TimeoutEvent("id1"); EXPECT_EQ(actor->funcAgentTable_.count("id1"), size_t(0)); @@ -237,7 +242,7 @@ TEST_F(FuncAgentMgrActorTest, QueryDebugInstanceInfos) EXPECT_EQ(response->kvs.size(), static_cast(1)); EXPECT_EQ(response->kvs[0].key(), "/yr/debug/test_instID1"); messages::DebugInstanceInfo info; - google::protobuf::util::JsonStringToMessage(response->kvs[0].value(),&info); + (void)google::protobuf::util::JsonStringToMessage(response->kvs[0].value(),&info); EXPECT_EQ(info.instanceid(),"test_instID1"); EXPECT_EQ(info.debugserver(),"test_gdbserverAddr"); @@ -254,4 +259,108 @@ TEST_F(FuncAgentMgrActorTest, QueryDebugInstanceInfos) } +TEST_F(FuncAgentMgrActorTest, TenantEventCase2) +{ + // same node + TenantEvent event1 = { + .tenantID = TENANT_ID1, + .functionProxyID = "nodeID", + .functionAgentID = FUNC_AGENT_ID1, + .instanceID = FUNC_INSTANCE_ID1, + .agentPodIp = "10.42.1.221", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event1); + + // same tenant on other node + TenantEvent event2 = { + .tenantID = TENANT_ID1, + .functionProxyID = FUNC_PROXY_ID2, + .functionAgentID = FUNC_AGENT_ID2, + .instanceID = FUNC_INSTANCE_ID2, + .agentPodIp = "10.42.2.222", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event2); + + // another instance but same pod in this node + TenantEvent event3 = { + .tenantID = TENANT_ID1, + .functionProxyID = "nodeID", + .functionAgentID = FUNC_AGENT_ID1, + .instanceID = FUNC_INSTANCE_ID2, + .agentPodIp = "10.42.1.221", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event3); + + auto tenantCacheMap = agentMgrActorHelper_->GetTenantCacheMap(); + auto tenantCache = tenantCacheMap[event1.tenantID]; + EXPECT_EQ(tenantCache->podIps.size(), 2u); + + agentMgrActorHelper_->OnTenantDeleteInstance(event3); + EXPECT_EQ(tenantCache->podIps.size(), 2u); + + agentMgrActorHelper_->OnTenantDeleteInstance(event1); + EXPECT_EQ(tenantCache->podIps.size(), 1u); + EXPECT_FALSE(tenantCache->functionAgentCacheMap[event1.functionAgentID].isAgentOnThisNode); +} + +TEST_F(FuncAgentMgrActorTest, TenantEventCase3) +{ + auto mockAgent = std::make_shared(); + litebus::Spawn(mockAgent); + + FuncAgentMgrActorHelper::FuncAgentInfo mockAgentInfo; + mockAgentInfo.aid = mockAgent->GetAID(); + + EXPECT_CALL(*mockAgent, SetNetworkIsolationRequest).WillOnce(Return()).WillOnce(Return()).WillOnce(Return()); + + agentMgrActorHelper_->InsertAgent(FUNC_AGENT_ID1, mockAgentInfo); + agentMgrActorHelper_->InsertAgent(FUNC_AGENT_ID2, mockAgentInfo); + + // same tenant on other node + TenantEvent event2 = { + .tenantID = TENANT_ID1, + .functionProxyID = FUNC_PROXY_ID2, + .functionAgentID = FUNC_AGENT_ID2, + .instanceID = FUNC_INSTANCE_ID2, + .agentPodIp = "10.42.2.222", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event2); + + // same node + TenantEvent event1 = { + .tenantID = TENANT_ID1, + .functionProxyID = "nodeID", + .functionAgentID = FUNC_AGENT_ID1, + .instanceID = FUNC_INSTANCE_ID1, + .agentPodIp = "10.42.1.221", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event1); + + TenantEvent event3 = { + .tenantID = TENANT_ID1, + .functionProxyID = "nodeID", + .functionAgentID = "fake_agent_id", + .instanceID = FUNC_INSTANCE_ID1, + .agentPodIp = "10.42.1.221", + .code = static_cast(InstanceState::RUNNING), + }; + agentMgrActorHelper_->OnTenantUpdateInstance(event3); + + auto tenantCacheMap = agentMgrActorHelper_->GetTenantCacheMap(); + auto tenantCache = tenantCacheMap[event1.tenantID]; + EXPECT_EQ(tenantCache->podIps.size(), 2u); + + agentMgrActorHelper_->OnTenantDeleteInstance(event1); + EXPECT_EQ(tenantCache->podIps.size(), 1u); + EXPECT_FALSE(tenantCache->functionAgentCacheMap[event2.functionAgentID].isAgentOnThisNode); + + litebus::Terminate(mockAgent->GetAID()); + litebus::Await(mockAgent->GetAID()); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_test.cpp index 49b00a71358cbd6b70e562f64b1e3ca140c89520..33e231c53000a522d0b3053f37179ebac373ce7e 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/function_agent_manager/function_agent_mgr_test.cpp @@ -25,12 +25,12 @@ #include #include "common/constants/signal.h" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" #include "function_agent/code_deployer/s3_deployer.h" #include "function_agent_helper.h" +#include "utils/port_helper.h" #include "mocks/mock_bundle_mgr.h" #include "mocks/mock_function_agent.h" -#include "mocks/mock_heartbeat_observer_driver_ctrl.h" #include "mocks/mock_instance_ctrl.h" #include "mocks/mock_local_sched_srv.h" #include "mocks/mock_meta_store_client.h" @@ -74,7 +74,6 @@ const string TEST_LOCAL_SCHEDULER_AID = "testLocalScheduler_01-32379"; // agent address formatted as string "127.0.0.1:58866" const string SETUP_FUNC_AGENT_AID_NAME = "AgentServiceActor"; const string SETUP_LOCAL_SCHEDULER_AID = "setupLocalScheduler_01-32379"; -const string SETUP_FUNC_AGENT_ADDRESS = "127.0.0.1:32279"; const string SETUP_RUNTIME_MANAGER_AID = "setup-RuntimeManagerSrv"; const string SETUP_RUNTIME_MANAGER_RANDOM_ID = "setup-runtimemanager-random-id"; const string SETUP_INSTANCE_ID = "setup-instance-id"; @@ -86,12 +85,12 @@ const local_scheduler::FunctionAgentMgrActor::Param PARAM = { .pingCycleMs = 500, .enableTenantAffinity = true, .tenantPodReuseTimeWindow = 3, + .enableIpv4TenantIsolation = true, .enableForceDeletePod = true, .getAgentInfoRetryMs = 100, .invalidAgentGCInterval = 100, }; -const string TEST_META_STORE_ADDRESS = "127.0.0.1:32279"; const RuntimeConfig runtimeConfig{ .runtimeHeartbeatEnable = "true", .runtimeMaxHeartbeatTimeoutTimes = 3, .runtimeHeartbeatTimeoutMS = 2000, @@ -101,17 +100,21 @@ const RuntimeConfig runtimeConfig{ .runtimeHeartbeatEnable = "true", class FuncAgentMgrTest : public ::testing::Test { friend class FunctionAgentMgrActor; protected: - void SetUp() override + [[maybe_unused]] static void SetUpTestSuite() { - heartbeatObserverDriverCtrl_ = make_shared(); + metaStoreAddress_ = "127.0.0.1:" + std::to_string(FindAvailablePort()); + funcAgentAddress_ = metaStoreAddress_; + } - mockMetaStoreClient_ = std::make_shared(TEST_META_STORE_ADDRESS); + void SetUp() override + { + mockMetaStoreClient_ = std::make_shared(metaStoreAddress_); auto getResponse = std::make_shared(); EXPECT_CALL(*mockMetaStoreClient_, Get).WillRepeatedly(Return(getResponse)); funcAgentRegisInfoInit_.set_agentaidname(SETUP_FUNC_AGENT_AID_NAME); - funcAgentRegisInfoInit_.set_agentaddress(SETUP_FUNC_AGENT_ADDRESS); + funcAgentRegisInfoInit_.set_agentaddress(funcAgentAddress_); funcAgentRegisInfoInit_.set_runtimemgraid(SETUP_RUNTIME_MANAGER_AID); funcAgentRegisInfoInit_.set_runtimemgrid(SETUP_RUNTIME_MANAGER_RANDOM_ID); funcAgentRegisInfoInit_.set_statuscode(1); @@ -135,11 +138,12 @@ protected: localSchedSrv_ = make_shared(); mockBundleMgr_ = make_shared(); - funcAgentMgr_->Start(instCtrl_, resourceView_, heartbeatObserverDriverCtrl_); + funcAgentMgr_->Start(instCtrl_, resourceView_); funcAgentMgr_->BindLocalSchedSrv(localSchedSrv_); funcAgentMgr_->BindBundleMgr(mockBundleMgr_); litebus::Spawn(funcAgent_); + auto putResponse = std::make_shared(); EXPECT_CALL(*mockMetaStoreClient_, Put).WillRepeatedly(Return(putResponse)); std::string jsonStr; @@ -160,9 +164,6 @@ protected: EXPECT_CALL(*funcAgent_.get(), MockRegistered(testing::_, testing::_, testing::_)) .WillRepeatedly(testing::Return()); - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); - EXPECT_CALL(*resourceView_.get(), AddResourceUnit(testing::_)) .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); @@ -206,6 +207,14 @@ protected: litebus::Terminate(funcAgent_->GetAID()); litebus::Await(funcAgent_); funcAgentMgr_->ClearFuncAgentsRegis(); + funcAgentMgr_ = nullptr; + funcAgent_ = nullptr; + instCtrl_ = nullptr; + localSchedSrv_ = nullptr; + mockBundleMgr_ = nullptr; + resourceView_ = nullptr; + funcAgentHelper_ = nullptr; + mockMetaStoreClient_ = nullptr; } std::vector> RegisterFuncAgents(std::string testName, @@ -221,9 +230,6 @@ protected: litebus::Spawn(funcAgents[i]); } - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); - EXPECT_CALL(*resourceView_.get(), AddResourceUnit(testing::_)) .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); @@ -297,7 +303,6 @@ protected: return GenerateRandomName("randomFuncAgent"); } - shared_ptr heartbeatObserverDriverCtrl_; shared_ptr funcAgentMgr_; shared_ptr funcAgent_; shared_ptr instCtrl_; @@ -308,6 +313,8 @@ protected: shared_ptr mockMetaStoreClient_; messages::FuncAgentRegisInfo funcAgentRegisInfoInit_; string randomFuncAgentName_; + inline static std::string funcAgentAddress_; + inline static std::string metaStoreAddress_; }; TEST_F(FuncAgentMgrTest, CreateSuccess) @@ -344,9 +351,6 @@ TEST_F(FuncAgentMgrTest, RegisterSuccess) EXPECT_CALL(*funcAgent.get(), MockRegistered(testing::_, testing::_, testing::_)) .WillRepeatedly(testing::DoAll(test::FutureArg<2>(®isteredMsg))); - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .WillOnce(testing::Return(Status(StatusCode::SUCCESS))); - litebus::Future addResourceUnitMsg; EXPECT_CALL(*resourceView_.get(), AddResourceUnit(testing::_)) .WillOnce(testing::DoAll(FutureArg<0>(&addResourceUnitMsg), testing::Return(Status(StatusCode::SUCCESS)))); @@ -540,22 +544,11 @@ TEST_F(FuncAgentMgrTest, RegisterBuildLinkFail) EXPECT_CALL(*funcAgent.get(), MockRegistered(testing::_, testing::_, testing::_)) .WillRepeatedly(testing::DoAll(test::FutureArg<2>(®isteredMsg))); - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .WillOnce(testing::Return(Status(StatusCode::FAILED))); - - litebus::Future heartbeatDeleteMsg; - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Delete(testing::_)) - .WillOnce(testing::DoAll(FutureArg<0>(&heartbeatDeleteMsg))); - litebus::Async(funcAgent->GetAID(), &MockFunctionAgent::RegisterToLocalScheduler, funcAgentMgr_->GetActorAID()); auto registerVal = registeredMsg.Get(100); ASSERT_TRUE(registerVal.IsSome()); - auto heartbeatDeleteVal = heartbeatDeleteMsg.Get(100); - ASSERT_TRUE(heartbeatDeleteVal.IsSome()); - EXPECT_STREQ(heartbeatDeleteVal.Get().c_str(), TEST_FUNC_AGENT_NAME.c_str()); - std::cout << funcAgentMgr_->Dump() << std::endl; litebus::Terminate(funcAgent->GetAID()); @@ -601,14 +594,6 @@ TEST_F(FuncAgentMgrTest, RegisterSyncInstanceFail) EXPECT_CALL(*funcAgent.get(), MockRegistered(testing::_, testing::_, testing::_)).WillRepeatedly(testing::Return()); - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .Times(2) - .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); - - litebus::Future heartbeatDeleteMsg; - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Delete(testing::_)) - .WillRepeatedly(testing::DoAll(FutureArg<0>(&heartbeatDeleteMsg))); - EXPECT_CALL(*resourceView_.get(), AddResourceUnit(testing::_)) .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); @@ -621,14 +606,16 @@ TEST_F(FuncAgentMgrTest, RegisterSyncInstanceFail) litebus::Async(funcAgent->GetAID(), &MockFunctionAgent::RegisterToLocalScheduler, funcAgentMgr_->GetActorAID()); - auto heartbeatDeleteVal = heartbeatDeleteMsg.Get(100); - ASSERT_TRUE(heartbeatDeleteVal.IsSome()); - EXPECT_STREQ(heartbeatDeleteVal.Get().c_str(), std::string("RegisterSyncInstanceFail_func_agent_AID").c_str()); - std::cout << funcAgentMgr_->Dump() << std::endl; litebus::Async(funcAgent->GetAID(), &MockFunctionAgent::RegisterToLocalScheduler, funcAgentMgr_->GetActorAID()); + litebus::AID heartBeatAID = + litebus::AID(HEARTBEAT_OBSERVER_BASENAME + FUNCTION_AGENT_AGENT_MGR_ACTOR_NAME, funcAgentMgr_->GetActorAID().Url()); + HeartbeatClientDriver pingPongDriver("RegisterSyncInstanceFail_func_agent_AID_2", + [](const litebus::AID &) {}); + (void)pingPongDriver.Start(heartBeatAID); + ASSERT_AWAIT_TRUE([=]() -> bool { return CheckIsRegister(funcAgentMgr_, "RegisterSyncInstanceFail_func_agent_AID_2", nullptr); }); @@ -690,6 +677,16 @@ inline std::shared_ptr GenDeployInstanceRequest return req; } +inline std::shared_ptr GenStaticFunctionChangeRequest( + const std::string &requestID, const std::string &instanceID, int32_t status) +{ + auto staticFunctionChangeRequest = std::make_shared(); + staticFunctionChangeRequest->set_instanceid(instanceID); + staticFunctionChangeRequest->set_requestid(requestID); + staticFunctionChangeRequest->set_status(status); + return staticFunctionChangeRequest; +} + TEST_F(FuncAgentMgrTest, DeployInstanceSuccess) { litebus::Future mockMsg; @@ -1046,9 +1043,6 @@ TEST_F(FuncAgentMgrTest, UpdateResourcesNoInit) EXPECT_CALL(*funcAgent.get(), MockRegistered(testing::_, testing::_, testing::_)) .WillRepeatedly(testing::DoAll(test::FutureArg<2>(®isteredMsg))); - EXPECT_CALL(*heartbeatObserverDriverCtrl_.get(), Add(testing::_, testing::_, testing::_)) - .WillOnce(testing::Return(Status(StatusCode::SUCCESS))); - EXPECT_CALL(*resourceView_.get(), AddResourceUnit(testing::_)) .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); @@ -1446,13 +1440,102 @@ TEST_F(FuncAgentMgrTest, RetrieveAgentRegisInfoWithFailedStatusSuccess) funcAgentsRegis.clear(); } +TEST_F(FuncAgentMgrTest, StaticFunctionScheduleRequest) +{ + auto funcAgentMgr = + make_shared(make_shared( + "RecoverHeartBeatSuccessActor", PARAM, "nodeID", mockMetaStoreClient_)); + funcAgentMgr->SetNodeID("nodeID"); + funcAgentMgr->ToReady(); + + std::unordered_map funcAgentsRegis; + messages::FuncAgentRegisInfo info1; + info1.set_agentaidname("RecoverHeartBeatSuccess_agent_aid_name"); + info1.set_agentaddress("RecoverHeartBeatSuccess_agent_address"); + info1.set_runtimemgraid("RecoverHeartBeatSuccess_runtime_manager_aid"); + info1.set_runtimemgrid("RecoverHeartBeatSuccess_runtime_manager_randomid"); + info1.set_statuscode(static_cast(FunctionAgentMgrActor::RegisStatus::FAILED)); + funcAgentsRegis["RecoverHeartBeatSuccess_agent_id"] = info1; + + messages::FuncAgentRegisInfo info2; + info2.set_agentaidname("RecoverHeartBeatSuccessActor"); + uint16_t port = GetPortEnv("LITEBUS_PORT", 0); + info2.set_agentaddress("127.0.0.1:" + std::to_string(port)); + info2.set_runtimemgraid("funcAgentMgr_runtime_manager_aid"); + info2.set_runtimemgrid("funcAgentMgr_runtime_manager_randomid"); + info2.set_statuscode(static_cast(FunctionAgentMgrActor::RegisStatus::SUCCESS)); + funcAgentsRegis["funcAgentMgr"] = info2; + + std::string jsonStr = FuncAgentRegisToCollectionStrHelper(funcAgentsRegis); + std::cout << "serialized jsonStr:" << jsonStr << std::endl; + std::string nodeID = funcAgentMgr->GetNodeID(); + KeyValue kv; + kv.set_key("funcAgentRegisInfos"); + kv.set_value(jsonStr); + + auto getResponse = std::make_shared(); + getResponse->kvs.push_back(std::move(kv)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + + EXPECT_CALL(*localSchedSrv_, NotifyEvictResult(_)).WillOnce(Return()); + EXPECT_CALL(*mockBundleMgr_, SyncBundles).WillOnce(Return(Status::OK())); + + std::shared_ptr resourceUnit = std::make_shared(); + resourceUnit->set_id("funcAgentMgr"); + auto instances = resourceUnit->mutable_instances(); + resource_view::InstanceInfo instanceInfo; + instanceInfo.set_instanceid("funcAgentMgr_instance_id"); + instances->insert({ "funcAgentMgr_instance_id", instanceInfo }); + + messages::UpdateResourcesRequest resourceViewReq; + auto resourceUnit2 = resourceViewReq.mutable_resourceunit(); + resourceUnit2->set_id("funcAgentMgr"); + auto capacity = resourceUnit2->mutable_capacity(); + auto resources = capacity->mutable_resources(); + + auto instances2 = resourceUnit2->mutable_instances(); + resource_view::InstanceInfo instanceInfo2; + instanceInfo2.set_instanceid("funcAgentMgr_instance_id"); + instances2->insert({ "funcAgentMgr_instance_id", instanceInfo2 }); + + resource_view::Resource resource; + resource.set_name("CPU"); + resource.set_type(resource_view::ValueType::Value_Type_SCALAR); + + auto scalar = resource.mutable_scalar(); + scalar->set_limit(100); + scalar->set_value(50); + resources->insert({ "CPU", resource }); + + funcAgentMgr->Start(instCtrl_, resourceView_); + funcAgentMgr->BindLocalSchedSrv(localSchedSrv_); + funcAgentMgr->BindBundleMgr(mockBundleMgr_); + auto future = funcAgentMgr->Sync(); + ASSERT_AWAIT_READY(future); + future = funcAgentMgr->Recover(); + ASSERT_AWAIT_READY(future); + + ASSERT_AWAIT_TRUE([=]() -> bool { + auto lambda = [=](){ + std::string name = "funcAgentMgr"; + funcAgentMgr->UpdateResources(funcAgentMgr->GetActorAID(), std::move(name), + resourceViewReq.SerializeAsString()); + }; + return CheckIsRegister(funcAgentMgr, "funcAgentMgr", lambda); + }); + + // clean + resourceViewReq.clear_resourceunit(); + funcAgentsRegis.clear(); +} + TEST_F(FuncAgentMgrTest, RecoverHeartBeatEmptySuccess) { auto funcAgentMgr = make_shared(make_shared( "RecoverHeartBeatSuccessActor", PARAM, "nodeID", mockMetaStoreClient_)); funcAgentMgr->SetNodeID("nodeID"); - funcAgentMgr->Start(instCtrl_, resourceView_, heartbeatObserverDriverCtrl_); + funcAgentMgr->Start(instCtrl_, resourceView_); funcAgentMgr->BindLocalSchedSrv(localSchedSrv_); funcAgentMgr->BindBundleMgr(mockBundleMgr_); funcAgentMgr->ToReady(); @@ -1658,7 +1741,7 @@ TEST_F(FuncAgentMgrTest, InvalidAgentGC) funcAgentMgr_->SetFuncAgentsRegis(funcAgentsRegis); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse));; + EXPECT_CALL(*mockMetaStoreClient_, Put).WillRepeatedly(Return(putResponse)); litebus::Async(funcAgentMgr_->GetActorAID(), &FunctionAgentMgrActor::StopHeartbeat, "agent_id"); ASSERT_AWAIT_TRUE([=]() -> bool { auto info = litebus::Async(funcAgentMgr_->GetActorAID(), &FunctionAgentMgrActor::GetFuncAgentsRegis).Get(); @@ -1733,8 +1816,9 @@ TEST_F(FuncAgentMgrTest, TenantEventCase1) EXPECT_TRUE(tenantCache->functionAgentCacheMap[event.functionAgentID].isAgentOnThisNode); funcAgentMgr_->OnTenantDeleteInstance(event); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - EXPECT_EQ(tenantCache->functionAgentCacheMap.count(event.functionAgentID), (size_t)0); + ASSERT_AWAIT_TRUE([=]() -> bool { + return tenantCache->functionAgentCacheMap.count(event.functionAgentID) == (size_t)0; + }); } /* @@ -1744,7 +1828,7 @@ TEST_F(FuncAgentMgrTest, OnHealthyStatusTest) { auto funcAgentMgr = make_shared( make_shared("funcAgentMgr-OnHealthyStatusTest", PARAM, "nodeID", mockMetaStoreClient_)); - funcAgentMgr->Start(instCtrl_, resourceView_, heartbeatObserverDriverCtrl_); + funcAgentMgr->Start(instCtrl_, resourceView_); Status status(StatusCode::FAILED); funcAgentMgr->OnHealthyStatus(status); @@ -1782,8 +1866,7 @@ TEST_F(FuncAgentMgrTest, GracefulShutdown) auto deleteResponse = std::make_shared(); EXPECT_CALL(*mockMetaStoreClient_, Delete).WillRepeatedly(Return(deleteResponse)); - EXPECT_CALL(*instCtrl_.get(), EvictInstanceOnAgent(_)) - .WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); + EXPECT_CALL(*instCtrl_.get(), EvictInstanceOnAgent(_)).WillRepeatedly(testing::Return(Status(StatusCode::SUCCESS))); EXPECT_CALL(*resourceView_.get(), UpdateUnitStatus(_, _)).WillRepeatedly(Return(Status(StatusCode::SUCCESS))); EXPECT_CALL(*mockBundleMgr_, UpdateBundlesStatus).WillRepeatedly(Return()); @@ -1804,4 +1887,74 @@ TEST_F(FuncAgentMgrTest, GracefulShutdown) EXPECT_EQ(regis.empty(), true); } +TEST_F(FuncAgentMgrTest, CreateStaticFunctionInstance) +{ + messages::ScheduleResponse scheduleResponse = GenScheduleResponse(0, "", "", REQUEST_ID); + litebus::Future> scheduleFuture; + EXPECT_CALL(*instCtrl_.get(), Schedule) + .WillOnce(testing::DoAll(FutureArg<0>(&scheduleFuture), testing::Return(scheduleResponse))); + litebus::Future resp; + EXPECT_CALL(*funcAgent_.get(), MockStaticFunctionScheduleResponse(testing::_, testing::_, testing::_)) + .WillOnce(testing::DoAll(testing::DoAll(FutureArg<2>(&resp)))); + + messages::ScheduleRequest scheduleReq; + scheduleReq.set_requestid(REQUEST_ID); + litebus::Async(funcAgent_->GetAID(), &MockFunctionAgent::StaticFunctionScheduleRequest, funcAgentMgr_->GetActorAID(), + scheduleReq); + + auto ret = scheduleFuture.Get(1000); + ASSERT_TRUE(ret.IsSome()); + EXPECT_TRUE(ret.Get()->requestid() == REQUEST_ID); + + auto respStr = resp.Get(1000); + ASSERT_TRUE(respStr.IsSome()); + messages::ScheduleResponse respVal; + auto parseVal = respVal.ParseFromString(respStr.Get()); + EXPECT_TRUE(parseVal); + EXPECT_STREQ(respVal.requestid().c_str(), REQUEST_ID.c_str()); +} + +TEST_F(FuncAgentMgrTest, NotifyFunctionStatusChangeSuccess) +{ + litebus::Future mockMsg; + + messages::StaticFunctionChangeResponse mockResp = + GenStaticFunctionChangeResponse(StatusCode::SUCCESS, "notify success", REQUEST_ID, INSTANCE_ID); + EXPECT_CALL(*funcAgent_.get(), MockNotifyFunctionStatusChange(testing::_, testing::_, testing::_)) + .WillRepeatedly(testing::DoAll(test::FutureArg<2>(&mockMsg), + testing::Return(std::pair(true, mockResp.SerializeAsString())))); + + auto req = GenStaticFunctionChangeRequest(REQUEST_ID, INSTANCE_ID, 3); + auto ret = funcAgentMgr_->NotifyFunctionStatusChange(req, randomFuncAgentName_); + + auto resp = ret.Get(1000); + ASSERT_TRUE(resp.IsSome()); + EXPECT_STREQ(resp.Get().requestid().c_str(), REQUEST_ID.c_str()); + EXPECT_EQ(resp.Get().code(), StatusCode::SUCCESS); + + auto msg = mockMsg.Get(1000); + ASSERT_TRUE(msg.IsSome()); + ASSERT_FALSE(msg.Get().empty()); + +} + +TEST_F(FuncAgentMgrTest, NotifyFunctionStatusChangeRetrySuccess) +{ + messages::StaticFunctionChangeResponse mockResp = + GenStaticFunctionChangeResponse(StatusCode::SUCCESS, "notify success", REQUEST_ID, INSTANCE_ID); + EXPECT_CALL(*funcAgent_.get(), MockNotifyFunctionStatusChange(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(std::pair{ false, "" })) + .WillOnce(testing::Return(std::pair{ false, "" })) + .WillRepeatedly(testing::Return(std::pair{ true, mockResp.SerializeAsString() })); + + auto req = GenStaticFunctionChangeRequest(REQUEST_ID, INSTANCE_ID, 3); + + auto ret = funcAgentMgr_->NotifyFunctionStatusChange(req, randomFuncAgentName_); + + auto resp = ret.Get(1000); + ASSERT_EQ(resp.IsSome(), true); + EXPECT_STREQ(resp.Get().requestid().c_str(), REQUEST_ID.c_str()); + EXPECT_EQ(resp.Get().code(), StatusCode::SUCCESS); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/grpc_server/grpc_server_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/grpc_server/grpc_server_test.cpp index ae2ca7da6c7a8f37471b37465075fd41570c4870..bc8bca6fa3e887d506243c3a2749796ea02d8ee0 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/grpc_server/grpc_server_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/grpc_server/grpc_server_test.cpp @@ -24,13 +24,14 @@ #include #include -#include "logs/logging.h" +#include "common/logs/logging.h" #include "common/posix_client/shared_client/shared_client_manager.h" #include "common/posix_client/shared_client/posix_stream_manager_proxy.h" -#include "proto/pb/posix/bus_service.grpc.pb.h" -#include "proto/pb/posix/bus_service.pb.h" -#include "proto/pb/posix_pb.h" -#include "status/status.h" +#include "common/proto/pb/posix/bus_service.grpc.pb.h" +#include "common/proto/pb/posix/bus_service.pb.h" +#include "common/proto/pb/posix_pb.h" +#include "common/status/status.h" +#include "function_proxy/common/iam/internal_iam.h" #include "gmock/gmock-actions.h" #include "gmock/gmock-generated-actions.h" #include "gmock/gmock-more-actions.h" @@ -55,6 +56,9 @@ class GrpcServerTest : public ::testing::Test { public: void SetUp() override { + functionsystem::function_proxy::InternalIAM::Param param; + param.isEnableIAM = false; + internalIAM_ = std::make_shared(param); controlPlaneObserver_ = std::make_shared(); sharedClientMgr_ = std::make_shared("SharedPosixClientMgr"); litebus::Spawn(sharedClientMgr_); @@ -76,6 +80,7 @@ public: auto nodeID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); instanceCtrl_ = InstanceCtrl::Create(nodeID, instanceCtrlconfig); instanceCtrl_->Start(funcAgentMgr_, resourceViewMgr_, controlPlaneObserver_); + instanceCtrl_->BindInternalIAM(internalIAM_); mockLocalSchedSrv_ = std::make_shared(); } void TearDown() override @@ -83,6 +88,7 @@ public: litebus::Terminate(sharedClientMgr_->GetAID()); litebus::Await(sharedClientMgr_->GetAID()); + internalIAM_ = nullptr; controlPlaneObserver_ = nullptr; resourceViewMgr_ = nullptr; funcAgentMgr_ = nullptr; @@ -97,6 +103,7 @@ protected: std::shared_ptr sharedPosixClientManager_; std::shared_ptr instanceCtrl_; std::shared_ptr mockLocalSchedSrv_; + std::shared_ptr internalIAM_{ nullptr }; std::shared_ptr sharedClientMgr_; std::shared_ptr funcAgentMgr_; @@ -131,6 +138,7 @@ TEST_F(GrpcServerTest, DiscoverDriverStatus) auto mockInstanceCtrl = std::make_shared( std::make_shared("mockInstanceCtrl", "nodeID", instanceCtrlconfig)); mockInstanceCtrl->BindControlInterfaceClientManager(mockControlInterfaceClientManagerProxy); + mockInstanceCtrl->BindInternalIAM(internalIAM_); litebus::Future statusFut; statusFut.SetValue(Status()); EXPECT_CALL(*controlPlaneObserver_, PutInstance).WillOnce(Return(statusFut)); @@ -149,7 +157,7 @@ TEST_F(GrpcServerTest, DiscoverDriverStatus) .instanceCtrl = mockInstanceCtrl, .localSchedSrv = mockLocalSchedSrv_, .isEnableServerMode = true, - .hostIP = "10.27.15.58", + .hostIP = "127.0.0.1", }; EXPECT_CALL(*mockLocalSchedSrv_, IsRegisteredToGlobal).WillRepeatedly(Return(Status::OK())); auto service = BusService(std::move(param)); @@ -158,7 +166,7 @@ TEST_F(GrpcServerTest, DiscoverDriverStatus) ::grpc::Status status = service.DiscoverDriver(&context, &request, &response); EXPECT_EQ(status.ok(), true); EXPECT_EQ(response.nodeid(), "nodeID"); - EXPECT_EQ(response.hostip(), "10.27.15.58"); + EXPECT_EQ(response.hostip(), "127.0.0.1"); // 3. Mock PutInstance returning Status with error code. statusFut = litebus::Future(); diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_actor_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_actor_test.cpp index 0f9e7f85da123a0203097dc41cb1d3c61a3ae86b..a7e759d238340730b6b2c9f84e95e9a0e05d9451 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_actor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_actor_test.cpp @@ -13,29 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "constants.h" -#include "logs/logging.h" +#include "common/constants/constants.h" +#include "common/logs/logging.h" #include "function_proxy/local_scheduler/instance_control/instance_ctrl_actor.h" #include #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" -#include "metrics/metrics_adapter.h" -#include "http/http_server.h" -#include "rpc/server/common_grpc_server.h" +#include "common/metrics/metrics_adapter.h" +#include "common/http/http_server.h" +#include "common/rpc/server/common_grpc_server.h" #include "common/scheduler_framework/utils/label_affinity_selector.h" -#include "files.h" +#include "common/utils/files.h" #include "common/utils/struct_transfer.h" -#include "hex/hex.h" +#include "common/hex/hex.h" +#include "function_proxy/common/iam/internal_iam.h" #include "function_proxy/common/posix_client/shared_client/posix_stream_manager_proxy.h" #include "function_proxy/common/posix_client/shared_client/shared_client_manager.h" +#include "function_proxy/local_scheduler/instance_control/instance_ctrl_message.h" #include "httpd/http.hpp" #include "mocks/mock_function_agent_mgr.h" #include "mocks/mock_instance_control_view.h" #include "mocks/mock_instance_state_machine.h" #include "mocks/mock_observer.h" #include "mocks/mock_runtime_client.h" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_cloud_api_gateway.h" #include "mocks/mock_scheduler.h" #include "mocks/mock_local_sched_srv.h" @@ -51,7 +54,7 @@ using namespace functionsystem::grpc; using namespace functionsystem::function_proxy; using namespace testing; namespace { -const std::string RESOURCE_DIR = "/home/lwy/sn/resource"; +const std::string RESOURCE_DIR = "/home/sn/resource"; const std::string A_TXT_CONTENT = "f48f9d5a9706088947ac438ebe005aa26c9370579f2231c538b28894a315562182da0eb18002c86728c4cdc0df5efb19e1c2060e93370fd891d4f3d9e5b2b61376643f86d0210ce996446a985759b15112037a5a2f6463cf5fd6afc7ff30fe814bf960eb0c16c5059407c74d6a93a8b3110405cbc935dff672da3b648d62e0d5cecd91bc7063211e6b33210afb6899e8322eabffe167318a5ac5d591aa7579efd37e9e4c7fcf390e97c1151b7c1bf00b4a18764a1a0cac1fda1ea6389b39d755127f0e5bc072e6d5936738be1585535dc63b71ad58686f71c821325009de36bdbac31c1c044845bd1bb41230ec9815695ef3f9e7143a16410113ff3286147a76"; const std::string B_TXT_CONTENT = @@ -74,16 +77,9 @@ const std::string GRPC_SERVER_IP = "127.0.0.1"; const std::string HTTP_SERVER_NAME = "v3.0"; -// OS path -const std::string OIDC_TOKEN_DIR = "/var/run/secrets/tokens/"; -const std::string OIDC_TOKEN_PATH = "/var/run/secrets/tokens/oidc-token"; -const std::string MOCK_OIDC_TOKEN_CONTENT = "test_oidc_token"; -const std::string MOCK_IAM_TOKEN = "mock-iam-token"; - // sub-pub const std::string SUBSCRIBER_ID = "subscriber"; const std::string PUBLISHER_ID = "publisher"; - } class MockUtilClass { @@ -101,7 +97,7 @@ class InstanceCtrlActorTest : public ::testing::Test { public: inline static std::string metaStoreServerHost_; inline static uint16_t grpcServerPort_; - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { int metaStoreServerPort = functionsystem::test::FindAvailablePort(); metaStoreServerHost_ = "127.0.0.1:" + std::to_string(metaStoreServerPort); @@ -145,6 +141,10 @@ public: etcdSrvDriver_->StartServer(metaStoreServerHost_); metaStoreClient_ = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); auto metaStorageAccessor = std::make_shared(metaStoreClient_); + param_.policyPath = RESOURCE_DIR; + mockInternalIAM_ = std::make_shared(param_); + mockInternalIAM_->BindMetaStorageAccessor(metaStorageAccessor); + EXPECT_TRUE(mockInternalIAM_->Init()); CommonGrpcServerConfig serverConfig; serverConfig.ip = GRPC_SERVER_IP; @@ -152,6 +152,7 @@ public: serverConfig.creds = ::grpc::InsecureServerCredentials(); server_ = std::make_shared(serverConfig); posixService_ = std::make_shared(); + posixService_->BindInternalIAM(mockInternalIAM_); server_->RegisterService(posixService_); server_->Start(); ASSERT_TRUE(server_->WaitServerReady()); @@ -186,6 +187,8 @@ public: instanceCtrlConfig_); instanceCtrlActor_->BindMetaStoreClient(metaStoreClient_); + EXPECT_CALL(*mockInternalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); + instanceCtrlActor_->BindInternalIAM(mockInternalIAM_); instanceCtrlActor_->BindFunctionAgentMgr(mockFunctionAgentMgr_); instanceCtrlActor_->BindObserver(mockObserver_); instanceCtrlActor_->BindInstanceControlView(mockInstanceCtrlView_); @@ -202,6 +205,7 @@ public: litebus::Terminate(instanceCtrlActor_->GetAID()); litebus::Await(instanceCtrlActor_); instanceCtrlActor_= nullptr; + mockInternalIAM_ = nullptr; litebus::Terminate(sharedClientManager_->GetAID()); litebus::Await(sharedClientManager_); sharedClientManager_ = nullptr; @@ -242,6 +246,8 @@ public: protected: InstanceCtrlConfig instanceCtrlConfig_; bool isResourceExisted_{ false }; + InternalIAM::Param param_{ true, "" }; + std::shared_ptr mockInternalIAM_; std::unique_ptr etcdSrvDriver_; std::shared_ptr metaStoreClient_; std::shared_ptr instanceCtrlActor_; @@ -328,6 +334,40 @@ TEST_F(InstanceCtrlActorTest, StopAppDriver) instanceCtrlActor_->StopAppDriver(killContext); } +TEST_F(InstanceCtrlActorTest, AuthorizeKillTest) +{ + EXPECT_CALL(*mockInternalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); + EXPECT_CALL(*mockInternalIAM_, GetCredType).WillRepeatedly(Return(function_proxy::IAMCredType::TOKEN)); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("", nullptr).Get().IsOk()); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("job-killer-123", nullptr).Get().IsOk()); + auto mockInstanceStateMachine = std::make_shared("TEST_PROXY_ID"); + EXPECT_CALL(*mockInstanceCtrlView_, GetInstance).WillOnce(Return(nullptr)).WillOnce(Return(mockInstanceStateMachine)) + .WillOnce(Return(nullptr)).WillRepeatedly(Return(mockInstanceStateMachine)); + InstanceInfo ins1; + ins1.set_tenantid("tenant002"); + ins1.set_instanceid(TEST_INSTANCE_ID); + auto killReq = std::make_shared(); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("ins-123", killReq).Get().IsError()); + EXPECT_CALL(*mockInstanceStateMachine, GetInstanceInfo).WillOnce(Return(ins1)); + EXPECT_TRUE( + instanceCtrlActor_->AuthorizeKill("ins-123", killReq).Get().StatusCode() == StatusCode::ERR_INSTANCE_NOT_FOUND); + + InstanceInfo emptyTenant1; + emptyTenant1.set_instanceid(TEST_INSTANCE_ID); + EXPECT_CALL(*mockInstanceStateMachine, GetInstanceInfo).WillOnce(Return(emptyTenant1)); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("ins-123", killReq).Get().IsOk()); + EXPECT_CALL(*mockInstanceStateMachine, GetInstanceInfo).WillOnce(Return(ins1)).WillOnce(Return(emptyTenant1)); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("ins-123", killReq).Get().IsOk()); + InstanceInfo ins; + ins.set_tenantid("tenant002"); + ins.set_instanceid(TEST_INSTANCE_ID); + ins.set_issystemfunc(true); + EXPECT_CALL(*mockInstanceStateMachine, GetInstanceInfo).WillRepeatedly(Return(ins)); + EXPECT_CALL(*mockInternalIAM_, Authorize).WillOnce(Return(Status::OK())); + EXPECT_TRUE(instanceCtrlActor_->AuthorizeKill("ins-123", killReq).Get().IsOk()); + +} + TEST_F(InstanceCtrlActorTest, SetTenantAffinityOpt_instance) { auto scheduleReq = std::make_shared(); @@ -496,7 +536,7 @@ TEST_F(InstanceCtrlActorTest, SetInstanceBillingContext) * SetScheduleReqConfigSuccess * Test Set ScheduleReq config successfully * Steps: - * 1. execute SetScheduleReqFunctionAgentIDAndHeteroConfig and set ScheduleReq + * 1. execute MergeScheduleResultToRequest and set ScheduleReq * * Expectations: * 1. set ScheduleReq successfully @@ -526,7 +566,7 @@ TEST_F(InstanceCtrlActorTest, SetScheduleReqConfigSuccess) (*cg.mutable_vectors())["uuid"].add_values(1010); } - SetScheduleReqFunctionAgentIDAndHeteroConfig(scheduleReq, result); + MergeScheduleResultToRequest(scheduleReq, result); ASSERT_TRUE(scheduleReq->mutable_instance()->functionagentid() == "agent-id-0"); EXPECT_EQ(scheduleReq->instance().schedulerchain().size(), 1); EXPECT_EQ(scheduleReq->instance().schedulerchain().Get(0), "agent-id-0"); @@ -554,7 +594,7 @@ TEST_F(InstanceCtrlActorTest, ShutdownWithNoInstanceClient) inst.set_instanceid(id); inst.set_requestid(id2); - metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(id, std::map{}); + metrics::MetricsAdapter::GetInstance().GetMetricsContext().InitBillingInstance(id, "", std::map{}); auto res = instanceCtrlActor_->ShutDownInstance(inst, 10); EXPECT_EQ(res.Get(), Status::OK()); auto endTime = metrics::MetricsAdapter::GetInstance().GetMetricsContext().GetBillingInstance(id).endTimeMillis; @@ -562,7 +602,6 @@ TEST_F(InstanceCtrlActorTest, ShutdownWithNoInstanceClient) EXPECT_TRUE(endTime != 0); metrics::MetricsAdapter::GetInstance().GetMetricsContext().EraseBillingInstance(); } - /** * RetryForwardSchedule * Test is transition version is incorrect, and retry RetryForwardSchedule @@ -659,11 +698,14 @@ TEST_F(InstanceCtrlActorTest, TryDispatchOnLocal) result.savedInfo = instanceInfoSaved; EXPECT_CALL(*mockInstanceStateMachine, TransitionToImpl(_, _, _, _, _)).WillRepeatedly(Return(result)); bool isCalled = false; - EXPECT_CALL(*mockScheduler, ScheduleConfirm).WillOnce(Return(Status::OK())) - .WillOnce(Return(Status::OK())).WillOnce(DoAll(Assign(&isCalled, true), Return(Status::OK()))); // mock schedule successfully + EXPECT_CALL(*mockScheduler, ScheduleConfirm) + .WillOnce(Return(Status::OK())) + .WillOnce(Return(Status::OK())) + .WillOnce(DoAll(Assign(&isCalled, true), Return(Status::OK()))); // mock schedule successfully // test instance parentfunctionproxyaid is empty - auto future = instanceCtrlActor_->TryDispatchOnLocal(status, scheduleRequest, scheduleResult, InstanceState::SCHEDULING, mockInstanceStateMachine); + auto future = instanceCtrlActor_->TryDispatchOnLocal(status, scheduleRequest, scheduleResult, + InstanceState::SCHEDULING, mockInstanceStateMachine); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get().code(), StatusCode::INSTANCE_TRANSACTION_WRONG_VERSION); @@ -775,6 +817,7 @@ TEST_F(InstanceCtrlActorTest, CancelSchedule) request->set_requestid(TEST_REQUEST_ID); request->mutable_instance()->CopyFrom(info); EXPECT_CALL(*mockInstanceStateMachine, GetScheduleRequest).WillRepeatedly(Return(request)); + EXPECT_CALL(*mockInternalIAM_, IsIAMEnabled).WillOnce(Return(false)).WillOnce(Return(false)); InstanceState state; EXPECT_CALL(*mockInstanceStateMachine, TransitionToImpl) .WillOnce(DoAll(testing::SaveArg<0>(&state), Return(TransitionResult{ litebus::None(), info, info }))); @@ -808,7 +851,7 @@ TEST_F(InstanceCtrlActorTest, CancelSchedule) */ TEST_F(InstanceCtrlActorTest, RetryNotificationSignal) { auto scheduleReq = std::make_shared(); - scheduleReq->mutable_instance()->mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + scheduleReq->mutable_instance()->mutable_instancestatus()->set_code(static_cast(InstanceState::RUNNING)); scheduleReq->mutable_instance()->set_functionproxyid(TEST_NODE_ID); scheduleReq->mutable_instance()->set_instanceid(SUBSCRIBER_ID); scheduleReq->set_requestid("requestId"); @@ -855,4 +898,135 @@ TEST_F(InstanceCtrlActorTest, RetryNotificationSignal) { EXPECT_EQ(response.code(), common::ErrorCode::ERR_NONE); } +TEST_F(InstanceCtrlActorTest, GetStaticFunctionChangeRequest) { + InstanceInfo info; + info.set_instanceid("test-instanceID"); + info.set_requestid("test-requestID"); + auto req = GetStaticFunctionChangeRequest(info, 0); + EXPECT_EQ(req->instanceid(), info.instanceid()); + EXPECT_EQ(req->requestid(), info.requestid()); + EXPECT_EQ(req->status(), 0); +} + +TEST_F(InstanceCtrlActorTest, CheckExistInstanceStateTest) +{ + auto runtimePromise = std::make_shared>(); + auto scheduleReq = std::make_shared(); + + // running but instance is not existed + auto instanceControlView = std::make_shared("nodeID"); + instanceCtrlActor_->instanceControlView_ = instanceControlView; + EXPECT_CALL(*instanceControlView, GetInstance).WillOnce(Return(nullptr)); + + auto result = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::CheckExistInstanceState, + InstanceState::RUNNING, runtimePromise, scheduleReq, false); + ASSERT_AWAIT_READY(result); + EXPECT_FALSE(result.Get()); + + // running local instance + runtimePromise = std::make_shared>(); + scheduleReq = std::make_shared(); + + auto mockInstanceStateMachine = std::make_shared("machine1"); + EXPECT_CALL(*instanceControlView, GetInstance).WillOnce(Return(mockInstanceStateMachine)); + EXPECT_CALL(*mockInstanceStateMachine, GetOwner).WillOnce(Return(TEST_NODE_ID)); + + result = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::CheckExistInstanceState, + InstanceState::RUNNING, runtimePromise, scheduleReq, false); + ASSERT_AWAIT_READY(runtimePromise->GetFuture()); + EXPECT_EQ(runtimePromise->GetFuture().Get().code(), static_cast(StatusCode::ERR_INSTANCE_DUPLICATED)); + + // running remote instance, aid ok + EXPECT_CALL(*mockObserver_, GetLocalSchedulerAID) + .WillOnce(Return(litebus::Option(instanceCtrlActor_->GetAID()))); + EXPECT_CALL(*instanceControlView, GetInstance) + .WillOnce(Return(mockInstanceStateMachine)) + .WillOnce(Return(mockInstanceStateMachine)) + .WillOnce(Return(mockInstanceStateMachine)); + EXPECT_CALL(*mockInstanceStateMachine, GetOwner) + .WillOnce(Return("remote_owner")) + .WillOnce(Return("remote_owner")) + .WillOnce(Return("remote_owner")) + .WillOnce(Return(TEST_NODE_ID)); + EXPECT_CALL(*mockInstanceStateMachine, GetInstanceState).WillOnce(Return(InstanceState::FATAL)); + + runtimePromise = std::make_shared>(); + result = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::CheckExistInstanceState, + InstanceState::RUNNING, runtimePromise, scheduleReq, false); + ASSERT_AWAIT_READY(runtimePromise->GetFuture()); + EXPECT_EQ(runtimePromise->GetFuture().Get().code(), static_cast(StatusCode::ERR_INSTANCE_EXITED)); +} + +TEST_F(InstanceCtrlActorTest, ExitWithoutCodeTest) +{ + auto request = std::make_shared(); + request->set_code(common::ERR_NONE); + auto future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ErrorCode::ERR_NONE); +} + +TEST_F(InstanceCtrlActorTest, ExitInvalidCodeTest) +{ + auto request = std::make_shared(); + request->set_code(common::ERR_REQUEST_BETWEEN_RUNTIME_BUS); // invalid code + auto future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_PARAM_INVALID); + EXPECT_EQ(future.Get().message(), "invalid exit code"); +} + +TEST_F(InstanceCtrlActorTest, ExitWithNPUFaultTest) +{ + auto mockInstanceCtrlView = std::make_shared("TEST_NODE_ID"); + instanceCtrlActor_->instanceControlView_ = mockInstanceCtrlView; + + auto stateMachine = std::make_shared("TEST_NODE_ID"); + auto &mockStateMachine = *stateMachine; + + // instance not found + EXPECT_CALL(*mockInstanceCtrlView, GetInstance).WillOnce(Return(nullptr)); + auto request = std::make_shared(); + request->set_code(common::ERR_NPU_FAULT_ERROR); + auto future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_INSTANCE_NOT_FOUND); + EXPECT_EQ(future.Get().message(), "instance not found"); + + // instance not on this node + EXPECT_CALL(*mockInstanceCtrlView, GetInstance).WillOnce(Return(stateMachine)); + EXPECT_CALL(mockStateMachine, GetOwner).WillOnce(Return("fake node")); + future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_INSTANCE_NOT_FOUND); + EXPECT_EQ(future.Get().message(), "instance not found"); + + // invalid state + EXPECT_CALL(*mockInstanceCtrlView, GetInstance).WillRepeatedly(Return(stateMachine)); + EXPECT_CALL(mockStateMachine, GetOwner).WillRepeatedly(Return("TEST_NODE_ID")); + EXPECT_CALL(mockStateMachine, GetInstanceState).WillOnce(Return(InstanceState::FATAL)); + future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(future.Get().message(), "invalid status, instance is not running"); + + // instance is creating + EXPECT_CALL(mockStateMachine, GetInstanceState).WillOnce(Return(InstanceState::CREATING)); + EXPECT_CALL(mockStateMachine, AddStateChangeCallback).WillOnce(Return()); + future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_NONE); + + // running -> sub-health + EXPECT_CALL(mockStateMachine, GetInstanceState).WillOnce(Return(InstanceState::RUNNING)); + EXPECT_CALL(mockStateMachine, IsSaving).WillOnce(Return(false)); + InstanceInfo info; + EXPECT_CALL(mockStateMachine, GetInstanceInfo).WillOnce(Return(info)); + EXPECT_CALL(mockStateMachine, TransitionToImpl(InstanceState::SUB_HEALTH, _, _, _, ERR_NPU_FAULT_ERROR)) + .WillOnce(Return(TransitionResult{ InstanceState::SUB_HEALTH, InstanceInfo(), InstanceInfo(), 3 })); + future = litebus::Async(instanceCtrlActor_->GetAID(), &InstanceCtrlActor::HandleExit, "InstanceA", request); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.Get().code(), common::ERR_NONE); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_test.cpp index a39c661df6d53355693acdf0a6743ad2dadb2b78..afa71ec51d350cb1ae388e06079f031c36ae5d71 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/instance_ctrl_test.cpp @@ -20,11 +20,11 @@ #include "common/constants/actor_name.h" #include "common/constants/signal.h" #include "common/etcd_service/etcd_service_driver.h" -#include "logs/logging.h" -#include "metadata/metadata.h" -#include "proto/pb/message_pb.h" -#include "proto/pb/posix_pb.h" -#include "resource_type.h" +#include "common/logs/logging.h" +#include "common/metadata/metadata.h" +#include "common/proto/pb/message_pb.h" +#include "common/proto/pb/posix_pb.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" #include "common/utils/generate_message.h" #include "function_proxy/common/posix_client/shared_client/shared_client_manager.h" @@ -37,6 +37,7 @@ #include "mocks/mock_instance_control_view.h" #include "mocks/mock_instance_operator.h" #include "mocks/mock_instance_state_machine.h" +#include "mocks/mock_internal_iam.h" #include "mocks/mock_local_instance_ctrl_actor.h" #include "mocks/mock_local_sched_srv.h" #include "mocks/mock_meta_store_client.h" @@ -168,9 +169,14 @@ public: auto sharedPosixClientManager = std::make_shared(sharedClientMgr_->GetAID()); auto metaClient = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); metaStorageAccessor_ = std::make_shared(metaClient); + InternalIAM::Param internalIAM_Param; + internalIAM_Param.isEnableIAM = false; observerActor_ = std::make_shared( FUNCTION_PROXY_OBSERVER_ACTOR_NAME, nodeID_, metaStorageAccessor_, function_proxy::ObserverParam{}); + mockInternalIAM_ = std::make_shared(internalIAM_Param); observerActor_->BindDataInterfaceClientManager(sharedPosixClientManager); + observerActor_->BindInternalIAM(mockInternalIAM_); + EXPECT_CALL(*mockInternalIAM_, IsSystemTenant).WillRepeatedly(testing::Return(false)); litebus::Spawn(observerActor_); litebus::Async(observerActor_->GetAID(), &function_proxy::ObserverActor::Register); @@ -188,6 +194,7 @@ public: instanceCtrl_->BindObserver(observer_); instanceCtrl_->BindFunctionAgentMgr(funcAgentMgr_); instanceCtrl_->BindResourceView(resourceViewMgr_); + instanceCtrl_->BindInternalIAM(mockInternalIAM_); mockSharedClientManagerProxy_ = std::make_shared(); instanceCtrl_->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); @@ -196,6 +203,7 @@ public: instanceCtrlWithMockObserver_->BindInstanceControlView(instanceControlView_); instanceCtrlWithMockObserver_->Start(funcAgentMgr_, resourceViewMgr_, mockObserver_); instanceCtrlWithMockObserver_->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); + instanceCtrlWithMockObserver_->BindInternalIAM(mockInternalIAM_); resource_view::Resources metaResources; resource_view::Resource resource; resource.set_type(resource_view::ValueType::Value_Type_SCALAR); @@ -238,13 +246,14 @@ public: observer_ = nullptr; mockObserver_ = nullptr; funcAgentMgr_ = nullptr; + mockInternalIAM_ = nullptr; mockResourceViewMgr_ = nullptr; primary_ = nullptr; virtual_ = nullptr; } protected: - [[maybe_unused]] static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { etcdSrvDriver_ = std::make_unique(); int metaStoreServerPort = functionsystem::test::FindAvailablePort(); @@ -253,7 +262,7 @@ protected: InstanceCtrlActor::SetGetLocalInterval(100); } - [[maybe_unused]] static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { etcdSrvDriver_->StopServer(); } @@ -267,6 +276,7 @@ protected: std::shared_ptr primary_; std::shared_ptr virtual_; std::shared_ptr mockSharedClientManagerProxy_; + std::shared_ptr mockInternalIAM_; std::shared_ptr sharedClientMgr_; std::shared_ptr metaStorageAccessor_; std::shared_ptr observer_; @@ -400,6 +410,72 @@ TEST_F(InstanceCtrlTest, ScheduleInvalidRequest) } } +TEST_F(InstanceCtrlTest, ScheduleValidDiskRequest) +{ + auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); + auto instanceCtrl = InstanceCtrl(actor); + auto observer = std::make_shared(); + instanceCtrl.Start(nullptr, mockResourceViewMgr_, observer); + ASSERT_TRUE(observer != nullptr); + EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(FunctionMeta{})); + + EXPECT_CALL(*observer, IsSystemFunction).WillRepeatedly(Return(false)); + + auto scheduleReq = std::make_shared(); + (*scheduleReq->mutable_instance()) = view_utils::Get1DInstance(); + auto runtimePromise = std::make_shared>(); + + // a valid request -- disk zise is +100 + + resources::Resource validDisk; + validDisk.mutable_scalar()->set_value(100); + scheduleReq->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + validDisk; + runtimePromise = std::make_shared>(); + auto result = instanceCtrl.Schedule(scheduleReq, runtimePromise); + ASSERT_AWAIT_READY(result); + EXPECT_EQ(result.Get().message(), "resources is invalid"); +} + +TEST_F(InstanceCtrlTest, ScheduleInvalidDiskRequest) +{ + auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); + auto instanceCtrl = InstanceCtrl(actor); + auto observer = std::make_shared(); + instanceCtrl.Start(nullptr, mockResourceViewMgr_, observer); + ASSERT_TRUE(observer != nullptr); + EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(FunctionMeta{})); + + EXPECT_CALL(*observer, IsSystemFunction).WillRepeatedly(Return(false)); + + auto scheduleReq = std::make_shared(); + (*scheduleReq->mutable_instance()) = view_utils::Get1DInstance(); + auto runtimePromise = std::make_shared>(); + + // a invalid request -- disk zise is 0 + { + resources::Resource invalidDisk; + invalidDisk.mutable_scalar()->set_value(0); + scheduleReq->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + invalidDisk; + runtimePromise = std::make_shared>(); + auto result = instanceCtrl.Schedule(scheduleReq, runtimePromise); + ASSERT_AWAIT_READY(result); + EXPECT_EQ(result.Get().message(), "resources is invalid"); + } + // a invalid request -- disk zise is -100 + { + resources::Resource invalidDisk; + invalidDisk.mutable_scalar()->set_value(-100); + scheduleReq->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + invalidDisk; + runtimePromise = std::make_shared>(); + auto result = instanceCtrl.Schedule(scheduleReq, runtimePromise); + ASSERT_AWAIT_READY(result); + EXPECT_EQ(result.Get().message(), "resources is invalid"); + } +} + TEST_F(InstanceCtrlTest, ScheduleExistInstance) { auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); @@ -416,6 +492,7 @@ TEST_F(InstanceCtrlTest, ScheduleExistInstance) auto stateMachine = std::make_shared("nodeID"); auto &mockStateMachine = *stateMachine; EXPECT_CALL(*instanceControlView, GetInstance).WillRepeatedly(Return(stateMachine)); + EXPECT_CALL(mockStateMachine, GetOwner()).WillRepeatedly(Return("nodeID")); auto scheduleReq = std::make_shared(); scheduleReq->mutable_instance()->set_instanceid("123"); @@ -483,6 +560,7 @@ TEST_F(InstanceCtrlTest, DeployInstanceRetry) auto observer = std::make_shared(); auto instanceCtrl = std::make_shared(actor); instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); litebus::Future agentIdFut; { InSequence s; @@ -529,7 +607,7 @@ TEST_F(InstanceCtrlTest, DeployInstanceRetry) auto failedAllocated = std::make_shared>(); failedAllocated->SetValue(Status(StatusCode::FAILED)); EXPECT_CALL(*scheduler, ScheduleDecision(_)) - .WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "", {}, "", {}, failedAllocated})) + .WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "", {}, "", {}, {}, failedAllocated})) .WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "" })); EXPECT_CALL(*scheduler, ScheduleConfirm).WillOnce(Return(Status::OK())).WillOnce(Return(Status::OK())); instanceCtrl->BindScheduler(scheduler); @@ -594,13 +672,17 @@ TEST_F(InstanceCtrlTest, ScheduleCancelAfterScheduling) EXPECT_CALL(*instanceControlView, TryGenerateNewInstance).WillOnce(Return(genStates)); EXPECT_CALL(*instanceControlView, GetInstance).WillRepeatedly(Return(stateMachine)); EXPECT_CALL(mockStateMachine, IsSaving()).WillRepeatedly(Return(false)); + EXPECT_CALL(mockStateMachine, GetInstanceState()) + .WillOnce(Return(InstanceState::SCHEDULING)) + .WillOnce(Return(InstanceState::SCHEDULING)); EXPECT_CALL(mockStateMachine, TransitionToImpl).WillOnce(Return(NEW_RESULT)).WillOnce(Return(FAILED_RESULT)); auto cancelFuture = litebus::Future(); cancelFuture.SetValue("cancel"); EXPECT_CALL(mockStateMachine, GetCancelFuture).WillOnce(Return(cancelFuture)); auto scheduler = std::make_shared(); - EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::RESOURCE_NOT_ENOUGH, ""})); + EXPECT_CALL(*scheduler, ScheduleDecision(_)) + .WillOnce(Return(ScheduleResult{ "", StatusCode::RESOURCE_NOT_ENOUGH, "" })); instanceCtrl.BindScheduler(scheduler); auto scheduleReq = std::make_shared(); @@ -620,6 +702,7 @@ TEST_F(InstanceCtrlTest, ScheduleCancelAfterCreating) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillOnce(Return(functionMeta_)); + instanceCtrl->BindInternalIAM(mockInternalIAM_); instanceCtrl->BindFunctionAgentMgr(funcAgentMgr_); auto stateMachine = std::make_shared("nodeID"); @@ -686,10 +769,12 @@ TEST_F(InstanceCtrlTest, CreateInstanceFailedForResourceNotEnough) ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully auto scheduler = std::make_shared(); // mock schedule failed - EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::RESOURCE_NOT_ENOUGH, "" })); + EXPECT_CALL(*scheduler, ScheduleDecision(_)) + .WillOnce(Return(ScheduleResult{ "", StatusCode::RESOURCE_NOT_ENOUGH, "" })); instanceCtrl->BindScheduler(scheduler); auto metaClient = MetaStoreClient::Create({ .etcdAddress = metaStoreServerHost_ }); @@ -709,7 +794,9 @@ TEST_F(InstanceCtrlTest, CreateInstanceFailedForResourceNotEnough) ASSERT_AWAIT_READY(result); EXPECT_EQ(result.Get().code(), StatusCode::RESOURCE_NOT_ENOUGH); - ASSERT_AWAIT_TRUE([&]() { return scheduleReq->instance().instancestatus().code() == static_cast(InstanceState::SCHEDULE_FAILED); }); + ASSERT_AWAIT_TRUE([&]() { + return scheduleReq->instance().instancestatus().code() == static_cast(InstanceState::SCHEDULE_FAILED); + }); auto machine = instanceControlView->GetInstance("DesignatedInstanceID"); ASSERT_AWAIT_TRUE([&]() { return machine->GetInstanceState() == InstanceState::SCHEDULE_FAILED; }); @@ -756,6 +843,7 @@ TEST_F(InstanceCtrlTest, CreateInstanceFailedForDeployInstanceFailed) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "" })); @@ -835,6 +923,8 @@ TEST_F(InstanceCtrlTest, CreateInstanceFailedForInitRuntimeFailed) ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully + auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "" })); EXPECT_CALL(*scheduler, ScheduleConfirm).Times(1); // mock schedule successfully @@ -927,10 +1017,11 @@ TEST_F(InstanceCtrlTest, CreateInstanceSuccess) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, ScheduleDecision(_)) - .WillOnce(Return(ScheduleResult{ "agent", StatusCode::SUCCESS, "", {}, "", {}, nullptr, "bundleUnit" })); + .WillOnce(Return(ScheduleResult{ "agent", StatusCode::SUCCESS, "", {}, "", {}, {}, nullptr, "bundleUnit" })); EXPECT_CALL(*scheduler, ScheduleConfirm).Times(1); // mock schedule successfully instanceCtrl->BindScheduler(scheduler); @@ -982,6 +1073,7 @@ TEST_F(InstanceCtrlTest, CreateInstanceSuccess) ASSERT_AWAIT_TRUE([&]() { return machine->GetInstanceState() == InstanceState::RUNNING; }); ASSERT_AWAIT_TRUE([&]() { return scheduleReq->instance().unitid() == "bundleUnit"; }); ASSERT_AWAIT_TRUE([&]() { return machine->GetInstanceInfo().unitid() == "bundleUnit"; }); + EXPECT_TRUE(mockInternalIAM_->VerifyInstance("DesignatedInstanceID")); litebus::Terminate(stateActor->GetAID()); litebus::Await(stateActor->GetAID()); @@ -1011,6 +1103,7 @@ TEST_F(InstanceCtrlTest, ScheduleSuccess) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); + instanceCtrl->BindInternalIAM(mockInternalIAM_); resources::InstanceInfo instanceInfo; instanceInfo.set_parentfunctionproxyaid(actor->GetAID()); instanceInfo.set_parentid("parent"); @@ -1137,6 +1230,7 @@ TEST_F(InstanceCtrlTest, ScheduleRecoverInstanceSuccess) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); + instanceCtrl->BindInternalIAM(mockInternalIAM_); resources::InstanceInfo instanceInfo; instanceInfo.set_functionproxyid("nodeID"); instanceInfo.set_instanceid("DesignatedInstanceID"); @@ -1701,7 +1795,6 @@ TEST_F(InstanceCtrlTest, ForwardCustomSignalRequestDuplicate) instanceCtrlActor->ToReady(); litebus::Spawn(instanceCtrlActor); auto stateMachine = std::make_shared("proxyID1"); - auto &mockStateMachine = *stateMachine; EXPECT_CALL(*instanceControlView_, GetInstance).WillRepeatedly(Return(stateMachine)); const std::string srcInstance = "srcInstance"; @@ -1755,7 +1848,7 @@ TEST_F(InstanceCtrlTest, SendForwardCustomSignalRequestDuplicate) litebus::Spawn(instanceCtrlActor); auto killReq = GenKillRequest(instanceID1, customSignal); - auto requestID(killReq->instanceid() + "-" + std::to_string(killReq->signal())); + auto requestID(killReq->requestid() + "-" + std::to_string(killReq->signal())); auto notifyPromise = std::make_shared>(); instanceCtrlActor->forwardCustomSignalNotifyPromise_.emplace(requestID, notifyPromise); // mock promise is exist, don't execute @@ -2259,6 +2352,7 @@ TEST_F(InstanceCtrlTest, CreateInstanceClientTest) auto observer = std::make_shared(); auto instanceCtrl = std::make_shared(actor); instanceCtrl->Start(nullptr, mockResourceViewMgr_, mockObserver_); + instanceCtrl->BindInternalIAM(mockInternalIAM_); instanceCtrl->BindFunctionAgentMgr(funcAgentMgr_); scheduleReqA->mutable_instance()->set_parentfunctionproxyaid(actor->GetAID()); @@ -2539,8 +2633,12 @@ TEST_F(InstanceCtrlTest, SyncInstanceRecoverFailed) * 2. send request of sync instances. * Expectation: invoke Recover method second times and check consistency successfully. */ -TEST_F(InstanceCtrlTest, DISABLED_SyncInstanceRecoverSuccess) +TEST_F(InstanceCtrlTest, SyncInstanceRecoverSuccess) { + InternalIAM::Param internalIAM_Param; + internalIAM_Param.isEnableIAM = true; + auto mockInternalIAM = std::make_shared(internalIAM_Param); + instanceCtrlWithMockObserver_->BindInternalIAM(mockInternalIAM); function_proxy::InstanceInfoMap instanceInfoMap; resource_view::InstanceInfo instanceInfo; instanceInfo.set_instanceid("instance1"); @@ -2585,7 +2683,8 @@ TEST_F(InstanceCtrlTest, DISABLED_SyncInstanceRecoverSuccess) EXPECT_CALL(*localSchedSrv, ForwardSchedule).WillRepeatedly(Return(scheduleResponse)); instanceCtrlWithMockObserver_->BindLocalSchedSrv(localSchedSrv); EXPECT_CALL(*funcAgentMgr_.get(), KillInstance(testing::_, testing::_, testing::_)) - .WillRepeatedly(Return(GenKillInstanceResponse(StatusCode::SUCCESS, "kill instance successfully", "requestID"))); + .WillRepeatedly( + Return(GenKillInstanceResponse(StatusCode::SUCCESS, "kill instance successfully", "requestID"))); EXPECT_CALL(*distributedCacheClient, Get(Matcher("instance1"), Matcher(Eq("")))) .WillRepeatedly(DoAll(SetArgReferee<1>(state), Return(Status::OK()))); @@ -2834,6 +2933,7 @@ TEST_F(InstanceCtrlTest, RescheduleTest) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); functionMeta_.codeMetaData.storageType = "S3"; + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto resourceViewMgr = std::make_shared(); auto primary = MockResourceView::CreateMockResourceView(); resourceViewMgr->primary_ = primary; @@ -3006,6 +3106,7 @@ TEST_F(InstanceCtrlTest, CreateLocalNotEnoughAndRemoteNotEnough) ASSERT_IF_NULL(instanceCtrl); instanceCtrl->Start(nullptr, resourceViewMgr, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto localSchedSrv = std::make_shared(); messages::ScheduleResponse scheduleResponse; @@ -3026,6 +3127,9 @@ TEST_F(InstanceCtrlTest, CreateLocalNotEnoughAndRemoteNotEnough) .WillRepeatedly(testing::DoAll(SaveArg<0>(&mockStateMachineState), SaveArg<1>(&mockStateMachineInstanceStatusMsg), Return(NEW_RESULT))); + EXPECT_CALL(mockStateMachine, GetInstanceState()) + .WillOnce(Return(InstanceState::SCHEDULING)) + .WillOnce(Return(InstanceState::SCHEDULING)); EXPECT_CALL(mockStateMachine, ReleaseOwner).WillRepeatedly(Return()); EXPECT_CALL(mockStateMachine, GetCancelFuture).WillRepeatedly(Return(litebus::Future())); @@ -3108,6 +3212,7 @@ TEST_F(InstanceCtrlTest, CreateLocalNotEnoughAndRemoteEnough) auto instanceCtrl = std::make_shared(actor); ASSERT_IF_NULL(instanceCtrl); instanceCtrl->Start(nullptr, resourceViewMgr, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); instanceCtrl->BindLocalSchedSrv(localSchedSrv); auto stateMachine = std::make_shared("nodeN"); @@ -3190,6 +3295,7 @@ TEST_F(InstanceCtrlTest, CreateLocalNotEnoughButNotForward) auto instanceCtrl = std::make_shared(actor); ASSERT_IF_NULL(instanceCtrl); instanceCtrl->Start(nullptr, resourceViewMgr, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto localSchedSrv = std::make_shared(); EXPECT_CALL(*localSchedSrv, ForwardSchedule).Times(0); @@ -3252,6 +3358,7 @@ TEST_F(InstanceCtrlTest, NewInstanceWithDuplicate) EXPECT_CALL(*observer, IsSystemFunction).WillRepeatedly(Return(false)); instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto stateMachine = std::make_shared("nodeID"); auto &mockStateMachine = *stateMachine; @@ -3331,6 +3438,7 @@ TEST_F(InstanceCtrlTest, SchedulingWithDuplicate) ASSERT_IF_NULL(instanceCtrl); instanceCtrl->Start(nullptr, resourceViewMgr, observer); + instanceCtrl->BindInternalIAM(mockInternalIAM_); auto stateMachine = std::make_shared("nodeN"); @@ -3604,6 +3712,10 @@ TEST_F(InstanceCtrlTest, CheckLowReliabilityNoRecover) auto res3 = actor->CheckSchedRequestValid(req); EXPECT_EQ(res3.StatusCode(), StatusCode::ERR_PARAM_INVALID); + (*instanceInfo.mutable_createoptions())["RecoverRetryTimes"] = "0"; + (*instanceInfo.mutable_createoptions())["DELEGATE_POD_LABELS"] = "{}"; + res3 = actor->CheckSchedRequestValid(req); + EXPECT_EQ(res3.StatusCode(), StatusCode::ERR_PARAM_INVALID); } /** @@ -3660,6 +3772,43 @@ TEST_F(InstanceCtrlTest, CheckHeteroResourceValid) EXPECT_TRUE(res); } +/** + * Feature: instance ctrl. + * Description: CheckHeteroResourceValid. + * Steps: + * Expectation: return bool. + */ +TEST_F(InstanceCtrlTest, CheckDiskResourceValid) +{ + auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); + auto req = std::make_shared(); + req->set_requestid("rq1"); + req->set_traceid("id1"); + + // a valid request -- disk size is 100 + resources::Resource validDisk; + validDisk.mutable_scalar()->set_value(100); + req->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + validDisk; + auto res = actor->CheckHeteroResourceValid(req); + EXPECT_FALSE(res); + + // a invalid request -- disk size is 0 + validDisk.mutable_scalar()->set_value(0); + req->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + validDisk; + res = actor->CheckHeteroResourceValid(req); + EXPECT_FALSE(res); + + // a invalid request -- disk size is -100 + validDisk.mutable_scalar()->set_value(-100); + req->mutable_instance()->mutable_resources()->mutable_resources()->operator[](DISK_RESOURCE_NAME) = + validDisk; + res = actor->CheckHeteroResourceValid(req); + EXPECT_FALSE(res); + +} + /** * Feature: instance ctrl. * Description: DeployInstance after maxInstanceRedeployTimes. @@ -3905,6 +4054,7 @@ TEST_F(InstanceCtrlTest, HandleHeartbeatLostQueryExceptionSuccess) resourceViewMgr->primary_ = primary; resourceViewMgr->virtual_ = MockResourceView::CreateMockResourceView(); actor->BindResourceView(resourceViewMgr); + actor->BindInternalIAM(mockInternalIAM_); litebus::Spawn(actor); actor->AddHeartbeatTimer("instanceid"); EXPECT_CALL(*mockSharedClientManagerProxy_, GetControlInterfacePosixClient).WillOnce(Return(nullptr)); @@ -3945,6 +4095,55 @@ TEST_F(InstanceCtrlTest, HandleHeartbeatLostQueryExceptionSuccess) litebus::Await(actor->GetAID()); } +TEST_F(InstanceCtrlTest, HandleHeartbeatLostQueryExceptionFailed) +{ + auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); + actor->BindFunctionAgentMgr(funcAgentMgr_); + actor->BindInstanceControlView(instanceControlView_); + actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); + auto resourceViewMgr = std::make_shared(); + auto primary = MockResourceView::CreateMockResourceView(); + resourceViewMgr->primary_ = primary; + resourceViewMgr->virtual_ = MockResourceView::CreateMockResourceView(); + actor->BindResourceView(resourceViewMgr); + actor->BindInternalIAM(mockInternalIAM_); + litebus::Spawn(actor); + actor->AddHeartbeatTimer("instanceid"); + EXPECT_CALL(*mockSharedClientManagerProxy_, GetControlInterfacePosixClient).WillOnce(Return(nullptr)); + auto stateMachine = std::make_shared("nodeID"); + auto &mockStateMachine = *stateMachine; + EXPECT_CALL(*instanceControlView_, GetInstance).WillRepeatedly(Return(stateMachine)); + + litebus::Promise instanceStatusInfoPromise; + EXPECT_CALL(*funcAgentMgr_, QueryInstanceStatusInfo("functionAgentID", "instanceid", "runtimeid")) + .WillOnce(Return(instanceStatusInfoPromise.GetFuture())); + resource_view::InstanceInfo instanceInfo; + instanceInfo.set_instanceid("instanceid"); + instanceInfo.set_runtimeid("runtimeid"); + instanceInfo.set_functionagentid("functionAgentID"); + EXPECT_CALL(mockStateMachine, GetInstanceInfo).WillRepeatedly(Return(instanceInfo)); + litebus::Future> deleteInstance; + InstanceState mockStateMachineState; + int32_t errCode; + EXPECT_CALL(mockStateMachine, GetOwner).WillRepeatedly(Return("nodeID")); + EXPECT_CALL(mockStateMachine, IsSaving()).WillRepeatedly(Return(false)); + EXPECT_CALL(mockStateMachine, TransitionToImpl) + .WillRepeatedly(testing::DoAll(SaveArg<0>(&mockStateMachineState), SaveArg<4>(&errCode), Return(NEW_RESULT))); + EXPECT_CALL(*mockSharedClientManagerProxy_, DeleteClient(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*funcAgentMgr_, KillInstance(testing::_, testing::_, testing::_)) + .WillRepeatedly( + Return(GenKillInstanceResponse(StatusCode::SUCCESS, "kill instance successfully", "requestID"))); + EXPECT_CALL(*primary, DeleteInstances) + .WillRepeatedly(DoAll(FutureArg<0>(&deleteInstance), Return(Status::OK()))); + litebus::Async(actor->GetAID(), &InstanceCtrlActor::HandleRuntimeHeartbeatLost, "instanceid", "runtimeid"); + instanceStatusInfoPromise.SetFailed(-1); + ASSERT_AWAIT_READY(deleteInstance); + EXPECT_EQ(mockStateMachineState, InstanceState::FATAL); + EXPECT_EQ(errCode, common::ErrorCode::ERR_INSTANCE_EXITED); + litebus::Terminate(actor->GetAID()); + litebus::Await(actor->GetAID()); +} + /** * Feature: instance ctrl. * Description: HandleHeartbeatLost instance info change @@ -3960,6 +4159,7 @@ TEST_F(InstanceCtrlTest, HandleHeartbeatLostInstanceInfoChange) resourceViewMgr->primary_ = primary; resourceViewMgr->virtual_ = MockResourceView::CreateMockResourceView(); actor->BindResourceView(resourceViewMgr); + actor->BindInternalIAM(mockInternalIAM_); litebus::Spawn(actor); actor->AddHeartbeatTimer("instanceidA"); EXPECT_CALL(*mockSharedClientManagerProxy_, GetControlInterfacePosixClient).WillOnce(Return(nullptr)); @@ -4332,6 +4532,7 @@ TEST_F(InstanceCtrlTest, RescheduleAfterJudgeRecoverableTest) resourceViewMgr->primary_ = primary; resourceViewMgr->virtual_ = MockResourceView::CreateMockResourceView(); actor->BindResourceView(resourceViewMgr); + actor->BindInternalIAM(mockInternalIAM_); actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); actor->BindFunctionAgentMgr(funcAgentMgr_); EXPECT_CALL(*mockSharedClientManagerProxy_, DeleteClient(_)).WillRepeatedly(Return(Status::OK())); @@ -4380,7 +4581,9 @@ TEST_F(InstanceCtrlTest, DeleteDriverClient) auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); actor->BindObserver(mockObserver_); actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); + actor->BindInternalIAM(mockInternalIAM_); EXPECT_CALL(*mockObserver_, DelInstance).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockInternalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); auto localSchedSrv = std::make_shared(); actor->BindLocalSchedSrv(localSchedSrv); @@ -4403,7 +4606,9 @@ TEST_F(InstanceCtrlTest, GracefulShutdown) auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); actor->BindObserver(mockObserver_); actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); + actor->BindInternalIAM(mockInternalIAM_); EXPECT_CALL(*mockObserver_, DelInstance).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInternalIAM_, IsIAMEnabled).WillRepeatedly(Return(true)); auto localSchedSrv = std::make_shared(); actor->BindLocalSchedSrv(localSchedSrv); @@ -4481,6 +4686,7 @@ TEST_F(InstanceCtrlTest, NotifyDsWorkerHealthy) { auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); actor->BindFunctionAgentMgr(funcAgentMgr_); + actor->BindInternalIAM(mockInternalIAM_); actor->BindInstanceControlView(instanceControlView_); actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); auto resourceViewMgr = std::make_shared(); @@ -4762,6 +4968,7 @@ TEST_F(InstanceCtrlTest, ToSchedulingSuccessful) GeneratedInstanceStates genStates{ "DesignatedInstanceID", InstanceState::NEW, false }; EXPECT_CALL(*instanceControlView_, TryGenerateNewInstance).WillOnce(Return(genStates)); EXPECT_CALL(*mockObserver_, PutInstanceEvent).WillOnce(Return()); + EXPECT_CALL(*mockObserver_, WatchInstance).WillOnce(Return()); auto future = instanceCtrlWithMockObserver_->ToScheduling(scheduleReq); ASSERT_AWAIT_READY(future); @@ -5010,11 +5217,13 @@ TEST_F(InstanceCtrlTest, DeleteSchedulingInstance) instance.set_functionagentid("agentID"); instance.mutable_instancestatus()->set_code(static_cast(InstanceState::SCHEDULING)); EXPECT_CALL(*stateMachine, GetInstanceInfo).WillOnce(Return(instance)); + EXPECT_CALL(*stateMachine, GetModRevision).WillOnce(Return(1000)); auto callPromise = std::make_shared>(); - EXPECT_CALL(*mockObserver_, DelInstanceEvent).WillOnce(DoAll(Invoke([callPromise](const std::string &instanceID) { - callPromise->SetValue(true); - return Status::OK(); - }))); + EXPECT_CALL(*mockObserver_, DelInstanceEvent) + .WillOnce(DoAll(Invoke([callPromise](const std::string &instanceID, int64_t) { + callPromise->SetValue(true); + return Status::OK(); + }))); instanceCtrlWithMockObserver_->DeleteSchedulingInstance("instanceID", "req-1"); instanceCtrlWithMockObserver_->DeleteSchedulingInstance("instanceID", "req-2"); ASSERT_AWAIT_READY(callPromise->GetFuture()); @@ -5061,7 +5270,10 @@ TEST_F(InstanceCtrlTest, SyncInstanceKillCreating) TEST_F(InstanceCtrlTest, OnHealthyStatusTest) { auto actor = std::make_shared("InstanceCtrlActor", "nodeID", instanceCtrlConfig); + actor->BindControlInterfaceClientManager(mockSharedClientManagerProxy_); auto instanceControlView = std::make_shared("nodeID"); + auto mockInstanceOperator = std::make_shared(); + actor->instanceOpt_ = mockInstanceOperator; actor->BindInstanceControlView(instanceControlView); auto instanceCtrl = std::make_shared(actor); auto observer = std::make_shared(); @@ -5071,47 +5283,69 @@ TEST_F(InstanceCtrlTest, OnHealthyStatusTest) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); instanceCtrl->OnHealthyStatus(Status(StatusCode::FAILED)); - auto stateMachine = std::make_shared("nodeID"); - std::unordered_map> instanceMap; - instanceMap.emplace("instance1", stateMachine); - instanceMap.emplace("instance2", stateMachine); - instanceMap.emplace("instance3", stateMachine); - instanceMap.emplace("instance4", stateMachine); + function_proxy::InstanceInfoMap instanceInfoMap; - bool subHealthTrans = false; - bool fatalTrans = false; - resource_view::InstanceInfo instanceInfo; - litebus::Promise promise; - promise.SetValue(instanceInfo); - EXPECT_CALL(*instanceControlView, GetInstances).WillOnce(Return(instanceMap)); - EXPECT_CALL(*stateMachine, GetOwner).WillRepeatedly(Return("nodeID")); - EXPECT_CALL(*stateMachine, GetLastSaveFailedState) - .WillOnce(Return(-1)) // INVALID - .WillOnce(Return(11)) // SUB_HEALTH - .WillOnce(Return(11)) // SUB_HEALTH - .WillOnce(Return(2)); // CREATING - EXPECT_CALL(*stateMachine, ResetLastSaveFailedState).WillOnce(Return()).WillOnce(Return()).WillOnce(Return()); - EXPECT_CALL(*stateMachine, SyncInstanceFromMetaStore) - .WillOnce(Return(promise.GetFuture())) - .WillOnce(Return(promise.GetFuture())) - .WillOnce(Return(promise.GetFuture())); - EXPECT_CALL(*stateMachine, UpdateInstanceInfo).WillOnce(Return()).WillOnce(Return()).WillOnce(Return()); - EXPECT_CALL(*stateMachine, GetInstanceState()) - .WillOnce(Return(InstanceState::SUB_HEALTH)) - .WillOnce(Return(InstanceState::RUNNING)) - .WillOnce(Return(InstanceState::RUNNING)); - EXPECT_CALL(*stateMachine, IsSaving).WillOnce(Return(false)).WillOnce(Return(false)); - EXPECT_CALL(*stateMachine, GetVersion()).WillOnce(Return(0)).WillOnce(Return(0)); - EXPECT_CALL(*stateMachine, TransitionToImpl(InstanceState::SUB_HEALTH, _, _, _, _)) - .WillOnce(DoAll(Assign(&subHealthTrans, true), - Return(TransitionResult{ InstanceState::SUB_HEALTH, InstanceInfo(), InstanceInfo(), 0 }))); + // not belong to this node + resource_view::InstanceInfo info1; + info1.set_instanceid("instance1"); + info1.set_function("function"); + info1.set_functionproxyid("fake_node"); + info1.mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + instanceInfoMap.insert({ "instance1", info1 }); + EXPECT_CALL(*observer, GetAllInstanceInfos).WillRepeatedly(Return(instanceInfoMap)); + instanceCtrl->OnHealthyStatus(Status::OK()); + // state machine nullptr + instanceInfoMap.clear(); + resource_view::InstanceInfo info2; + info2.set_instanceid("instance2"); + info2.set_function("12345678901234561234567890123456/0-test-helloWorld/$latest"); + info2.set_functionproxyid("nodeID"); + info2.set_jobid("job"); + info2.mutable_instancestatus()->set_code(static_cast(InstanceState::CREATING)); + instanceInfoMap.insert({ "instance2", info2 }); + EXPECT_CALL(*instanceControlView, GetInstance("instance2")).WillRepeatedly(Return(nullptr)); + + litebus::Future result; + result.SetValue({}); + bool deleted = false; + EXPECT_CALL(*mockInstanceOperator, ForceDelete).WillRepeatedly(DoAll(Assign(&deleted, true), Return(result))); + EXPECT_CALL(*observer, GetAllInstanceInfos).WillRepeatedly(Return(instanceInfoMap)); + instanceCtrl->OnHealthyStatus(Status::OK()); + ASSERT_AWAIT_TRUE([&]() { return deleted; }); + + instanceInfoMap.clear(); + info2.set_instanceid("instance3"); + instanceInfoMap.insert({ "instance3", info2 }); + EXPECT_CALL(*instanceControlView, GetInstance("instance3")).WillRepeatedly(Return(stateMachine)); + bool skipped = false; + EXPECT_CALL(*stateMachine, GetLastSaveFailedState).WillOnce(DoAll(Assign(&skipped, true), Return(-1))); + EXPECT_CALL(*observer, GetAllInstanceInfos).WillOnce(Return(instanceInfoMap)); + instanceCtrl->OnHealthyStatus(Status::OK()); + ASSERT_AWAIT_TRUE([&]() { return skipped; }); + + instanceInfoMap.clear(); + info2.set_instanceid("instance4"); + instanceInfoMap.insert({ "instance4", info2 }); + EXPECT_CALL(*instanceControlView, GetInstance("instance4")).WillRepeatedly(Return(stateMachine)); + EXPECT_CALL(*stateMachine, GetLastSaveFailedState).WillOnce(Return(3)); + EXPECT_CALL(*stateMachine, ResetLastSaveFailedState).WillOnce(Return()); + EXPECT_CALL(*stateMachine, GetInstanceInfo).WillOnce(Return(info2)); + EXPECT_CALL(*stateMachine, UpdateInstanceInfo).WillOnce(Return()); + EXPECT_CALL(*stateMachine, GetOwner).WillOnce(Return("nodeID")); + EXPECT_CALL(*stateMachine, GetInstanceState()).WillOnce(Return(InstanceState::CREATING)); + EXPECT_CALL(*stateMachine, IsSaving).WillOnce(Return(false)); + EXPECT_CALL(*stateMachine, GetVersion()).WillOnce(Return(0)); + + bool fatalTrans = false; EXPECT_CALL(*stateMachine, TransitionToImpl(InstanceState::FATAL, _, _, _, _)) .WillOnce(DoAll(Assign(&fatalTrans, true), - Return(TransitionResult{ InstanceState::FATAL, InstanceInfo(), InstanceInfo(), 0 }))); + Return(TransitionResult{ InstanceState::SUB_HEALTH, InstanceInfo(), InstanceInfo(), 0 }))); + EXPECT_CALL(*observer, GetAllInstanceInfos).WillOnce(Return(instanceInfoMap)); instanceCtrl->OnHealthyStatus(Status::OK()); - ASSERT_AWAIT_TRUE([&]() { return subHealthTrans && fatalTrans; }); + + ASSERT_AWAIT_TRUE([&]() { return fatalTrans; }); } TEST_F(InstanceCtrlTest, InstanceRouteInfoSyncerTest) @@ -5137,11 +5371,10 @@ TEST_F(InstanceCtrlTest, InstanceRouteInfoSyncerTest) auto stateMachine = std::make_shared("nodeID"); EXPECT_CALL(*stateMachine, GetOwner).WillRepeatedly(Return("nodeID")); - EXPECT_CALL(*stateMachine, GetInstanceInfo) - .WillRepeatedly(Return(instanceInfo)); + EXPECT_CALL(*stateMachine, GetInstanceInfo).WillRepeatedly(Return(instanceInfo)); EXPECT_CALL(*stateMachine, GetLastSaveFailedState) - .WillOnce(Return(2)) // SUB_HEALTH - .WillOnce(Return(-1)); // INVALID and different state + .WillOnce(Return(2)) // SUB_HEALTH + .WillOnce(Return(-1)); // INVALID and different state litebus::Future result; result.SetValue({}); @@ -5332,7 +5565,9 @@ TEST_F(InstanceCtrlTest, KillFatalInstance) auto scheduleReq = std::make_shared(); scheduleReq->mutable_instance()->CopyFrom(instanceInfo); auto instanceContext = std::make_shared(scheduleReq); - EXPECT_CALL(*stateMachine, GetInstanceContextCopy).WillOnce(Return(instanceContext)).WillOnce(Return(instanceContext)); + EXPECT_CALL(*stateMachine, GetInstanceContextCopy) + .WillOnce(Return(instanceContext)) + .WillOnce(Return(instanceContext)); EXPECT_CALL(*stateMachine, GetCancelFuture).WillRepeatedly(Return(litebus::Future())); EXPECT_CALL(*stateMachine, GetVersion).WillRepeatedly(Return(0)); @@ -5387,6 +5622,7 @@ TEST_F(InstanceCtrlTest, PersistentNewToSchedulingFailed) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully auto scheduler = std::make_shared(); instanceCtrl->BindScheduler(scheduler); @@ -5455,6 +5691,7 @@ TEST_F(InstanceCtrlTest, PersistentSchedulingToCreatingFailed) ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "" })); @@ -5493,10 +5730,10 @@ TEST_F(InstanceCtrlTest, PersistentSchedulingToCreatingFailed) EXPECT_EQ(runtimePromise->GetFuture().Get().code(), 0); ASSERT_AWAIT_TRUE([&]() { - return scheduleReq->instance().instancestatus().code() == static_cast(InstanceState::CREATING); + return scheduleReq->instance().instancestatus().code() == static_cast(InstanceState::FATAL); }); auto machine = instanceControlView->GetInstance("DesignatedInstanceID"); - EXPECT_EQ(machine->GetInstanceState(), InstanceState::SCHEDULING); + EXPECT_EQ(machine->GetInstanceState(), InstanceState::FATAL); } @@ -5547,6 +5784,7 @@ TEST_F(InstanceCtrlTest, PersistentCreatingToRunningFailed) instanceCtrl->Start(nullptr, mockResourceViewMgr_, observer); ASSERT_TRUE(observer != nullptr); EXPECT_CALL(*observer, GetFuncMeta).WillRepeatedly(Return(functionMeta_)); // mock get function successfully + instanceCtrl->BindInternalIAM(mockInternalIAM_); // mock iam successfully auto scheduler = std::make_shared(); EXPECT_CALL(*scheduler, ScheduleDecision(_)).WillOnce(Return(ScheduleResult{ "", StatusCode::SUCCESS, "" })); diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/posix_api_handler/posix_api_handler_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/posix_api_handler/posix_api_handler_test.cpp index 15e526f5f586e44c13fcf1524878964a26505747..41f468b27cbf7901d63028252872673182039c27 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/posix_api_handler/posix_api_handler_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/instance_control/posix_api_handler/posix_api_handler_test.cpp @@ -21,8 +21,8 @@ #include "async/async.hpp" #include "async/future.hpp" -#include "logs/logging.h" -#include "proto/pb/posix_pb.h" +#include "common/logs/logging.h" +#include "common/proto/pb/posix_pb.h" #include "mocks/mock_instance_ctrl.h" #include "mocks/mock_local_sched_srv.h" #include "mocks/mock_local_group_ctrl.h" @@ -39,7 +39,7 @@ class PosixAPIHandlerTest : public ::testing::Test { public: void SetUp() override { - mockInstanceCtrl_ = std::make_shared(nullptr); + mockInstanceCtrl_ = std::make_shared(); mockLocalSchedSrv_ = std::make_shared(); mockSharedClientManagerProxy_ = std::make_shared(); mockLocalGroupCtrl_ = std::make_shared(); @@ -185,6 +185,22 @@ protected: return killRsp; } + std::shared_ptr GenExitReq(const common::ErrorCode &code) + { + auto request = std::make_unique(); + request->mutable_exitreq()->set_code(code); + request->mutable_exitreq()->set_message("ok"); + return request; + } + + ExitResponse GenExitRsp(const common::ErrorCode &code, const std::string &message) + { + ExitResponse killRsp; + killRsp.set_code(code); + killRsp.set_message(message); + return killRsp; + } + protected: std::shared_ptr mockInstanceCtrl_; std::shared_ptr mockLocalSchedSrv_; @@ -303,7 +319,7 @@ TEST_F(PosixAPIHandlerTest, GroupCreate) auto future = PosixAPIHandler::GroupCreate("instanceID", request); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.Get()->has_creatersps(), true); - EXPECT_EQ(future.Get()->creatersps().code(), common::ERR_INNER_SYSTEM_ERROR); + EXPECT_EQ(future.Get()->creatersps().code(), common::ERR_INNER_COMMUNICATION); } { auto request = std::make_shared(); @@ -366,4 +382,20 @@ TEST_F(PosixAPIHandlerTest, CreateResourceGroup) EXPECT_EQ(rgrsp.code(), common::ERR_NONE); } +TEST_F(PosixAPIHandlerTest, ExitTest) +{ + PosixAPIHandler::BindInstanceCtrl(mockInstanceCtrl_); + PosixAPIHandler::BindControlClientManager(mockSharedClientManagerProxy_); + + auto exitRsp = GenExitRsp(common::ERR_NONE, "ok"); + EXPECT_CALL(*mockInstanceCtrl_, Exit).WillOnce(testing::Return(exitRsp)); + + std::string from = "runtimeB"; + auto future = PosixAPIHandler::Exit(from, GenExitReq(common::ERR_NPU_FAULT_ERROR)); + ASSERT_AWAIT_READY(future); + auto response = future.Get()->exitrsp(); + EXPECT_EQ(response.code(), common::ERR_NONE); + EXPECT_EQ(response.message(), "ok"); +} + } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_group_ctrl/local_group_ctrl_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/local_group_ctrl/local_group_ctrl_test.cpp index 9d4d438b40c3bc5d24b23b4a7abfb650663c1d66..7b816197438e8d5e1fd46534e0cc3009b9646373 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_group_ctrl/local_group_ctrl_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_group_ctrl/local_group_ctrl_test.cpp @@ -42,7 +42,7 @@ public: ~DomainUnderlayerStub() = default; litebus::Future Reserve(const litebus::AID &dst, - const std::shared_ptr &req) + const std::shared_ptr &req) { Send(dst, "Reserve", req->SerializeAsString()); reservePromises_[req->requestid()] = std::make_shared>(); @@ -117,7 +117,7 @@ public: } litebus::Future ClearGroup(const litebus::AID &dst, - const std::shared_ptr &req) + const std::shared_ptr &req) { Send(dst, "ClearGroup", req->SerializeAsString()); killGroupPromises_[req->groupid()] = std::make_shared>(); @@ -148,7 +148,7 @@ private: std::unordered_map>> unReservePromises_; std::unordered_map>> bindPromises_; std::unordered_map>> unBindPromises_; - std::unordered_map>> killGroupPromises_; + std::unordered_map>> killGroupPromises_; }; class LocalGroupCtrlTest : public ::testing::Test { @@ -163,8 +163,9 @@ public: resourceViewMgr->virtual_ = virtual_; mockScheduler_ = std::make_shared(); mockLocalSchedSrv_ = std::make_shared(); - mockInstanceCtrl_ = std::make_shared(nullptr); + mockInstanceCtrl_ = std::make_shared(); EXPECT_CALL(*mockInstanceCtrl_, RegisterClearGroupInstanceCallBack).WillRepeatedly(Return()); + EXPECT_CALL(*mockInstanceCtrl_, IsInstanceRunning).WillRepeatedly(Return(false)); mockMetaStoreClient_ = std::make_shared(""); localGroupCtrlActor_ = std::make_shared(LOCAL_GROUP_CTRL_ACTOR_NAME, "nodeA", mockMetaStoreClient_); @@ -205,7 +206,7 @@ public: void Start() { auto getResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); auto future = localGroupCtrl_->Sync().Then([=](const Status &) { return localGroupCtrl_->Recover(); }); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -248,7 +249,7 @@ GroupInfoPair NewGroupInfoJson(std::string groupID, std::string ownerProxy, Grou std::string jsonStr; (void)google::protobuf::util::MessageToJsonString(*info, &jsonStr); kv.set_value(jsonStr); - return {kv, info}; + return { kv, info }; } // group schedule not started @@ -267,7 +268,7 @@ TEST_F(LocalGroupCtrlTest, LocalGroupCtrlNotStarted) TEST_F(LocalGroupCtrlTest, LocalGroupCtrlStartedWithEmpty) { auto getResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); auto future = localGroupCtrl_->Sync().Then([=](const Status &) { return localGroupCtrl_->Recover(); }); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -278,7 +279,7 @@ TEST_F(LocalGroupCtrlTest, LocalGroupCtrlStartedWithFailureGroupInfo) { auto getResponse = std::make_shared(); getResponse->status = Status(StatusCode::FAILED); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); auto future = localGroupCtrl_->Sync().Then([=](const Status &) { return localGroupCtrl_->Recover(); }); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -292,7 +293,7 @@ TEST_F(LocalGroupCtrlTest, LocalGroupCtrlStartedWithInvalidGroupInfo) kv.set_key("/yr/group/requestID/groupID"); kv.set_value("xxxxxxx"); getResponse->kvs.emplace_back(kv); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); auto future = localGroupCtrl_->Sync().Then([=](const Status &) { return localGroupCtrl_->Recover(); }); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -311,13 +312,13 @@ TEST_F(LocalGroupCtrlTest, LocalGroupCtrlStartedWithDifferGroupInfo) getResponse->kvs.emplace_back(kv1.kv); getResponse->kvs.emplace_back(kv2.kv); getResponse->kvs.emplace_back(kv3.kv); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); // for SCHEDULING EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback).WillRepeatedly(Return()); messages::GroupResponse resp; resp.set_code(SUCCESS); resp.set_message("SUCCESS"); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); // for FAILED EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient).WillOnce(Return(nullptr)); auto future = localGroupCtrl_->Sync().Then([=](const Status &) { return localGroupCtrl_->Recover(); }); @@ -498,34 +499,37 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleUseDefaultParam createRequests->clear_requests(); auto request = createRequests->add_requests(); request->mutable_schedulingops()->mutable_range(); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(256).WillRepeatedly(Return(Status::OK())); - auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); - schedule_decision::GroupScheduleResult result; - result.code = 0; - for (int i = 0; i < 256; ++i) { - (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); - } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); - EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) - .WillRepeatedly(DoAll( - Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, - InstanceReadyCallBack callback) { callback(Status::OK()); }))); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(Status::OK())); - auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); - litebus::Promise notifyCalled; - EXPECT_CALL(*mockSharedClient, NotifyResult(_)) - .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { - notifyCalled.SetValue(request); - return runtime::NotifyResponse(); - })); - auto future = localGroupCtrl_->GroupSchedule("srcInstnceID", createRequests); - ASSERT_AWAIT_READY(future); - EXPECT_EQ(future.IsOK(), true); - EXPECT_EQ(future.Get()->code(), common::ErrorCode::ERR_NONE); - ASSERT_AWAIT_READY(notifyCalled.GetFuture()); - EXPECT_EQ(notifyCalled.GetFuture().Get().code(), common::ErrorCode::ERR_NONE); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(256).WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto putResponse = std::make_shared(); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); + schedule_decision::GroupScheduleResult result; + result.code = 0; + for (int i = 0; i < 256; ++i) { + (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); + } + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); + EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) + .WillRepeatedly(DoAll( + Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, + InstanceReadyCallBack callback) { callback(Status::OK()); }))); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto mockSharedClient = std::make_shared(); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); + litebus::Promise notifyCalled; + EXPECT_CALL(*mockSharedClient, NotifyResult(_)) + .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { + notifyCalled.SetValue(request); + return runtime::NotifyResponse(); + })); + auto future = localGroupCtrl_->GroupSchedule("srcInstnceID", createRequests); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.IsOK(), true); + EXPECT_EQ(future.Get()->code(), common::ErrorCode::ERR_NONE); + ASSERT_AWAIT_READY(notifyCalled.GetFuture()); + EXPECT_EQ(notifyCalled.GetFuture().Get().code(), common::ErrorCode::ERR_NONE); } TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleUseMaxParamLocalSuccessful) @@ -538,28 +542,31 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleUseMaxParamLoca auto request = createRequests->add_requests(); request->mutable_schedulingops()->mutable_range(); request->mutable_schedulingops()->mutable_range()->set_max(256); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(256).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(256).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = 0; for (int i = 0; i < 256; ++i) { (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) - .WillRepeatedly(DoAll( - Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, - InstanceReadyCallBack callback) { callback(Status::OK()); }))); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(DoAll( + Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, + InstanceReadyCallBack callback) { callback(Status::OK()); }))); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) - .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { - notifyCalled.SetValue(request); - return runtime::NotifyResponse(); - })); + .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { + notifyCalled.SetValue(request); + return runtime::NotifyResponse(); + })); auto future = localGroupCtrl_->GroupSchedule("srcInstnceID", createRequests); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -579,34 +586,37 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleSuccessful) request->mutable_schedulingops()->mutable_range()->set_max(3); request->mutable_schedulingops()->mutable_range()->set_min(1); request->mutable_schedulingops()->mutable_range()->set_step(1); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); - auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); - schedule_decision::GroupScheduleResult result; - result.code = 0; - for (int i = 0; i < 3; ++i) { - (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); - } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); - EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) - .WillRepeatedly(DoAll( - Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, - InstanceReadyCallBack callback) { callback(Status::OK()); }))); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(Status::OK())); - auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); - litebus::Promise notifyCalled; - EXPECT_CALL(*mockSharedClient, NotifyResult(_)) - .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { - notifyCalled.SetValue(request); - return runtime::NotifyResponse(); - })); - auto future = localGroupCtrl_->GroupSchedule("srcInstnceID", createRequests); - ASSERT_AWAIT_READY(future); - EXPECT_EQ(future.IsOK(), true); - EXPECT_EQ(future.Get()->code(), common::ErrorCode::ERR_NONE); - ASSERT_AWAIT_READY(notifyCalled.GetFuture()); - EXPECT_EQ(notifyCalled.GetFuture().Get().code(), common::ErrorCode::ERR_NONE); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto putResponse = std::make_shared(); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); + schedule_decision::GroupScheduleResult result; + result.code = 0; + for (int i = 0; i < 3; ++i) { + (void)result.results.emplace_back(schedule_decision::ScheduleResult{ "agent", 0, "" }); + } + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); + EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) + .WillRepeatedly(DoAll( + Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, + InstanceReadyCallBack callback) { callback(Status::OK()); }))); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(AsyncReturn(Status::OK()))); + auto mockSharedClient = std::make_shared(); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); + litebus::Promise notifyCalled; + EXPECT_CALL(*mockSharedClient, NotifyResult(_)) + .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { + notifyCalled.SetValue(request); + return runtime::NotifyResponse(); + })); + auto future = localGroupCtrl_->GroupSchedule("srcInstnceID", createRequests); + ASSERT_AWAIT_READY(future); + EXPECT_EQ(future.IsOK(), true); + EXPECT_EQ(future.Get()->code(), common::ErrorCode::ERR_NONE); + ASSERT_AWAIT_READY(notifyCalled.GetFuture()); + EXPECT_EQ(notifyCalled.GetFuture().Get().code(), common::ErrorCode::ERR_NONE); EXPECT_EQ(localGroupCtrlActor_->groupCtxs_.size(), (size_t)1); // clear group info std::string requestID; @@ -619,7 +629,9 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleSuccessful) auto clearGroupReq = std::make_shared(); clearGroupReq->set_grouprequestid(requestID); clearGroupReq->set_groupid(groupID); - EXPECT_CALL(*mockInstanceCtrl_,DeleteSchedulingInstance).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, DeleteSchedulingInstance) + .Times(3) + .WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto clearFuture = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::ClearGroup, localGroupCtrlActor_->GetAID(), clearGroupReq); ASSERT_AWAIT_READY(clearFuture); @@ -659,9 +671,11 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleLocalSuccessful) (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = 0; for (int i = 0; i < num; ++i) { @@ -673,17 +687,20 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleLocalSuccessful) auto allocatedPromise = std::make_shared>(); allocatedPromise->SetValue(Status(StatusCode::FAILED)); (void)allocatedFailedResult.results.emplace_back( - schedule_decision::ScheduleResult{ "agent", 0, "", {}, "", {}, allocatedPromise }); + schedule_decision::ScheduleResult{ "agent", 0, "", {}, "", {}, {}, allocatedPromise }); } - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(allocatedFailedResult)).WillOnce(Return(result)); - EXPECT_CALL(*primary_, DeleteInstances).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)) + .WillOnce(Return(AsyncReturn(allocatedFailedResult))) + .WillOnce(Return(AsyncReturn(result))); + EXPECT_CALL(*primary_, DeleteInstances).WillRepeatedly(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) { callback(Status::OK()); }))); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { @@ -715,10 +732,10 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleFailedByETCDFailed) for (int i = 0; i < num; ++i) { (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); putResponse->status = Status(StatusCode::FAILED); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(AsyncReturn(putResponse))); auto future = localGroupCtrl_->GroupSchedule("srcInstanceID", createRequests); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -738,7 +755,9 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleFailedByToSchedulingFailed) for (int i = 0; i < num; ++i) { (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status(StatusCode::FAILED))); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling) + .Times(3) + .WillRepeatedly(Return(AsyncReturn(Status(StatusCode::FAILED)))); auto future = localGroupCtrl_->GroupSchedule("srcInstanceID", createRequests); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -758,23 +777,26 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleForwardSuccessful) for (int i = 0; i < num; ++i) { (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_requestid(createRequests->requestid()); resp.set_code(SUCCESS); resp.set_message("SUCCESS"); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) { callback(Status::OK()); }))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { @@ -800,24 +822,27 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleForwardSuccessf request->mutable_schedulingops()->mutable_range()->set_max(3); request->mutable_schedulingops()->mutable_range()->set_min(1); request->mutable_schedulingops()->mutable_range()->set_step(1); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_requestid(createRequests->requestid()); resp.set_code(SUCCESS); resp.set_rangesuccessnum(3); resp.set_message("SUCCESS"); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) { callback(Status::OK()); }))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { @@ -843,30 +868,33 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleForwardGetLessI request->mutable_schedulingops()->mutable_range()->set_max(3); request->mutable_schedulingops()->mutable_range()->set_min(1); request->mutable_schedulingops()->mutable_range()->set_step(1); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_requestid(createRequests->requestid()); resp.set_code(SUCCESS); resp.set_rangesuccessnum(2); resp.set_message("SUCCESS"); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) { callback(Status::OK()); }))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { - notifyCalled.SetValue(request); - return runtime::NotifyResponse(); - })); + notifyCalled.SetValue(request); + return runtime::NotifyResponse(); + })); auto future = localGroupCtrl_->GroupSchedule("srcInstanceID", createRequests); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); @@ -888,23 +916,23 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleForwardFailed) for (int i = 0; i < num; ++i) { (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).Times(2).WillRepeatedly(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put).Times(2).WillRepeatedly(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_code(static_cast(StatusCode::ERR_GROUP_SCHEDULE_FAILED)); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule) - .WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) {}))); - EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { @@ -914,7 +942,8 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleForwardFailed) auto future = localGroupCtrl_->GroupSchedule("srcInstanceID", createRequests); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); - EXPECT_EQ(future.Get()->code(), common::ErrorCode::ERR_NONE); + // for mock, rsp maybe modified by forward response + EXPECT_TRUE(future.Get()->code() == common::ErrorCode::ERR_NONE || future.Get()->code() == common::ErrorCode::ERR_GROUP_SCHEDULE_FAILED); ASSERT_AWAIT_READY(notifyCalled.GetFuture()); EXPECT_EQ(notifyCalled.GetFuture().Get().code(), common::ErrorCode::ERR_GROUP_SCHEDULE_FAILED); } @@ -930,24 +959,27 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleWithRangeInstanceScheduleForwardFailed) request->mutable_schedulingops()->mutable_range()->set_max(3); request->mutable_schedulingops()->mutable_range()->set_min(1); request->mutable_schedulingops()->mutable_range()->set_step(1); - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); - schedule_decision::GroupScheduleResult result; - result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); + schedule_decision::GroupScheduleResult result; + result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_code(static_cast(StatusCode::ERR_GROUP_SCHEDULE_FAILED)); EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule) - .WillOnce(Return(resp)); + .WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) {}))); - EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); - litebus::Promise notifyCalled; + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(AsyncReturn( + (std::dynamic_pointer_cast(mockSharedClient))))); + litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { notifyCalled.SetValue(request); @@ -977,23 +1009,27 @@ TEST_F(LocalGroupCtrlTest, GroupScheduleRuningFailed) (void)createRequests->add_requests(); } - EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToScheduling).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto putResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)).WillOnce(Return(putResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Put) + .WillOnce(Return(AsyncReturn(putResponse))) + .WillOnce(Return(AsyncReturn(putResponse))); schedule_decision::GroupScheduleResult result; result.code = static_cast(StatusCode::RESOURCE_NOT_ENOUGH); - EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(result)); + EXPECT_CALL(*mockScheduler_, GroupScheduleDecision(_)).WillOnce(Return(AsyncReturn(result))); messages::GroupResponse resp; resp.set_code(SUCCESS); resp.set_message("SUCCESS"); - EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(resp)); + EXPECT_CALL(*mockLocalSchedSrv_, ForwardGroupSchedule).WillOnce(Return(AsyncReturn(resp))); EXPECT_CALL(*mockInstanceCtrl_, RegisterReadyCallback) .WillRepeatedly(DoAll( Invoke([](const std::string &instanceID, const std::shared_ptr &scheduleReq, InstanceReadyCallBack callback) { callback(Status(StatusCode::ERR_USER_CODE_LOAD)); }))); auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).Times(3).WillRepeatedly(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce( + Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce( @@ -1040,11 +1076,14 @@ TEST_F(LocalGroupCtrlTest, ReserveAndUnReserveSuccessful) auto allocatedPromise = std::make_shared>(); allocatedPromise->SetValue(Status(StatusCode::FAILED)); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {}, {}, "", {}, allocatedPromise})) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {} })); + .WillOnce( + Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {}, {}, "", {}, {}, allocatedPromise }))) + .WillOnce(Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {} }))); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); + EXPECT_CALL(*primary_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); { auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, @@ -1077,10 +1116,13 @@ TEST_F(LocalGroupCtrlTest, ReserveFailed) { auto scheduleReq = NewScheduleRequest(); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", StatusCode::RESOURCE_NOT_ENOUGH, {} })); + .WillOnce( + Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", StatusCode::RESOURCE_NOT_ENOUGH, {} }))); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); + EXPECT_CALL(*primary_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, localGroupCtrlActor_->GetAID(), scheduleReq); @@ -1094,8 +1136,8 @@ TEST_F(LocalGroupCtrlTest, ReserveFailed) TEST_F(LocalGroupCtrlTest, BindFailedByNoReserve) { auto scheduleReq = NewScheduleRequest(); - auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, - localGroupCtrlActor_->GetAID(), scheduleReq); + auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, localGroupCtrlActor_->GetAID(), + scheduleReq); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); EXPECT_EQ(future.Get().code(), StatusCode::ERR_INNER_SYSTEM_ERROR); @@ -1107,11 +1149,11 @@ TEST_F(LocalGroupCtrlTest, ReserveAndBindAndUnBindSuccessful) auto scheduleReq = NewScheduleRequest(); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {} })); + .WillOnce(Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {} }))); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); { auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, @@ -1122,7 +1164,7 @@ TEST_F(LocalGroupCtrlTest, ReserveAndBindAndUnBindSuccessful) EXPECT_EQ(result.code(), 0); } - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillRepeatedly(Return(AsyncReturn(Status::OK()))); auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, localGroupCtrlActor_->GetAID(), scheduleReq); ASSERT_AWAIT_READY(future); @@ -1132,7 +1174,7 @@ TEST_F(LocalGroupCtrlTest, ReserveAndBindAndUnBindSuccessful) ASSERT_AWAIT_READY(litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, localGroupCtrlActor_->GetAID(), scheduleReq)); - EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).WillOnce(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*primary_, DeleteInstances).Times(1); future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::UnBind, localGroupCtrlActor_->GetAID(), scheduleReq); @@ -1147,10 +1189,12 @@ TEST_F(LocalGroupCtrlTest, BindFailedByToCreating) auto scheduleReq = NewScheduleRequest(); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {} })); + .WillOnce(Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {} }))); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); + EXPECT_CALL(*primary_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); { auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, @@ -1161,10 +1205,11 @@ TEST_F(LocalGroupCtrlTest, BindFailedByToCreating) EXPECT_EQ(result.code(), 0); } - EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).WillOnce(Return(Status::OK())); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillOnce(Return(Status(StatusCode::ERR_ETCD_OPERATION_ERROR))); - auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, - localGroupCtrlActor_->GetAID(), scheduleReq); + EXPECT_CALL(*mockInstanceCtrl_, ForceDeleteInstance).WillOnce(Return(AsyncReturn(Status::OK()))); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_ETCD_OPERATION_ERROR)))); + auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, localGroupCtrlActor_->GetAID(), + scheduleReq); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); EXPECT_EQ(future.Get().code(), StatusCode::ERR_ETCD_OPERATION_ERROR); @@ -1176,10 +1221,12 @@ TEST_F(LocalGroupCtrlTest, BindFailedByToCreatingTxnFailedAlreadyScheduleToAnoth auto scheduleReq = NewScheduleRequest(); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {} })); + .WillOnce(Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {} }))); auto changes = std::make_shared(); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); + EXPECT_CALL(*primary_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); { auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, localGroupCtrlActor_->GetAID(), scheduleReq); @@ -1189,9 +1236,10 @@ TEST_F(LocalGroupCtrlTest, BindFailedByToCreatingTxnFailedAlreadyScheduleToAnoth EXPECT_EQ(result.code(), 0); } EXPECT_CALL(*primary_, DeleteInstances).Times(1); - EXPECT_CALL(*mockInstanceCtrl_, ToCreating).WillOnce(Return(Status(StatusCode::ERR_INSTANCE_DUPLICATED))); - auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, - localGroupCtrlActor_->GetAID(), scheduleReq); + EXPECT_CALL(*mockInstanceCtrl_, ToCreating) + .WillOnce(Return(AsyncReturn(Status(StatusCode::ERR_INSTANCE_DUPLICATED)))); + auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Bind, localGroupCtrlActor_->GetAID(), + scheduleReq); ASSERT_AWAIT_READY(future); EXPECT_EQ(future.IsOK(), true); EXPECT_EQ(future.Get().code(), StatusCode::SUCCESS); @@ -1217,12 +1265,15 @@ TEST_F(LocalGroupCtrlTest, ReserveAndTimoutToReserve) auto scheduleReq = NewScheduleRequest(); EXPECT_CALL(*mockScheduler_, ScheduleDecision(_)) - .WillOnce(Return(schedule_decision::ScheduleResult{ "agent", 0, {} })); + .WillOnce(Return(AsyncReturn(schedule_decision::ScheduleResult{ "agent", 0, {} }))); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(std::make_shared())); + EXPECT_CALL(*primary_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); litebus::Future> deletedIns; - EXPECT_CALL(*primary_, DeleteInstances).WillOnce(DoAll(FutureArg<0>(&deletedIns), Return(Status::OK()))); + EXPECT_CALL(*primary_, DeleteInstances) + .WillOnce(DoAll(FutureArg<0>(&deletedIns), Return(AsyncReturn(Status::OK())))); auto future = litebus::Async(underlayerSrv_->GetAID(), &DomainUnderlayerStub::Reserve, localGroupCtrlActor->GetAID(), scheduleReq); ASSERT_AWAIT_READY(future); @@ -1239,8 +1290,8 @@ TEST_F(LocalGroupCtrlTest, ReserveAndTimoutToReserve) // fallback metastore recover test TEST_F(LocalGroupCtrlTest, OnHealthyStatusTest) { - auto localGroupCtrlActor = - std::make_shared(LOCAL_GROUP_CTRL_ACTOR_NAME + "-OnHealthyStatusTest", "nodeA", mockMetaStoreClient_); + auto localGroupCtrlActor = std::make_shared( + LOCAL_GROUP_CTRL_ACTOR_NAME + "-OnHealthyStatusTest", "nodeA", mockMetaStoreClient_); localGroupCtrlActor->BindInstanceCtrl(mockInstanceCtrl_); litebus::Spawn(localGroupCtrlActor); auto localGroupCtrl = std::make_shared(localGroupCtrlActor); @@ -1261,9 +1312,11 @@ TEST_F(LocalGroupCtrlTest, OnHealthyStatusTest) getResponse->kvs.emplace_back(kv1.kv); getResponse->kvs.emplace_back(kv2.kv); getResponse->kvs.emplace_back(kv3.kv); - EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(getResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Get).WillOnce(Return(AsyncReturn(getResponse))); auto deleteResponse = std::make_shared(); - EXPECT_CALL(*mockMetaStoreClient_, Delete).WillOnce(Return(deleteResponse)).WillOnce(Return(deleteResponse)); + EXPECT_CALL(*mockMetaStoreClient_, Delete) + .WillOnce(Return(AsyncReturn(deleteResponse))) + .WillOnce(Return(AsyncReturn(deleteResponse))); localGroupCtrlActor->NewGroupCtx(kv4.info); localGroupCtrl->OnHealthyStatus(Status::OK()); ASSERT_AWAIT_TRUE([&]() -> bool { @@ -1382,7 +1435,8 @@ TEST_F(LocalGroupCtrlTest, SfmdGroupScheduleLocalSuccessful) TEST_F(LocalGroupCtrlTest, ResponseLater) { auto mockSharedClient = std::make_shared(); - EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)).WillOnce(Return(mockSharedClient)); + EXPECT_CALL(*clientManager_, GetControlInterfacePosixClient(_)) + .WillOnce(Return(AsyncReturn(std::dynamic_pointer_cast(mockSharedClient)))); litebus::Promise notifyCalled; EXPECT_CALL(*mockSharedClient, NotifyResult(_)) .WillOnce(Invoke([notifyCalled](runtime::NotifyRequest &&request) -> litebus::Future { @@ -1392,8 +1446,8 @@ TEST_F(LocalGroupCtrlTest, ResponseLater) auto putResponse = std::make_shared(); EXPECT_CALL(*mockMetaStoreClient_, Put).WillOnce(Return(putResponse)); - auto localGroupCtrlActor = - std::make_shared(LOCAL_GROUP_CTRL_ACTOR_NAME + "-OnHealthyStatusTest", "nodeA", mockMetaStoreClient_); + auto localGroupCtrlActor = std::make_shared( + LOCAL_GROUP_CTRL_ACTOR_NAME + "-OnHealthyStatusTest", "nodeA", mockMetaStoreClient_); localGroupCtrlActor->BindInstanceCtrl(mockInstanceCtrl_); localGroupCtrlActor->BindControlInterfaceClientManager(clientManager_); litebus::Spawn(localGroupCtrlActor); @@ -1401,7 +1455,7 @@ TEST_F(LocalGroupCtrlTest, ResponseLater) localGroupCtrl->ToReady(); auto kv = NewGroupInfoJson("group-" + litebus::uuid_generator::UUID::GetRandomUUID().ToString(), "nodeA", - GroupState::SCHEDULING, 3); + GroupState::SCHEDULING, 3); auto ctx = localGroupCtrlActor->NewGroupCtx(kv.info); localGroupCtrlActor->OnGroupSuccessful(ctx); // notifyCalled should not be called diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/constants.h b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/constants.h index 14eb940d140147aa30793cc85a633a7eea47797f..45cbd698b8a41e443cd47611e3155b98402b2240 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/constants.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/constants.h @@ -17,7 +17,7 @@ #ifndef UNIT_FUNCTION_PROXY_LOCAL_SCHEDULER_LOCAL_SCHEDULER_SERVICE_CONSTANTS_H #define UNIT_FUNCTION_PROXY_LOCAL_SCHEDULER_LOCAL_SCHEDULER_SERVICE_CONSTANTS_H -#include "status/status.h" +#include "common/status/status.h" namespace functionsystem::test { const std::string REGISTERED_GLOBAL_SCHED_SUCCESS_MSG = "register global scheduler successful"; diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/domain_sched_stub_actor.h b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/domain_sched_stub_actor.h index a5e45ac13fb18f9639b67772047b1abc4f499799..80c1646842c833d53dfac8a17491795c55448b24 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/domain_sched_stub_actor.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/domain_sched_stub_actor.h @@ -22,7 +22,7 @@ #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::test { class DomainSchedStubActor : public litebus::ActorBase { diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/global_sched_stub_actor.h b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/global_sched_stub_actor.h index 102f61ac0a380560c6dd046e558c3076349781c5..2d01d33a4c365d586452e254d7556f59f14038d4 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/global_sched_stub_actor.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/global_sched_stub_actor.h @@ -84,6 +84,11 @@ public: return evictResultPromise_->GetFuture(); } + void UpdateSchedTopoView(const litebus::AID &to, const messages::ScheduleTopology &topo) + { + Send(to, "UpdateSchedTopoView", std::move(topo.SerializeAsString())); + } + MOCK_METHOD0(MockRegister, std::string()); MOCK_METHOD0(MockUnRegister, std::string()); diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/group_manager_stub_actor.h b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/group_manager_stub_actor.h index d312e458fe3dd7311f799dc28a27c41413211012..893b44168a941b8cd502754be2b8ffa95aa8cb55 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/group_manager_stub_actor.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/group_manager_stub_actor.h @@ -22,7 +22,7 @@ #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::test { class GroupManagerStubActor : public litebus::ActorBase { diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test.cpp index aa52a8258d434f9d6210c09e601dd71c5356fd33..4645767600cee2af1cd372349669b6ddfe4662ff 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test.cpp @@ -22,9 +22,9 @@ #include "common/constants/actor_name.h" #include "common/explorer/explorer.h" -#include "logs/logging.h" +#include "common/logs/logging.h" #include "common/resource_view/resource_tool.h" -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/resource_view/view_utils.h" #include "common/utils/generate_message.h" #include "constants.h" @@ -58,8 +58,7 @@ public: functionAgentMgr_ = std::make_shared("FunctionAgentMgr", nullptr); subscriptionMgr_ = SubscriptionMgr::Init("SubscriptionMgr", SubscriptionMgrConfig{ .isPartialWatchInstances = true }); - auto pingPongActor = std::make_shared(); - mockInstanceCtrl_ = std::make_shared(nullptr); + mockInstanceCtrl_ = std::make_shared(); LocalSchedSrvActor::Param param = { .nodeID = "localSchedSrvDstActor", .globalSchedAddress = driverActor_->GetAID().UnfixUrl(), .isK8sEnabled = true, @@ -77,7 +76,6 @@ public: resourceViewMgr->virtual_ = virtual_; dstActor_->BindResourceView(resourceViewMgr); dstActor_->BindInstanceCtrl(mockInstanceCtrl_); - dstActor_->BindPingPongDriver(pingPongActor); dstActor_->BindFunctionAgentMgr(functionAgentMgr_); dstActor_->BindSubscriptionMgr(subscriptionMgr_); litebus::Spawn(dstActor_); @@ -98,7 +96,9 @@ public: topo.mutable_leader()->CopyFrom(leader); litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::UpdateDomainSchedulerAddress, domainSchedStubActor_->GetAID()); - litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::UpdateSchedTopoView, dstActor_->GetAID(), + litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::UpdateGlobalSchedulerAddress, + globalSchedStubActor_->GetAID()); + litebus::Async(globalSchedStubActor_->GetAID(), &GlobalSchedStubActor::UpdateSchedTopoView, dstActor_->GetAID(), topo); } @@ -147,9 +147,9 @@ public: auto unit = view_utils::Get1DResourceUnit(); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared(unit))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(unit)))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared(unit))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(unit)))); litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::UpdateMasterInfo, GetLeaderInfo(globalSchedStubActor_->GetAID())); @@ -191,10 +191,10 @@ TEST_F(LocalSchedSrvActorTest, ScheduleSuccess) rsp.set_message(successMsg); rsp.set_instanceid(instanceID); rsp.set_requestid(requestID); - EXPECT_CALL(*mockInstanceCtrl_, Schedule).WillOnce(testing::Return(rsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule).WillOnce(testing::Return(AsyncReturn(rsp))); auto changes = std::make_shared(); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(changes)); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(changes)); + EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(AsyncReturn(changes))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(AsyncReturn(changes))); dstActor_->domainSchedRegisterInfo_.aid = driverActor_->GetAID(); messages::ScheduleRequest req; req.set_requestid(requestID); @@ -230,10 +230,10 @@ TEST_F(LocalSchedSrvActorTest, ScheduleResourceNotEnough) rsp.set_message(successMsg); rsp.set_instanceid(instanceID); rsp.set_requestid(requestID); - EXPECT_CALL(*mockInstanceCtrl_, Schedule).WillOnce(testing::Return(rsp)); + EXPECT_CALL(*mockInstanceCtrl_, Schedule).WillOnce(testing::Return(AsyncReturn(rsp))); auto changes = std::make_shared(); - EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(changes)); - EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(changes)); + EXPECT_CALL(*primary_, GetResourceViewChanges()).WillRepeatedly(Return(AsyncReturn(changes))); + EXPECT_CALL(*virtual_, GetResourceViewChanges()).WillRepeatedly(Return(AsyncReturn(changes))); dstActor_->domainSchedRegisterInfo_.aid = driverActor_->GetAID(); messages::ScheduleRequest req; req.set_requestid(requestID); @@ -245,15 +245,38 @@ TEST_F(LocalSchedSrvActorTest, ScheduleResourceNotEnough) EXPECT_EQ(getRsp.instanceid(), instanceID); EXPECT_EQ(getRsp.requestid(), requestID); } +#if 0 +/** + * Feature: LocalSchedSrvActor + * Description: Simulates receiving a Schedule request from DomainSchedule and LocalScheduler resource not enough + * Steps: + * 1. ScheduleRequest param is empty requestID + * 2. send Schedule request to LocalSchedSrvActorTestDriver(LocalSchedSrvActorTestDriver simulates DomainScheduler + * and send schedule request to LocalSchedSrvActor). + * 3. get Schedule result. + * Expectation: schedule error code result is ScheduleResourceNotEnough + */ +TEST_F(LocalSchedSrvActorTest, ScheduleWithEmptyRequest) +{ + RegisterLocalScheduler(); + messages::ScheduleRequest req; + auto rspFuture = + litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::Schedule, dstActor_->GetAID(), req); + const auto &getRsp = rspFuture.Get(); + EXPECT_EQ(getRsp.code(), StatusCode::PARAMETER_ERROR); +} +#endif // test for LocalSchedSrvActor::UpdateSchedTopoViewTest // receive update domain scheduler request from global scheduler -TEST_F(LocalSchedSrvActorTest, UpdateSchedTopoView) -{ - auto domainSchedulerAID = litebus::Async(driverActor_->GetAID(), - &LocalSchedSrvActorTestDriver::GetDomainSchedulerAID, dstActor_->GetAID()) - .Get(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); +TEST_F(LocalSchedSrvActorTest, UpdateSchedTopoView) { + auto future = litebus::Async(driverActor_->GetAID(), + &LocalSchedSrvActorTestDriver::GetDomainSchedulerAID, + dstActor_->GetAID()); + + EXPECT_AWAIT_READY_FOR(future, 100); + + auto domainSchedulerAID = future.Get(); EXPECT_EQ(std::string(domainSchedulerAID), std::string(domainSchedStubActor_->GetAID())); } @@ -265,6 +288,69 @@ TEST_F(LocalSchedSrvActorTest, RegisterSuccess) RegisterLocalScheduler(); } +// test for LocalSchedSrvActor::Register +// send register request to global scheduler(response success) then send register request to domain scheduler(response +// success) +#if 0 +TEST_F(LocalSchedSrvActorTest, RegisterSuccessButHearbeatWithoutPing) +{ + // registry response of global scheduler + litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::BindPingPongDriver, nullptr); + messages::Registered registeredToGlobal; + registeredToGlobal.set_code(StatusCode::SUCCESS); + registeredToGlobal.set_message(REGISTERED_GLOBAL_SCHED_SUCCESS_MSG); + messages::ScheduleTopology topo; + topo.mutable_leader()->set_name(REGISTERED_DOMAIN_SCHED_NAME); + topo.mutable_leader()->set_address(domainSchedStubActor_->GetAID().UnfixUrl()); + registeredToGlobal.mutable_topo()->CopyFrom(topo); + EXPECT_CALL(*globalSchedStubActor_.get(), MockRegister).WillRepeatedly([registeredToGlobal]() { + // avoid re-register too fast, failed to get heartbeat + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return registeredToGlobal.SerializeAsString(); + }); + + // registry response of domain scheduler + messages::Registered registeredToDomain; + registeredToDomain.set_code(StatusCode::SUCCESS); + registeredToDomain.set_message(REGISTERED_DOMAIN_SCHED_SUCCESS_MSG); + EXPECT_CALL(*domainSchedStubActor_.get(), MockRegister) + .WillRepeatedly(Return(registeredToDomain.SerializeAsString())); + + EXPECT_CALL(*primary_, GetFullResourceView()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + EXPECT_CALL(*virtual_, GetFullResourceView()) + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); + + litebus::AID pingpong; + pingpong.SetName("localSchedSrvDstActor" + PINGPONG_BASENAME); + pingpong.SetUrl(dstActor_->GetAID().Url()); + auto heartbeatDriver = + std::make_shared("localSchedSrvDstActor", pingpong, [](const litebus::AID &) {}); + EXPECT_CALL(*domainSchedStubActor_, RegisterCall) + .WillOnce(Return()) + .WillOnce([&heartbeatDriver]() { heartbeatDriver->Start(); }) + .WillRepeatedly(Return()); + + litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::UpdateMasterInfo, dstActor_->GetAID(), + GetLeaderInfo(globalSchedStubActor_->GetAID())); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return !litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::HeartBeatInvalid).Get(); }); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::GetEnableFlag).Get(); }); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::HeartBeatInvalid).Get(); }); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return !litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::HeartBeatInvalid).Get(); }); + // sleep to heartbeat available which would not influence the test result + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + heartbeatDriver->Stop(); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::HeartBeatInvalid).Get(); }); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return !litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::HeartBeatInvalid).Get(); }); +} +#endif + // test for LocalSchedSrvActor::Register // send register request to global scheduler(response failed) then send register request to domain scheduler(response // success) @@ -275,7 +361,7 @@ TEST_F(LocalSchedSrvActorTest, RegisterFailedToGlobalSchedudler) litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::UpdateMasterInfo, GetLeaderInfo(globalSchedStubActor_->GetAID())); litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::ToReady); - EXPECT_FALSE(dstActor_->HeartBeatInvalid()); + EXPECT_TRUE(dstActor_->HeartBeatInvalid()); } // test for LocalSchedSrvActor::Register and LocalSchedSrvActor::Registered @@ -304,9 +390,9 @@ TEST_F(LocalSchedSrvActorTest, RegisterFailedToDomainSchedudler) .WillOnce(Return(registeredToDomainSuccess.SerializeAsString())); EXPECT_CALL(*primary_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetFullResourceView()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); litebus::Async(dstActor_->GetAID(), &LocalSchedSrvActor::UpdateMasterInfo, GetLeaderInfo(globalSchedStubActor_->GetAID())); @@ -385,9 +471,9 @@ TEST_F(LocalSchedSrvActorTest, ForwardScheduleSuccess) EXPECT_CALL(*domainSchedStubActor_.get(), MockForwardSchedule).WillOnce(Return(rsp.SerializeAsString())); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto forwardScheduleFuture = litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::ForwardSchedule, dstActor_->GetAID(), req); @@ -415,9 +501,9 @@ TEST_F(LocalSchedSrvActorTest, ForwardScheduleFailedTest) EXPECT_CALL(*domainSchedStubActor_.get(), MockForwardSchedule).WillOnce(Return(rsp.SerializeAsString())); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto forwardScheduleFuture = litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::ForwardSchedule, dstActor_->GetAID(), req); @@ -440,9 +526,9 @@ TEST_F(LocalSchedSrvActorTest, ForwardScheduleTimeoutTest) auto req = std::make_shared(); req->set_requestid(requestID); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto forwardScheduleFuture = litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::ForwardSchedule, dstActor_->GetAID(), req); @@ -466,9 +552,9 @@ TEST_F(LocalSchedSrvActorTest, ForwardScheduleRetryTest) req->set_requestid(requestID); req->mutable_instance()->mutable_scheduleoption()->set_initcalltimeout(2); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared())); + .WillRepeatedly(Return(AsyncReturn(std::make_shared()))); auto forwardScheduleFuture = litebus::Async(driverActor_->GetAID(), &LocalSchedSrvActorTestDriver::ForwardSchedule, dstActor_->GetAID(), req); @@ -500,9 +586,9 @@ TEST_F(LocalSchedSrvActorTest, ForwardScheduleParamCheck) rsp.set_requestid("forwardSchedule123"); EXPECT_CALL(*domainSchedStubActor_.get(), MockForwardSchedule).WillOnce(Return(rsp.SerializeAsString())); EXPECT_CALL(*primary_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared(view_utils::Get1DResourceUnitChanges()))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(view_utils::Get1DResourceUnitChanges())))); EXPECT_CALL(*virtual_, GetResourceViewChanges()) - .WillRepeatedly(Return(std::make_shared(view_utils::Get1DResourceUnitChanges()))); + .WillRepeatedly(Return(AsyncReturn(std::make_shared(view_utils::Get1DResourceUnitChanges())))); litebus::Future msgName; litebus::Future msgValue; @@ -589,7 +675,7 @@ TEST_F(LocalSchedSrvActorTest, EvictAgent) req->set_agentid("agentID"); req->set_requestid("agentID"); req->set_timeoutsec(1); - EXPECT_CALL(*functionAgentMgr_, EvictAgent(_)).WillOnce(Return(Status(FAILED, "falied to evict"))); + EXPECT_CALL(*functionAgentMgr_, EvictAgent(_)).WillOnce(Return(AsyncReturn(Status(FAILED, "falied to evict")))); auto future = globalSchedStubActor_->SendEvictAgent(aid, req->SerializeAsString()); EXPECT_AWAIT_READY(future); auto rsp = future.Get(); @@ -604,7 +690,7 @@ TEST_F(LocalSchedSrvActorTest, EvictAgent) req->set_agentid("agentID"); req->set_requestid("agentID"); req->set_timeoutsec(1); - EXPECT_CALL(*functionAgentMgr_, EvictAgent(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*functionAgentMgr_, EvictAgent(_)).WillOnce(Return(AsyncReturn(Status::OK()))); auto future = globalSchedStubActor_->SendEvictAgent(aid, req->SerializeAsString()); EXPECT_AWAIT_READY(future); auto rsp = future.Get(); @@ -620,7 +706,7 @@ TEST_F(LocalSchedSrvActorTest, EvictAgent) req->set_timeoutsec(1); req->add_instances("ins1"); req->add_instances("ins2"); - EXPECT_CALL(*mockInstanceCtrl_, EvictInstances).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, EvictInstances).WillOnce(testing::Return(AsyncReturn(Status::OK()))); auto future = globalSchedStubActor_->SendPreemptInstance(aid, req->SerializeAsString()); EXPECT_AWAIT_READY(future); auto rsp = future.Get(); @@ -756,9 +842,9 @@ TEST_F(LocalSchedSrvActorTest, TryCancelSchedule) TEST_F(LocalSchedSrvActorTest, GracefulShutdownTest) { RegisterLocalScheduler(); - EXPECT_CALL(*functionAgentMgr_, GracefulShutdown()).WillOnce(Return(Status::OK())).WillOnce(Return(Status::OK())); + EXPECT_CALL(*functionAgentMgr_, GracefulShutdown()).WillOnce(Return(AsyncReturn(Status::OK()))).WillOnce(Return(AsyncReturn(Status::OK()))); EXPECT_CALL(*mockInstanceCtrl_, SetAbnormal()).WillOnce(Return()).WillOnce(Return()); - EXPECT_CALL(*mockInstanceCtrl_, GracefulShutdown()).WillOnce(Return(Status::OK())).WillOnce(Return(Status::OK())); + EXPECT_CALL(*mockInstanceCtrl_, GracefulShutdown()).WillOnce(Return(AsyncReturn(Status::OK()))).WillOnce(Return(AsyncReturn(Status::OK()))); messages::Registered unRegisteredToGlobal; unRegisteredToGlobal.set_code(StatusCode::SUCCESS); EXPECT_CALL(*globalSchedStubActor_.get(), MockUnRegister) diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test_driver.h b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test_driver.h index 7f4c9efc28e779e6991dc5cdd2e6f21ec048931e..1ceb4f8035e07ffeef1e8ecac3cc20da7c59d7a3 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test_driver.h +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/local_scheduler_service/local_sched_srv_actor_test_driver.h @@ -21,8 +21,8 @@ #include #include "common/explorer/explorer.h" -#include "proto/pb/message_pb.h" -#include "status/status.h" +#include "common/proto/pb/message_pb.h" +#include "common/status/status.h" namespace functionsystem::test { class LocalSchedSrvActorTestDriver : public litebus::ActorBase { diff --git a/functionsystem/tests/unit/function_proxy/local_scheduler/subscription_manager/subscription_mgr_actor_test.cpp b/functionsystem/tests/unit/function_proxy/local_scheduler/subscription_manager/subscription_mgr_actor_test.cpp index 0566b9822a5643cb7dd6620d2108eb650fe0dc5f..c0714ae3d796cf79e00d343362eff412eae9fd48 100644 --- a/functionsystem/tests/unit/function_proxy/local_scheduler/subscription_manager/subscription_mgr_actor_test.cpp +++ b/functionsystem/tests/unit/function_proxy/local_scheduler/subscription_manager/subscription_mgr_actor_test.cpp @@ -45,7 +45,7 @@ public: litebus::Spawn(subscriptionMgrActor_); mockInstanceCtrlView_ = std::make_shared(LOCAL_NODE_ID); subscriptionMgrActor_->BindInstanceControlView(mockInstanceCtrlView_); - mockInstanceCtrl_ = std::make_shared(nullptr); + mockInstanceCtrl_ = std::make_shared(); subscriptionMgrActor_->BindInstanceCtrl(mockInstanceCtrl_); mockLocalSchedSrv_ = std::make_shared(); subscriptionMgrActor_->BindLocalSchedSrv(mockLocalSchedSrv_); @@ -436,7 +436,7 @@ TEST_F(SubscriptionManagerActorTest, CleanFunctionMasterSubscriberSuccessfully) // case 2: subscriber is exited { // 1. mock subscriber is running, mock query ip is empty - auto exitSubscriberID = SUBSCRIBER_ID+"1"; + auto exitSubscriberID = SUBSCRIBER_ID + "1"; auto subscriber = GetInstanceMachine(exitSubscriberID, InstanceState::RUNNING); EXPECT_CALL(*mockInstanceCtrlView_, GetInstance).WillRepeatedly(Return(subscriber)); EXPECT_CALL(*mockLocalSchedSrv_, QueryMasterIP).WillOnce(Return("")); @@ -449,11 +449,12 @@ TEST_F(SubscriptionManagerActorTest, CleanFunctionMasterSubscriberSuccessfully) // 2. check subscriber successfully auto eventKey = "subscribe_master_" + exitSubscriberID; EXPECT_TRUE(subscriber->HasStateChangeCallback(eventKey)); - EXPECT_TRUE(subscriptionMgrActor_->masterSubscriberMap_.find(exitSubscriberID) != subscriptionMgrActor_->masterSubscriberMap_.end()); + EXPECT_TRUE(subscriptionMgrActor_->masterSubscriberMap_.find(exitSubscriberID) + != subscriptionMgrActor_->masterSubscriberMap_.end()); // 3. mock subscriber is exited subscriber->ExecuteStateChangeCallback("reqId", InstanceState::EXITED); - ASSERT_AWAIT_TRUE([&]() { return subscriptionMgrActor_->masterSubscriberMap_.size() == 0;}); + ASSERT_AWAIT_TRUE([&]() { return subscriptionMgrActor_->masterSubscriberMap_.size() == 0; }); } } diff --git a/functionsystem/tests/unit/main.cpp b/functionsystem/tests/unit/main.cpp index dbb33f6b05902b4c8386e7fb151a6edce3cf4b97..59d8cf5598c22d2c415a5d4c929b10cbb3e21833 100644 --- a/functionsystem/tests/unit/main.cpp +++ b/functionsystem/tests/unit/main.cpp @@ -18,11 +18,11 @@ #include -#include "logs/logging.h" -#include "function_proxy/common/state_machine/instance_state_machine.h" +#include "common/logs/logging.h" #include "common/utils/exception.h" #include "logs/sdk/log_param_parser.h" #include "utils/port_helper.h" +#include "utils/os_utils.hpp" using namespace functionsystem; @@ -42,7 +42,8 @@ const std::string LOG_CONFIG_JSON = R"( "threadCount": 1 }, "alsologtostderr": true, - "stdLogLevel": "DEBUG" + "stdLogLevel": "DEBUG", + "syncFlush": true } )"; @@ -51,6 +52,7 @@ namespace LogsApi = observability::api::logs; int main(int argc, char **argv) { + RegisterSigHandler(); testing::InitGoogleTest(&argc, argv); auto globalParam = LogsSdk::GetGlobalLogParam(LOG_CONFIG_JSON); @@ -59,21 +61,24 @@ int main(int argc, char **argv) lp->CreateYrLogger(param); LogsApi::Provider::SetLoggerProvider(lp); - RegisterSigHandler(); - int port = functionsystem::test::FindAvailablePort(); litebus::os::SetEnv("LITEBUS_PORT", std::to_string(port)); - std::cout << "port: " << port << std::endl; + YRLOG_INFO("Initialize litebus on network port: {}", port); auto res = litebus::Initialize("tcp://127.0.0.1:" + std::to_string(port), "", "udp://127.0.0.1:" + std::to_string(port)); if (res != BUS_OK) { - YRLOG_ERROR("failed to initialize litebus!"); + YRLOG_ERROR("Failed to initialize litebus!"); return -1; } + auto beg = std::chrono::high_resolution_clock::now(); int code = RUN_ALL_TESTS(); - InstanceStateMachine::UnBindControlPlaneObserver(); - litebus::TerminateAll(); - litebus::Finalize(); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - beg); + YRLOG_INFO("Run all gtest cases finished with code {} in {}ms", code, duration.count()); + if (code == 0) { + exit(EXIT_SUCCESS); + } + exit(EXIT_FAILURE); return code; } \ No newline at end of file diff --git a/functionsystem/tests/unit/mocks/group_ctrl_stub_actor.h b/functionsystem/tests/unit/mocks/group_ctrl_stub_actor.h index c9e9c78ce82ff7bbf4eab581583ba8e5e1c4702b..3ad8e6ceab79d1d9acf250918d9b917e049aed1c 100644 --- a/functionsystem/tests/unit/mocks/group_ctrl_stub_actor.h +++ b/functionsystem/tests/unit/mocks/group_ctrl_stub_actor.h @@ -22,7 +22,7 @@ #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::test { class DomainGroupCtrlActorStub : public litebus::ActorBase { diff --git a/functionsystem/tests/unit/mocks/mock_agent_s3_deployer.h b/functionsystem/tests/unit/mocks/mock_agent_s3_deployer.h index ee3eac65ff080f3fd777bbac3183930acb755840..e30ca27615ddd634b6bca993e66ee9b07a50fc1a 100644 --- a/functionsystem/tests/unit/mocks/mock_agent_s3_deployer.h +++ b/functionsystem/tests/unit/mocks/mock_agent_s3_deployer.h @@ -19,7 +19,7 @@ #include -#include "files.h" +#include "common/utils/files.h" #include "function_agent/code_deployer/deployer.h" #include "function_agent/code_deployer/s3_deployer.h" diff --git a/functionsystem/tests/unit/mocks/mock_cloud_api_gateway.h b/functionsystem/tests/unit/mocks/mock_cloud_api_gateway.h index f84d4a2da2ec6b39527abb98bcd3881349398ff6..3e2e452a96d18cf065675c6b309b8d38b6b8e9fe 100644 --- a/functionsystem/tests/unit/mocks/mock_cloud_api_gateway.h +++ b/functionsystem/tests/unit/mocks/mock_cloud_api_gateway.h @@ -22,8 +22,8 @@ #include -#include "http/api_router_register.h" -#include "http/http_server.h" +#include "common/http/api_router_register.h" +#include "common/http/http_server.h" #include "utils/future_test_helper.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/mocks/mock_distributed_cache_client.h b/functionsystem/tests/unit/mocks/mock_distributed_cache_client.h index e6d2548ba2eb5f5302e2ec41e8d27242dc26ed69..7cc9a042b505052fe17b71be2720f85ce7f033ef 100644 --- a/functionsystem/tests/unit/mocks/mock_distributed_cache_client.h +++ b/functionsystem/tests/unit/mocks/mock_distributed_cache_client.h @@ -32,6 +32,7 @@ public: MOCK_METHOD(Status, Del, (const std::string &key), (override)); MOCK_METHOD(Status, Del, (const std::vector &keys, std::vector &failedKeys), (override)); MOCK_METHOD(Status, GetHealthStatus, (), (override)); + MOCK_METHOD(Status, ShutDown, (), (override)); }; } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/mocks/mock_domain_sched_mgr.h b/functionsystem/tests/unit/mocks/mock_domain_sched_mgr.h index 31ace4222b11872e6b92fb2f92a0bf3343a051ac..089d01d13e0a7a8f81ca909def5be61924adf147 100644 --- a/functionsystem/tests/unit/mocks/mock_domain_sched_mgr.h +++ b/functionsystem/tests/unit/mocks/mock_domain_sched_mgr.h @@ -22,12 +22,13 @@ #include "scheduler_manager/domain_sched_mgr.h" namespace functionsystem::test { +const long long HEARTBEAT_TIMEOUT = 30000; class MockDomainSchedMgr : public global_scheduler::DomainSchedMgr { public: MockDomainSchedMgr(const std::string &name = "DomainSchedMgrActor") : global_scheduler::DomainSchedMgr( - std::make_shared(name)) + std::make_shared(name, HEARTBEAT_TIMEOUT)) { } explicit MockDomainSchedMgr(std::shared_ptr domainSchedMgrActor) diff --git a/functionsystem/tests/unit/mocks/mock_function_agent.h b/functionsystem/tests/unit/mocks/mock_function_agent.h index 0fadb04db9fbdd0d55c795f2855751a6579f2982..90bbd25958dae6ec4dbd4551f594a100046789fe 100644 --- a/functionsystem/tests/unit/mocks/mock_function_agent.h +++ b/functionsystem/tests/unit/mocks/mock_function_agent.h @@ -19,10 +19,7 @@ #include -#include - -#include "heartbeat/ping_pong_driver.h" -#include "logs/logging.h" +#include "common/heartbeat/heartbeat_client.h" #include "function_agent/agent_service_actor.h" namespace functionsystem::test { @@ -126,6 +123,26 @@ public: } MOCK_METHOD0(MockQueryDebugInstanceInfos, messages::QueryDebugInstanceInfosResponse()); + void StaticFunctionScheduleRequest(const litebus::AID &server, const messages::ScheduleRequest &request) + { + Send(server, "StaticFunctionScheduleRequest", request.SerializeAsString()); + } + + void StaticFunctionScheduleResponse(const litebus::AID &from, std::string &&name, std::string &&msg) + { + MockStaticFunctionScheduleResponse(from, name, msg); + } + MOCK_METHOD3(MockStaticFunctionScheduleResponse, void(const litebus::AID, std::string, std::string)); + + void NotifyFunctionStatusChange(const litebus::AID &from, std::string &&name, std::string &&msg) override + { + std::pair ret = MockNotifyFunctionStatusChange(from, name, msg); + if (ret.first) { + Send(from, "NotifyFunctionStatusChangeResp", std::move(ret.second)); + } + } + MOCK_METHOD3(MockNotifyFunctionStatusChange, std::pair(litebus::AID, std::string, std::string)); + protected: void Init() override { @@ -137,6 +154,8 @@ protected: Receive("CleanStatus", &MockFunctionAgent::CleanStatus); Receive("UpdateCred", &MockFunctionAgent::UpdateCred); Receive("QueryDebugInstanceInfos", &MockFunctionAgent::QueryDebugInstanceInfos); + Receive("StaticFunctionScheduleResponse", &MockFunctionAgent::StaticFunctionScheduleResponse); + Receive("NotifyFunctionStatusChange", &MockFunctionAgent::NotifyFunctionStatusChange); } }; diff --git a/functionsystem/tests/unit/mocks/mock_function_agent_mgr.h b/functionsystem/tests/unit/mocks/mock_function_agent_mgr.h index f1e05ee736861af08ed4607aa27bc43863366bb0..7cb717dea96884a2a0f42290e0931b7008c9062e 100644 --- a/functionsystem/tests/unit/mocks/mock_function_agent_mgr.h +++ b/functionsystem/tests/unit/mocks/mock_function_agent_mgr.h @@ -21,8 +21,8 @@ #include -#include "heartbeat/ping_pong_driver.h" -#include "logs/logging.h" +#include "common/heartbeat/heartbeat_client.h" +#include "common/logs/logging.h" #include "local_scheduler/function_agent_manager/function_agent_mgr.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/mocks/mock_instance_control_view.h b/functionsystem/tests/unit/mocks/mock_instance_control_view.h index 7fb9b7709925778c7389b3617a7cc3df33c18ab4..f98dd1112462023704648c8e66aeb64953598c35 100644 --- a/functionsystem/tests/unit/mocks/mock_instance_control_view.h +++ b/functionsystem/tests/unit/mocks/mock_instance_control_view.h @@ -36,7 +36,7 @@ public: MOCK_METHOD(void, Update, (const std::string &instanceID, const resources::InstanceInfo &instanceInfo, bool isForceUpdate), (override)); - MOCK_METHOD(void, Delete, (const std::string &instanceID), (override)); + MOCK_METHOD(void, Delete, (const std::string &instanceID, int64_t modRevision), (override)); MOCK_METHOD(litebus::Future, TryExitInstance, (const std::string &instanceID, bool isSynchronized), (override)); MOCK_METHOD(std::shared_ptr, GetInstance, (const std::string &instanceID), (override)); diff --git a/functionsystem/tests/unit/mocks/mock_instance_ctrl.h b/functionsystem/tests/unit/mocks/mock_instance_ctrl.h index 4dbfb7fc52f9c9022db72a59c54fb88d60c233a6..cfe8edea90575362a1dd67f1219da3d8b0b8c7b2 100644 --- a/functionsystem/tests/unit/mocks/mock_instance_ctrl.h +++ b/functionsystem/tests/unit/mocks/mock_instance_ctrl.h @@ -21,7 +21,7 @@ #include -#include "logs/logging.h" +#include "common/logs/logging.h" #include "function_proxy/local_scheduler/instance_control/instance_ctrl.h" #include "gmock/gmock-function-mocker.h" @@ -35,6 +35,11 @@ public: : InstanceCtrl(std::move(actor)) { } + MockInstanceCtrl() + : InstanceCtrl(std::make_shared("name", "node", + local_scheduler::InstanceCtrlConfig())) + { + } ~MockInstanceCtrl() override = default; MOCK_METHOD(litebus::Future, Schedule, @@ -43,6 +48,8 @@ public: (override)); MOCK_METHOD(litebus::Future, Kill, (const std::string &srcInstanceID, const std::shared_ptr &killReq), (override)); + MOCK_METHOD(litebus::Future, Exit, + (const std::string &srcInstanceID, const std::shared_ptr &exitReq), (override)); MOCK_METHOD(litebus::Future, SyncInstances, (const std::shared_ptr &view), (override)); MOCK_METHOD(litebus::Future, SyncAgent, @@ -79,6 +86,7 @@ public: MOCK_METHOD(litebus::Future, DeleteSchedulingInstance, (const std::string &instanceID, const std::string &requestID), (override)); MOCK_METHOD(void, RegisterClearGroupInstanceCallBack, (local_scheduler::ClearGroupInstanceCallBack callback), (override)); MOCK_METHOD(litebus::Future, GracefulShutdown, (), (override)); + MOCK_METHOD(litebus::Future, IsInstanceRunning, (const std::string &instanceID), (override)); MOCK_METHOD(litebus::Future, ForwardSubscriptionEvent, (const std::shared_ptr &ctx), (override)); }; diff --git a/functionsystem/tests/unit/mocks/mock_instance_state_machine.h b/functionsystem/tests/unit/mocks/mock_instance_state_machine.h index e9b7040c2bc0475e41864e6edccd6303bf09f482..14a6c1debc6cdecb55f48dfa42e87f40fa442ff6 100644 --- a/functionsystem/tests/unit/mocks/mock_instance_state_machine.h +++ b/functionsystem/tests/unit/mocks/mock_instance_state_machine.h @@ -85,12 +85,14 @@ public: MOCK_METHOD(int64_t, GetGracefulShutdownTime, (), (override)); MOCK_METHOD(void, SetGracefulShutdownTime, (const int64_t time), (override)); MOCK_METHOD(int32_t, GetLastSaveFailedState, (), (override)); + MOCK_METHOD(int32_t, GetLastSaveFailedErrCode, (), (override)); MOCK_METHOD(void, ResetLastSaveFailedState, (), (override)); MOCK_METHOD(litebus::Future, SyncInstanceFromMetaStore, (), (override)); MOCK_METHOD(void, ExecuteStateChangeCallback, (const std::string &requestID, const InstanceState newState), (override)); MOCK_METHOD(std::string, GetRequestID, (), (override)); MOCK_METHOD(std::shared_ptr, GetInstanceContextCopy, (), (override)); MOCK_METHOD(litebus::Future, GetCancelFuture, (), (override)); + MOCK_METHOD(int64_t, GetModRevision, (), (override)); }; } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/mocks/mock_internal_iam.h b/functionsystem/tests/unit/mocks/mock_internal_iam.h new file mode 100644 index 0000000000000000000000000000000000000000..29ad48d14b56de1f7932b71e4f51865ff5480202 --- /dev/null +++ b/functionsystem/tests/unit/mocks/mock_internal_iam.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#ifndef UT_MOCKS_MOCK_INTERNAL_IAM_H +#define UT_MOCKS_MOCK_INTERNAL_IAM_H + +#include +#include + +#include +#include + +#include "async/future.hpp" +#include "async/option.hpp" +#include "common/logs/logging.h" +#include "common/utils/token_transfer.h" +#include "function_proxy/common/iam/internal_iam.h" + +namespace functionsystem::test { +using namespace functionsystem::function_proxy; +class MockInternalIAM : public InternalIAM { +public: + explicit MockInternalIAM(const Param ¶m) : InternalIAM(param) + { + } + ~MockInternalIAM() override = default; + + MOCK_METHOD(bool, IsIAMEnabled, (), (override)); + MOCK_METHOD(IAMCredType, GetCredType, (), (override)); + // mock token + MOCK_METHOD(litebus::Future>, RequireEncryptToken, (const std::string &tenantID), + (override)); + MOCK_METHOD(litebus::Future, VerifyToken, (const std::string &token), (override)); + MOCK_METHOD(Status, Authorize, (AuthorizeParam & authorizeParam), (override)); + MOCK_METHOD(litebus::Future, AbandonTokenByTenantID, (const std::string &tenantID), (override)); + // mock ak sk + MOCK_METHOD(litebus::Future>, RequireCredentialByTenantID, + (const std::string &tenantID), (override)); + MOCK_METHOD(litebus::Future>, RequireCredentialByAK, (const std::string &accessKey), + (override)); + MOCK_METHOD(litebus::Future, AbandonCredentialByTenantID, (const std::string &tenantID), (override)); + MOCK_METHOD(litebus::Future, IsSystemTenant, (const std::string &tenantID), (override)); +}; +} // namespace functionsystem::test + +#endif // UT_MOCKS_MOCK_KUBE_CLIENT_H \ No newline at end of file diff --git a/functionsystem/tests/unit/mocks/mock_kube_client.h b/functionsystem/tests/unit/mocks/mock_kube_client.h new file mode 100644 index 0000000000000000000000000000000000000000..ebcd62ccb4c39a8941982e5fc6f6b6764c9bafe4 --- /dev/null +++ b/functionsystem/tests/unit/mocks/mock_kube_client.h @@ -0,0 +1,264 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UT_MOCKS_MOCK_KUBE_CLIENT_H +#define UT_MOCKS_MOCK_KUBE_CLIENT_H + +#include +#include + +#include +#include + +#include "async/future.hpp" +#include "async/option.hpp" +#include "common/logs/logging.h" +#include "common/kube_client/kube_client.h" +#include "common/kube_client/model/common/model_base.h" +#include "function_master/scaler/scaler_actor.h" + +namespace functionsystem::test { + +const std::string DEPLOYMENT_JSON = R"( +{"status":{"availableReplicas":"0","replicas":"1"},"metadata":{"annotations":{"deployment.kubernetes.io/revision":"2"},"labels":{"app":"function-agent","securityGroup":"manager"},"name":"function-agent","namespace":"default"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"function-agent"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app":"function-agent"}},"spec":{"containers":[{"command":["/home/sn/bin/entrypoint-runtime-manager"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RUNTIME_MGR_PORT","value":"21005"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"RUNTIME_INIT_PORT","value":"21006"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"RUNTIME_PORT_NUM","value":"65535"},{"name":"METRICS_COLLECTOR_TYPE","value":"proc"},{"name":"DISK_USAGE_MONITOR_PATH","value":"/tmp"},{"name":"DISK_USAGE_LIMIT","value":"-1"},{"name":"DISK_USAGE_MONITOR_DURATION","value":"20"},{"name":"CPU4COMP","value":"3000"},{"name":"MEM4COMP","value":"6144"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"},{"name":"RUNTIME_LOG_DIR","value":"/home/snuser/log"},{"name":"RUNTIME_LOG_LEVEL","value":"INFO"},{"name":"IS_NEW_RUNTIME_PATH","value":"true"},{"name":"JAVA_PRESTART_COUNT","value":"0"},{"name":"PYTHON_PRESTART_COUNT","value":"0"},{"name":"CPP_PRESTART_COUNT","value":"0"},{"name":"JVM_CUSTOM_ARGS","value":"[]"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"name":"runtime-manager","ports":[{"containerPort":21005,"name":"21005tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"3","memory":"6Gi"},"requests":{"cpu":"3","memory":"6Gi"}},"securityContext":{"capabilities":{"add":["SYS_ADMIN"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"},{"mountPath":"/home/snuser/config/python-runtime-log.json","name":"python-runtime-log-config","readOnly":true,"subPath":"python-runtime-log.json"},{"mountPath":"/home/sn/config/log.json","name":"java-runtime-log-config","subPath":"log.json"},{"mountPath":"/home/snuser/runtime/java/log4j2.xml","name":"java-runtime-log4j2-config","subPath":"log4j2.xml"},{"mountPath":"/home/uds","name":"datasystem-socket"},{"mountPath":"/dev/shm","name":"datasystem-shm"}]},{"command":["/home/sn/bin/entrypoint-function-agent"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"FSPROXY_PORT","value":"22772"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"S3_ACCESS_KEY","value":"root"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"S3_SECRET_KEY","value":"Huawei@123"},{"name":"S3_ADDR","value":"10.244.134.153:30110"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/function-agent:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"name":"function-agent","ports":[{"containerPort":58866,"name":"58866tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"500m","memory":"500Mi"}},"securityContext":{"capabilities":{"add":["NET_ADMIN","NET_RAW"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/resource/rdo/v1/apple","name":"apple","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/boy","name":"boy","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/cat","name":"cat","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/dog","name":"dog","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/egg","name":"egg","readOnly":true},{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"dnsPolicy":"ClusterFirst","imagePullSecrets":[{"name":"default-secret"}],"initContainers":[{"command":["sh","-c","chown 1002:1002 /home/sn/log; chmod 750 /home/sn/log; chown 1002:1002 /dcache; chmod 750 /dcache"],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","name":"function-agent-init","resources":{},"securityContext":{"runAsUser":0},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"priorityClassName":"system-cluster-critical","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{"fsGroup":1002},"automountServiceAccountToken": false,"terminationGracePeriodSeconds":30,"volumes":[{"hostPath":{"path":"/etc/localtime","type":""},"name":"time-zone"},{"emptyDir":{},"name":"docker-log"},{"emptyDir":{},"name":"pkg-dir"},{"configMap":{"defaultMode":288,"items":[{"key":"a.txt","path":"a.txt"}],"name":"a-config"},"name":"apple"},{"configMap":{"defaultMode":288,"items":[{"key":"b.txt","path":"b.txt"}],"name":"b-config"},"name":"boy"},{"configMap":{"defaultMode":288,"items":[{"key":"c.txt","path":"c.txt"}],"name":"c-config"},"name":"cat"},{"configMap":{"defaultMode":288,"items":[{"key":"d.txt","path":"d.txt"}],"name":"d-config"},"name":"dog"},{"configMap":{"defaultMode":288,"items":[{"key":"e.txt","path":"e.txt"}],"name":"e-config"},"name":"egg"},{"emptyDir":{},"name":"resource-volume"},{"configMap":{"defaultMode":288,"items":[{"key":"python-runtime-log.json","path":"python-runtime-log.json"}],"name":"function-agent-config"},"name":"python-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"java-runtime-log.json","path":"log.json"}],"name":"function-agent-config"},"name":"java-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"log4j2.xml","path":"log4j2.xml"}],"name":"function-agent-config"},"name":"java-runtime-log4j2-config"},{"hostPath":{"path":"/home/uds","type":""},"name":"datasystem-socket"},{"hostPath":{"path":"/dev/shm","type":""},"name":"datasystem-shm"}]}}}})"; +const std::string CREATE_DEPLOYMENT_JSON = + R"({"metadata":{"annotations":{"deployment.kubernetes.io/revision":"2"},"labels":{"app":"function-agent-600-600-1pool"},"name":"function-agent-600-600-1pool","namespace":"default"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"function-agent-600-600-1pool"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app":"function-agent-600-600-1pool"}},"spec":{"containers":[{"command":["/home/sn/bin/entrypoint-runtime-manager"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RUNTIME_MGR_PORT","value":"21005"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"RUNTIME_INIT_PORT","value":"21006"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"RUNTIME_PORT_NUM","value":"65535"},{"name":"METRICS_COLLECTOR_TYPE","value":"proc"},{"name":"DISK_USAGE_MONITOR_PATH","value":"/tmp"},{"name":"DISK_USAGE_LIMIT","value":"-1"},{"name":"DISK_USAGE_MONITOR_DURATION","value":"20"},{"name":"CPU4COMP","value":"3000"},{"name":"MEM4COMP","value":"6144"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"},{"name":"RUNTIME_LOG_DIR","value":"/home/snuser/log"},{"name":"RUNTIME_LOG_LEVEL","value":"INFO"},{"name":"IS_NEW_RUNTIME_PATH","value":"true"},{"name":"JAVA_PRESTART_COUNT","value":"0"},{"name":"PYTHON_PRESTART_COUNT","value":"0"},{"name":"CPP_PRESTART_COUNT","value":"0"},{"name":"JVM_CUSTOM_ARGS","value":"[]"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"name":"runtime-manager","ports":[{"containerPort":21005,"name":"21005tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"600m","memory":"600Mi"},"requests":{"cpu":"600m","memory":"600Mi"}},"securityContext":{"capabilities":{"add":["SYS_ADMIN"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"},{"mountPath":"/home/snuser/config/python-runtime-log.json","name":"python-runtime-log-config","readOnly":true,"subPath":"python-runtime-log.json"},{"mountPath":"/home/sn/config/log.json","name":"java-runtime-log-config","subPath":"log.json"},{"mountPath":"/home/snuser/runtime/java/log4j2.xml","name":"java-runtime-log4j2-config","subPath":"log4j2.xml"},{"mountPath":"/home/uds","name":"datasystem-socket"},{"mountPath":"/dev/shm","name":"datasystem-shm"}]},{"command":["/home/sn/bin/entrypoint-function-agent"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"FSPROXY_PORT","value":"22772"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"S3_ACCESS_KEY","value":"root"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"S3_SECRET_KEY","value":"Huawei@123"},{"name":"S3_ADDR","value":"10.244.134.153:30110"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/function-agent:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"name":"function-agent","ports":[{"containerPort":58866,"name":"58866tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"500m","memory":"500Mi"}},"securityContext":{"capabilities":{"add":["NET_ADMIN","NET_RAW"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/resource/rdo/v1/apple","name":"apple","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/boy","name":"boy","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/cat","name":"cat","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/dog","name":"dog","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/egg","name":"egg","readOnly":true},{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"dnsPolicy":"ClusterFirst","imagePullSecrets":[{"name":"default-secret"}],"initContainers":[{"command":["sh","-c","chown 1002:1002 /home/sn/log; chmod 750 /home/sn/log; chown 1002:1002 /dcache; chmod 750 /dcache"],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","name":"function-agent-init","resources":{},"securityContext":{"runAsUser":0},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"priorityClassName":"system-cluster-critical","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{"fsGroup":1002},"serviceAccount":"system-function","serviceAccountName":"system-function","terminationGracePeriodSeconds":30,"volumes":[{"hostPath":{"path":"/etc/localtime","type":""},"name":"time-zone"},{"emptyDir":{},"name":"docker-log"},{"emptyDir":{},"name":"pkg-dir"},{"configMap":{"defaultMode":288,"items":[{"key":"a.txt","path":"a.txt"}],"name":"a-config"},"name":"apple"},{"configMap":{"defaultMode":288,"items":[{"key":"b.txt","path":"b.txt"}],"name":"b-config"},"name":"boy"},{"configMap":{"defaultMode":288,"items":[{"key":"c.txt","path":"c.txt"}],"name":"c-config"},"name":"cat"},{"configMap":{"defaultMode":288,"items":[{"key":"d.txt","path":"d.txt"}],"name":"d-config"},"name":"dog"},{"configMap":{"defaultMode":288,"items":[{"key":"e.txt","path":"e.txt"}],"name":"e-config"},"name":"egg"},{"emptyDir":{},"name":"resource-volume"},{"configMap":{"defaultMode":288,"items":[{"key":"python-runtime-log.json","path":"python-runtime-log.json"}],"name":"function-agent-config"},"name":"python-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"java-runtime-log.json","path":"log.json"}],"name":"function-agent-config"},"name":"java-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"log4j2.xml","path":"log4j2.xml"}],"name":"function-agent-config"},"name":"java-runtime-log4j2-config"},{"hostPath":{"path":"/home/uds","type":""},"name":"datasystem-socket"},{"hostPath":{"path":"/dev/shm","type":""},"name":"datasystem-shm"}]}}}})"; +const std::string PATCH_DEPLOYMENT_JSON = + R"({"metadata":{"annotations":{"deployment.kubernetes.io/revision":"2"},"labels":{"app":"function-agent-600-600-2pool"},"name":"function-agent-600-600-2pool","namespace":"default"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"function-agent-600-600-1pool"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"labels":{"app":"function-agent-600-600-1pool"}},"spec":{"containers":[{"command":["/home/sn/bin/entrypoint-runtime-manager"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"RUNTIME_MGR_PORT","value":"21005"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"RUNTIME_INIT_PORT","value":"21006"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"RUNTIME_PORT_NUM","value":"65535"},{"name":"METRICS_COLLECTOR_TYPE","value":"proc"},{"name":"DISK_USAGE_MONITOR_PATH","value":"/tmp"},{"name":"DISK_USAGE_LIMIT","value":"-1"},{"name":"DISK_USAGE_MONITOR_DURATION","value":"20"},{"name":"CPU4COMP","value":"3000"},{"name":"MEM4COMP","value":"6144"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"},{"name":"RUNTIME_LOG_DIR","value":"/home/snuser/log"},{"name":"RUNTIME_LOG_LEVEL","value":"INFO"},{"name":"IS_NEW_RUNTIME_PATH","value":"true"},{"name":"JAVA_PRESTART_COUNT","value":"0"},{"name":"PYTHON_PRESTART_COUNT","value":"0"},{"name":"CPP_PRESTART_COUNT","value":"0"},{"name":"JVM_CUSTOM_ARGS","value":"[]"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"name":"runtime-manager","ports":[{"containerPort":21005,"name":"21005tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"21005tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"600m","memory":"600Mi"},"requests":{"cpu":"600m","memory":"600Mi"}},"securityContext":{"capabilities":{"add":["SYS_ADMIN"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"},{"mountPath":"/home/snuser/config/python-runtime-log.json","name":"python-runtime-log-config","readOnly":true,"subPath":"python-runtime-log.json"},{"mountPath":"/home/sn/config/log.json","name":"java-runtime-log-config","subPath":"log.json"},{"mountPath":"/home/snuser/runtime/java/log4j2.xml","name":"java-runtime-log4j2-config","subPath":"log4j2.xml"},{"mountPath":"/home/uds","name":"datasystem-socket"},{"mountPath":"/dev/shm","name":"datasystem-shm"}]},{"command":["/home/sn/bin/entrypoint-function-agent"],"env":[{"name":"POD_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.podIP"}}},{"name":"NODE_ID","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}},{"name":"HOST_IP","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"status.hostIP"}}},{"name":"FSPROXY_PORT","value":"22772"},{"name":"FUNCTION_AGENT_PORT","value":"58866"},{"name":"S3_ACCESS_KEY","value":"root"},{"name":"DECRYPT_ALGORITHM","value":"GCM"},{"name":"S3_SECRET_KEY","value":"Huawei@123"},{"name":"S3_ADDR","value":"10.244.134.153:30110"},{"name":"LOG_PATH","value":"/home/sn/log"},{"name":"LOG_LEVEL","value":"INFO"},{"name":"LOG_ROLLING_MAXSIZE","value":"1000"},{"name":"LOG_ROLLING_MAXFILES","value":"3"},{"name":"LOG_ASYNC_LOGBUFSECS","value":"30"},{"name":"LOG_ASYNC_MAXQUEUESIZE","value":"1048510"},{"name":"LOG_ASYNC_THREADCOUNT","value":"1"},{"name":"LOG_ALSOLOGTOSTDERR","value":"false"}],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/function-agent:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"name":"function-agent","ports":[{"containerPort":58866,"name":"58866tcp00","protocol":"TCP"}],"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":10,"periodSeconds":5,"successThreshold":1,"tcpSocket":{"port":"58866tcp00"},"timeoutSeconds":5},"resources":{"limits":{"cpu":"500m","memory":"500Mi"},"requests":{"cpu":"500m","memory":"500Mi"}},"securityContext":{"capabilities":{"add":["NET_ADMIN","NET_RAW"]}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/resource/rdo/v1/apple","name":"apple","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/boy","name":"boy","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/cat","name":"cat","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/dog","name":"dog","readOnly":true},{"mountPath":"/home/sn/resource/rdo/v1/egg","name":"egg","readOnly":true},{"mountPath":"/etc/localtime","name":"time-zone"},{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"dnsPolicy":"ClusterFirst","imagePullSecrets":[{"name":"default-secret"}],"initContainers":[{"command":["sh","-c","chown 1002:1002 /home/sn/log; chmod 750 /home/sn/log; chown 1002:1002 /dcache; chmod 750 /dcache"],"image":"cd-docker-hub.szxy5.artifactory.cd-cloud-artifact.tools.huawei.com/yuanrong_euleros_x86/runtime-manager:2.2.0.B060.20230321000714","imagePullPolicy":"IfNotPresent","name":"function-agent-init","resources":{},"securityContext":{"runAsUser":0},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/home/sn/log","name":"docker-log"},{"mountPath":"/dcache","name":"pkg-dir"}]}],"priorityClassName":"system-cluster-critical","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{"fsGroup":1002},"serviceAccount":"system-function","serviceAccountName":"system-function","terminationGracePeriodSeconds":30,"volumes":[{"hostPath":{"path":"/etc/localtime","type":""},"name":"time-zone"},{"emptyDir":{},"name":"docker-log"},{"emptyDir":{},"name":"pkg-dir"},{"configMap":{"defaultMode":288,"items":[{"key":"a.txt","path":"a.txt"}],"name":"a-config"},"name":"apple"},{"configMap":{"defaultMode":288,"items":[{"key":"b.txt","path":"b.txt"}],"name":"b-config"},"name":"boy"},{"configMap":{"defaultMode":288,"items":[{"key":"c.txt","path":"c.txt"}],"name":"c-config"},"name":"cat"},{"configMap":{"defaultMode":288,"items":[{"key":"d.txt","path":"d.txt"}],"name":"d-config"},"name":"dog"},{"configMap":{"defaultMode":288,"items":[{"key":"e.txt","path":"e.txt"}],"name":"e-config"},"name":"egg"},{"emptyDir":{},"name":"resource-volume"},{"configMap":{"defaultMode":288,"items":[{"key":"python-runtime-log.json","path":"python-runtime-log.json"}],"name":"function-agent-config"},"name":"python-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"java-runtime-log.json","path":"log.json"}],"name":"function-agent-config"},"name":"java-runtime-log-config"},{"configMap":{"defaultMode":288,"items":[{"key":"log4j2.xml","path":"log4j2.xml"}],"name":"function-agent-config"},"name":"java-runtime-log4j2-config"},{"hostPath":{"path":"/home/uds","type":""},"name":"datasystem-socket"},{"hostPath":{"path":"/dev/shm","type":""},"name":"datasystem-shm"}]}}}})"; +const std::string DEFAULT_LEASE_JSON_STR = R"( +{ + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": { + "name": "function-master", + "namespace": "default" + }, + "spec": { + "holderIdentity": "10.10.10.10:22770", + "leaseDurationSeconds": 1000000, + "leaseTransitions": 100000 + } +} +)"; + +using namespace functionsystem::kube_client::utility; +using ModelUtils = functionsystem::kube_client::model::ModelUtils; + +class MockKubeClient : public functionsystem::KubeClient { +public: + MockKubeClient() + { + YRLOG_INFO("entering mock kube client's constructor..."); + auto kubeClient = std::make_shared(); + auto apiConf = std::make_shared(); + auto k8sClient = std::make_shared(apiConf); + this->k8sClientApi_ = std::make_shared(k8sClient); + this->coreClientApi_ = std::make_shared(k8sClient); + this->coordinationV1Api_ = std::make_shared(k8sClient); + } + + + ~MockKubeClient() override = default; + + litebus::Future> ListNamespacedDeployment( + const std::string &rNamespace, const litebus::Option &watch = litebus::None()); + + litebus::Future> CreateNamespacedDeployment(const std::string &rNamespace, + const std::shared_ptr &body); + + litebus::Future> PatchNamespacedDeployment(const std::string &name, + const std::string &rNamespace, + const std::shared_ptr &body); + + MOCK_METHOD(litebus::Future>, + MOCKListNamespacedHorizontalPodAutoscaler, + (const std::string, const litebus::Option)); + litebus::Future> ListNamespacedHorizontalPodAutoscaler( + const std::string &rNamespace, const litebus::Option &watch) + { + return MOCKListNamespacedHorizontalPodAutoscaler(rNamespace, watch); + } + + MOCK_METHOD(litebus::Future>, + MOCKCreateNamespacedHorizontalPodAutoscaler, + (const std::string, const std::shared_ptr)); + litebus::Future> CreateNamespacedHorizontalPodAutoscaler( + const std::string &rNamespace, const std::shared_ptr &body) + { + return MOCKCreateNamespacedHorizontalPodAutoscaler(rNamespace, body); + } + + MOCK_METHOD(litebus::Future>, + MOCKPatchNamespacedHorizontalPodAutoscaler, + (const std::string, const std::string, const std::shared_ptr)); + litebus::Future> PatchNamespacedHorizontalPodAutoscaler( + const std::string &name, + const std::string &rNamespace, + const std::shared_ptr &body) + { + return MOCKPatchNamespacedHorizontalPodAutoscaler(name, rNamespace, body); + } + + MOCK_METHOD(litebus::Future>, + MOCKDeleteNamespacedHorizontalPodAutoscaler, + (const std::string, const std::string, const litebus::Option, const litebus::Option>)); + litebus::Future> DeleteNamespacedHorizontalPodAutoscaler( + const std::string &name, + const std::string &rNamespace, + const litebus::Option &orphanDependents, + const litebus::Option> &body) + { + return MOCKDeleteNamespacedHorizontalPodAutoscaler(name, rNamespace, orphanDependents, body); + } + + MOCK_METHOD(litebus::Future>, DeleteNamespacedDeployment, + (const std::string &name, const std::string &rNamespace), (override)); + + MOCK_METHOD(litebus::Future>, ListNamespacedPod, + (const std::string &rNamespace, const litebus::Option &watch), (override)); + + MOCK_METHOD(litebus::Future>, CreateNamespacedPod, + (const std::string &rNamespace, const std::shared_ptr &body), (override)); + + MOCK_METHOD(litebus::Future>, ReadNamespacedPod, + (const std::string &rNamespace, const std::string &name), (override)); + + MOCK_METHOD(litebus::Future>, PatchNamespacedPod, + (const std::string &name, const std::string &rNamespace, const std::shared_ptr &body), (override)); + + MOCK_METHOD(litebus::Future>, DeleteNamespacedPod, + (const std::string &name, const std::string &rNamespace, const litebus::Option> &body, + const litebus::Option &orphanDependents), + (override)); + + MOCK_METHOD(litebus::Future>, PatchNode, (const std::string &name, const std::shared_ptr &body), + (override)); + + MOCK_METHOD(litebus::Future>, ListNode, (const litebus::Option &watch), (override)); + + MOCK_METHOD1(MockCreateNamespacedDeployment, void(std::shared_ptr body)); + + MOCK_METHOD0(MockCreateNamespacedDeployment0, litebus::Future>()); + + MOCK_METHOD(litebus::Future>, CreateNamespacedLease, + (const std::string &rNamespace, const std::shared_ptr &body), (override)); + + MOCK_METHOD(litebus::Future>, ReadNamespacedLease, + (const std::string &name, const std::string &rNamespace), (override)); + + MOCK_METHOD(litebus::Future>, ReplaceNamespacedLease, + (const std::string &name, const std::string &rNamespace, const std::shared_ptr &body), (override)); + + static std::shared_ptr CreateLease(const std::string &proposal, const std::string &acquireTime = "", + const std::string &renewTime = ""); + static std::string TimePointToFormatTime(const std::chrono::time_point &time); + +protected: + std::shared_ptr m_apiClient; + std::shared_ptr localDeployment_; +}; + +inline std::string MockKubeClient::TimePointToFormatTime(const std::chrono::time_point &time) +{ + auto tp = std::chrono::system_clock::to_time_t(time); + std::stringstream ss; + std::put_time(std::localtime(&tp), "%Y-%m-%dT%H:%M:%SZ"); + return ss.str(); +} + +inline std::shared_ptr MockKubeClient::CreateLease(const std::string &proposal, + const std::string &acquireTime, + const std::string &renewTime) +{ + nlohmann::json podJson = nlohmann::json::parse(DEFAULT_LEASE_JSON_STR); + std::shared_ptr lease = std::make_shared(); + lease->FromJson(podJson); + lease->GetSpec()->SetHolderIdentity(proposal); + lease->GetSpec()->SetLeaseTransitions(1); + lease->GetSpec()->SetLeaseDurationSeconds(30); + if (acquireTime.empty()) { + lease->GetSpec()->SetAcquireTime(Datetime::FromString(TimePointToFormatTime(std::chrono::system_clock::now()))); + } else { + lease->GetSpec()->SetAcquireTime(Datetime::FromString(acquireTime)); + } + if (renewTime.empty()) { + lease->GetSpec()->SetRenewTime(Datetime::FromString(TimePointToFormatTime(std::chrono::system_clock::now()))); + } else { + lease->GetSpec()->SetAcquireTime(Datetime::FromString(renewTime)); + } + return lease; +} + +inline litebus::Future> MockKubeClient::ListNamespacedDeployment( + const std::string &rNamespace, const litebus::Option &watch) +{ + YRLOG_INFO("entering mock kube client's ListNamespacedDeployment..."); + litebus::Promise> promise; + auto localVarResult = std::make_shared(); + nlohmann::json localVarJson = nlohmann::json::parse(DEPLOYMENT_JSON); + + auto converted = localVarResult->FromJson(localVarJson); + if (!converted) { + YRLOG_ERROR("failed to convert deployment"); + } + auto normalPool1 = std::make_shared(); + normalPool1->FromJson(localVarJson); + normalPool1->GetMetadata()->SetName("function-agent-pool22-300-128-fusion"); + auto dynamicPool1 = std::make_shared(); + dynamicPool1->FromJson(localVarJson); + dynamicPool1->GetMetadata()->SetName("function-agent-pool2"); + dynamicPool1->GetMetadata()->GetLabels()["group2"] = "group2"; + auto otherDeployment = std::make_shared(); + otherDeployment->FromJson(localVarJson); + otherDeployment->GetMetadata()->SetName("non-function-agent-pool"); + std::vector> deploymentsList{ localVarResult, normalPool1, dynamicPool1, otherDeployment }; + auto v1DeploymentList = std::make_shared(); + v1DeploymentList->SetItems(deploymentsList); + promise.SetValue(v1DeploymentList); + return promise.GetFuture(); +} + +inline litebus::Future> MockKubeClient::CreateNamespacedDeployment( + const std::string &rNamespace, const std::shared_ptr &body) +{ + YRLOG_INFO("entering mock kube client's CreateNamespacedDeployment..."); + if (rNamespace == "mock") { + return MockCreateNamespacedDeployment0(); + } + MockCreateNamespacedDeployment(body); + litebus::Promise> promise; + std::shared_ptr localVarResult(new V1Deployment()); + nlohmann::json localVarJson = nlohmann::json::parse(CREATE_DEPLOYMENT_JSON); + ModelUtils::FromJson(localVarJson, localVarResult); + promise.SetValue(localVarResult); + return promise.GetFuture(); +} + +inline litebus::Future> MockKubeClient::PatchNamespacedDeployment( + const std::string &name, const std::string &rNamespace, const std::shared_ptr &body) { + YRLOG_INFO("entering mock kube client's PatchNamespacedDeployment..."); + litebus::Promise> promise; + std::shared_ptr localVarResult(new V1Deployment()); + nlohmann::json localVarJson = nlohmann::json::parse(PATCH_DEPLOYMENT_JSON); + + ModelUtils::FromJson(localVarJson, localVarResult); + promise.SetValue(localVarResult); + return promise.GetFuture(); +} + +} // namespace functionsystem::test + +#endif // UT_MOCKS_MOCK_KUBE_CLIENT_H diff --git a/functionsystem/tests/unit/mocks/mock_local_instance_ctrl_actor.h b/functionsystem/tests/unit/mocks/mock_local_instance_ctrl_actor.h index 3c818176843e0cfa3040ea4a6b6a657cd7ed8319..7fb9f61e67547318b539f7523cf516558c1530d3 100644 --- a/functionsystem/tests/unit/mocks/mock_local_instance_ctrl_actor.h +++ b/functionsystem/tests/unit/mocks/mock_local_instance_ctrl_actor.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef FUNCTIONCORE_CPP_INSTANCE_CTRL_HELPER_H #define FUNCTIONCORE_CPP_INSTANCE_CTRL_HELPER_H diff --git a/functionsystem/tests/unit/mocks/mock_local_sched_srv_actor.h b/functionsystem/tests/unit/mocks/mock_local_sched_srv_actor.h index 71b62883be7b52c72bab90489eefb4be242a26b5..c28b21ad9c4c29a226751ccc7ff0af0803b10781 100644 --- a/functionsystem/tests/unit/mocks/mock_local_sched_srv_actor.h +++ b/functionsystem/tests/unit/mocks/mock_local_sched_srv_actor.h @@ -27,14 +27,15 @@ using namespace functionsystem::local_scheduler; class MockLocalSchedSrvActor : public LocalSchedSrvActor { public: explicit MockLocalSchedSrvActor(const std::string &name) - : LocalSchedSrvActor({ + : LocalSchedSrvActor(LocalSchedSrvActor::Param{ name, "", false, 0, 0, 0, - 0 + "", + 0, }){}; ~MockLocalSchedSrvActor(){}; diff --git a/functionsystem/tests/unit/mocks/mock_meta_store_client.h b/functionsystem/tests/unit/mocks/mock_meta_store_client.h index 957bc09b01f19f424398d092456225769709b38e..b7c207be91734f307125d13e719697ffab13e93d 100644 --- a/functionsystem/tests/unit/mocks/mock_meta_store_client.h +++ b/functionsystem/tests/unit/mocks/mock_meta_store_client.h @@ -56,6 +56,12 @@ public: const SyncerFunction &syncer), (override)); + MOCK_METHOD(litebus::Future>, GetAndWatchWithHandler, + (const std::string &key, const WatchOption &option, + const std::function &, bool)> &observer, + const SyncerFunction &syncer, const ResponseHandler &handler), + (override)); + MOCK_METHOD(std::shared_ptr, BeginTransaction, (), (override)); MOCK_METHOD(litebus::Future>, Commit, diff --git a/functionsystem/tests/unit/mocks/mock_module_driver.h b/functionsystem/tests/unit/mocks/mock_module_driver.h index 8855180d27e2fd217fd69e22b270609146deac1a..44195f447d11782de87c0229aa408e16e1a70963 100644 --- a/functionsystem/tests/unit/mocks/mock_module_driver.h +++ b/functionsystem/tests/unit/mocks/mock_module_driver.h @@ -17,7 +17,7 @@ #ifndef TEST_UNIT_MOCKS_MOCK_MODULE_DRIVER_H #define TEST_UNIT_MOCKS_MOCK_MODULE_DRIVER_H -#include "module_driver.h" +#include "common/utils/module_driver.h" namespace functionsystem::test { class MockModuleDriver : public ModuleDriver { diff --git a/functionsystem/tests/unit/mocks/mock_obs_wrapper.h b/functionsystem/tests/unit/mocks/mock_obs_wrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..8304a2475f325043cd43f906e3835ae6ce433f89 --- /dev/null +++ b/functionsystem/tests/unit/mocks/mock_obs_wrapper.h @@ -0,0 +1,39 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef TEST_UNIT_MOCKS_MOCK_OBS_WRAPPER_H +#define TEST_UNIT_MOCKS_MOCK_OBS_WRAPPER_H + +#include + +#include "eSDKOBS.h" +#include "code_deployer/obs_wrapper.h" + +namespace functionsystem::test { +class MockObsWrapper : public functionsystem::function_agent::ObsWrapper { +public: + MockObsWrapper() : functionsystem::function_agent::ObsWrapper() + { + } + ~MockObsWrapper() = default; + + MOCK_METHOD0(DeinitializeObs, void()); + + MOCK_METHOD0(InitializeObs, obs_status()); +}; +} + +#endif //TEST_UNIT_MOCKS_OBS_WRAPPER_CALLER_H diff --git a/functionsystem/tests/unit/mocks/mock_observer.h b/functionsystem/tests/unit/mocks/mock_observer.h index fe621a0152d63191b5fbcf21bb942b905593a106..f8311c44f0192c4d12a8705b3e69ecb8c52ef32b 100644 --- a/functionsystem/tests/unit/mocks/mock_observer.h +++ b/functionsystem/tests/unit/mocks/mock_observer.h @@ -49,6 +49,8 @@ public: MOCK_METHOD(litebus::Future, IsSystemFunction, (const std::string &function), (const, override)); + MOCK_METHOD(litebus::Future, GetAllInstanceInfos, (), (const, override)); + MOCK_METHOD(void, PutInstanceEvent, (const resource_view::InstanceInfo &instanceInfo, bool synced, int64_t modRevision), (override)); @@ -57,7 +59,8 @@ public: (const resource_view::InstanceInfo &instanceInfo, bool synced, int64_t modRevision), (override)); - MOCK_METHOD(litebus::Future, DelInstanceEvent, (const std::string &instanceID), (override)); + MOCK_METHOD(litebus::Future, DelInstanceEvent, (const std::string &instanceID, int64_t modRevision), + (override)); MOCK_METHOD(litebus::Future>, GetLocalInstances, (), (override)); @@ -78,9 +81,14 @@ public: MOCK_METHOD(void, WatchInstance, (const std::string &instanceID, int64_t revision), (override)); + MOCK_METHOD(litebus::Future, IsInstanceWatched, (const std::string &instanceID), (override)); + MOCK_METHOD(litebus::Future, GetAndWatchInstance, (const std::string &instanceID), (override)); + MOCK_METHOD(litebus::Future, GetOrWatchInstance, (const std::string &instanceID), + (override)); + MOCK_METHOD(void, CancelWatchInstance, (const std::string &instanceID), (override)); }; diff --git a/functionsystem/tests/unit/mocks/mock_ping_pong_driver.h b/functionsystem/tests/unit/mocks/mock_ping_pong_driver.h index ae637fe9464e4366a2dbf10f59b89b53fd1ecd60..0e7b6e26b8431a01f09b0a2c900eb572a038d04f 100644 --- a/functionsystem/tests/unit/mocks/mock_ping_pong_driver.h +++ b/functionsystem/tests/unit/mocks/mock_ping_pong_driver.h @@ -1,17 +1,33 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef UT_MOCKS_MOCK_PING_PONG_DRIVER_H #define UT_MOCKS_MOCK_PING_PONG_DRIVER_H #include -#include "heartbeat/ping_pong_driver.h" +#include "common/heartbeat/heartbeat_client.h" namespace functionsystem::test { -class MockPingPongDriver : public PingPongDriver { +class MockHeartbeatClientDriver : public HeartbeatClientDriver { public: - MockPingPongDriver() : PingPongDriver("MockPingPongDriver", 12000, [](const litebus::AID &, HeartbeatConnection) {}) + MockHeartbeatClientDriver() : HeartbeatClientDriver("MockPingPongDriver", [](const litebus::AID &) {}) { } - ~MockPingPongDriver() = default; + ~MockHeartbeatClientDriver() = default; }; } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/mocks/mock_posix_control_stream_client.h b/functionsystem/tests/unit/mocks/mock_posix_control_stream_client.h index d6edce52b563ccd61e1506feed97283885d8afd0..63d7e1e70965b1bacfde374d2e9ff02ab4bf42cc 100644 --- a/functionsystem/tests/unit/mocks/mock_posix_control_stream_client.h +++ b/functionsystem/tests/unit/mocks/mock_posix_control_stream_client.h @@ -21,7 +21,7 @@ #include -#include "rpc/stream/posix/control_client.h" +#include "common/rpc/stream/posix/control_client.h" namespace functionsystem::test { class MockPosixControlWrapper : public grpc::PosixControlWrapper { diff --git a/functionsystem/tests/unit/mocks/mock_resource_group_ctrl.h b/functionsystem/tests/unit/mocks/mock_resource_group_ctrl.h index 828d97b8cbb34c5d4b71b37a172b66a09ab5e98a..c24dcf59acf7d3fe20bab664e251d1b16e986fc1 100644 --- a/functionsystem/tests/unit/mocks/mock_resource_group_ctrl.h +++ b/functionsystem/tests/unit/mocks/mock_resource_group_ctrl.h @@ -1,6 +1,18 @@ /* -* Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. -*/ + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef UNIT_MOCKS_MOCK_RESOURCE_GROUP_CTRL_H #define UNIT_MOCKS_MOCK_RESOURCE_GROUP_CTRL_H diff --git a/functionsystem/tests/unit/mocks/mock_resource_group_mgr_actor.h b/functionsystem/tests/unit/mocks/mock_resource_group_mgr_actor.h index 111a243f757ec73158c084993b27728c07b9a8bf..abb5f34df6a9a848e228c943db06acbd4ab07c6b 100644 --- a/functionsystem/tests/unit/mocks/mock_resource_group_mgr_actor.h +++ b/functionsystem/tests/unit/mocks/mock_resource_group_mgr_actor.h @@ -22,8 +22,8 @@ #include "actor/actor.hpp" #include "common/constants/actor_name.h" -#include "logs/logging.h" -#include "proto/pb/message_pb.h" +#include "common/logs/logging.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/mocks/mock_resource_view.h b/functionsystem/tests/unit/mocks/mock_resource_view.h index 265d2513ff4462260abfce9b8e93fe60c8861889..9d88897c2bcbfd51607246ccca1853433859b91e 100644 --- a/functionsystem/tests/unit/mocks/mock_resource_view.h +++ b/functionsystem/tests/unit/mocks/mock_resource_view.h @@ -23,6 +23,7 @@ #include "common/resource_view/resource_view.h" #include "common/resource_view/resource_view_actor.h" +#include "common/resource_view/resource_type.h" namespace functionsystem::test { @@ -86,6 +87,8 @@ public: MOCK_METHOD(litebus::Future, GetResourceInfo, (), (override)); MOCK_METHOD(litebus::Future, UpdateUnitStatus, (const std::string &unitID, resource_view::UnitStatus status), (override)); + + MOCK_METHOD(litebus::Future, GetUnitSnapshotInfo, (const std::string &id), (override)); }; } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/mocks/mock_runtime_client.h b/functionsystem/tests/unit/mocks/mock_runtime_client.h index fbcfd4c4a0682ab352d740f92d33d41e8b7be309..ff191f0411c9fde67cceff90b4b3fdf6f18170cc 100644 --- a/functionsystem/tests/unit/mocks/mock_runtime_client.h +++ b/functionsystem/tests/unit/mocks/mock_runtime_client.h @@ -22,8 +22,9 @@ #include #include -#include "proto/pb/posix/runtime_rpc.grpc.pb.h" -#include "rpc/stream/posix_reactor.h" +#include "common/proto/pb/posix/runtime_rpc.grpc.pb.h" +#include "common/rpc/stream/posix_reactor.h" +#include "common/aksk/aksk_util.h" namespace functionsystem::test { using namespace functionsystem::grpc; @@ -71,6 +72,12 @@ public: stub_ = RuntimeRPC::NewStub(channel); context_.AddMetadata("instance_id", config.instanceID); context_.AddMetadata("runtime_id", config.runtimeID); + context_.AddMetadata("authorization", config.token); + context_.AddMetadata("access_key", "ak"); + auto timestamp = litebus::time::GetCurrentUTCTime(); + context_.AddMetadata("timestamp", timestamp); + SensitiveValue sk("sk"); + context_.AddMetadata("signature", SignTimestamp("ak", sk, timestamp)); stub_->async()->MessageStream(&context_, reactor_.get()); } catch (std::exception &e) { YRLOG_ERROR( @@ -104,6 +111,8 @@ public: auto sendMsgID = request->messageid(); auto bodyType = request->body_case(); std::cout << "send msg id = " << sendMsgID << ", body type = " << bodyType << std::endl; + SensitiveValue sk("sk"); + functionsystem::SignStreamingMessage("ak", sk, request); if (reactor_ == nullptr || reactor_->IsDone()) { return false; } @@ -138,6 +147,8 @@ public: } std::cout << ", instance id = " << config_.instanceID << ", runtime id = " << config_.runtimeID << ", message id = " << recv->messageid() << std::endl; + SensitiveValue sk("sk"); + functionsystem::SignStreamingMessage("ak", sk, resp); Send(resp); MockReceiver(recv); } diff --git a/functionsystem/tests/unit/mocks/mock_schedule_framework.h b/functionsystem/tests/unit/mocks/mock_schedule_framework.h index 40b00a7201779329fd583c0a6eff9598bbe61489..c87386cd7fb3170956d31305a2e414a687f63169 100644 --- a/functionsystem/tests/unit/mocks/mock_schedule_framework.h +++ b/functionsystem/tests/unit/mocks/mock_schedule_framework.h @@ -19,7 +19,7 @@ #include -#include "resource_type.h" +#include "common/resource_view/resource_type.h" #include "common/scheduler_framework/framework/framework.h" namespace functionsystem::test { diff --git a/functionsystem/tests/unit/mocks/mock_shared_client.h b/functionsystem/tests/unit/mocks/mock_shared_client.h index e511be0e9b8ee3ef15bec485803888ba8fba8147..eb5145fc24b869022d01492ce7b3a23feaa879ba 100644 --- a/functionsystem/tests/unit/mocks/mock_shared_client.h +++ b/functionsystem/tests/unit/mocks/mock_shared_client.h @@ -46,6 +46,7 @@ public: (override)); MOCK_METHOD(litebus::Future, Recover, (runtime::RecoverRequest && request, uint64_t timeoutMs), (override)); + MOCK_METHOD(bool, IsDone, (), (override)); }; } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.cpp b/functionsystem/tests/unit/mocks/mock_test_agent_s3_deployer.h similarity index 37% rename from functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.cpp rename to functionsystem/tests/unit/mocks/mock_test_agent_s3_deployer.h index ba1307f8f6e85be3f79c45daa052b4c62f90673b..c631753e832d8f2870bd5ec47c51f0dfc78c3ca7 100644 --- a/functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.cpp +++ b/functionsystem/tests/unit/mocks/mock_test_agent_s3_deployer.h @@ -14,13 +14,26 @@ * limitations under the License. */ -#include "heartbeat/heartbeat_observer.h" +#ifndef FUNCTION_AGENT_MOCK_AGENT_DEPLOYER_H +#define FUNCTION_AGENT_MOCK_AGENT_DEPLOYER_H -#include "ChildHeartbeatObserver.h" +#include -namespace functionsystem { -[[maybe_unused]] ChildHeartbeatObserver::ChildHeartbeatObserver(const std::string &name, const litebus::AID &dst, - const HeartbeatObserver::TimeOutHandler &handler) - : HeartbeatObserver(name, dst, handler) -{} -} \ No newline at end of file +#include "function_agent/code_deployer/deployer.h" +#include "function_agent/code_deployer/s3_deployer.h" + +namespace functionsystem::test { +class MockTestAgentS3Deployer : public function_agent::S3Deployer { +public: + explicit MockTestAgentS3Deployer(std::shared_ptr config, messages::CodePackageThresholds codePackageThresholds, + bool enableSignatureValidation = false): function_agent::S3Deployer(config, nullptr, codePackageThresholds, enableSignatureValidation) + {} + MOCK_METHOD(bool, IsDeployed, (const std::string &destination, const bool isMonopoly), + (override)); + MOCK_METHOD(function_agent::DeployResult, Deploy, (const std::shared_ptr &req), + (override)); + MOCK_METHOD(bool, Clear, (const std::string &filePath, const std::string &objectKey), (override)); +}; +} // namespace functionsystem::test + +#endif // FUNCTION_AGENT_MOCK_AGENT_DEPLOYER_H \ No newline at end of file diff --git a/functionsystem/tests/unit/mocks/mock_volume_mount.h b/functionsystem/tests/unit/mocks/mock_volume_mount.h index 663fbb5edb4ac3414439cfd615adc570bc9975fd..5e796cf0696e56fa73179827790f70ebcf698c6e 100644 --- a/functionsystem/tests/unit/mocks/mock_volume_mount.h +++ b/functionsystem/tests/unit/mocks/mock_volume_mount.h @@ -19,8 +19,8 @@ #include -#include "proto/pb/message_pb.h" -#include "status/status.h" +#include "common/proto/pb/message_pb.h" +#include "common/status/status.h" #include "runtime_manager/executor/runtime_executor.h" #include "utils/volume_mount.h" diff --git a/functionsystem/tests/unit/runtime_manager/CMakeLists.txt b/functionsystem/tests/unit/runtime_manager/CMakeLists.txt index f071c4c942921f719accd0b095f09a57ca3dfb7c..a1593315ffcc1d7052eeba82b77fae7268aef29d 100644 --- a/functionsystem/tests/unit/runtime_manager/CMakeLists.txt +++ b/functionsystem/tests/unit/runtime_manager/CMakeLists.txt @@ -20,3 +20,5 @@ add_subdirectory(healthcheck) add_subdirectory(driver) add_subdirectory(config) add_subdirectory(log) +add_subdirectory(debug) +add_subdirectory(virtual_env_manager) diff --git a/functionsystem/tests/unit/runtime_manager/config/build_test.cpp b/functionsystem/tests/unit/runtime_manager/config/build_test.cpp index ee8d70190e0c8cd22153e025ad1a1ba002721250..4e0dfe0144a154f5b571980a870277836721e417 100644 --- a/functionsystem/tests/unit/runtime_manager/config/build_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/config/build_test.cpp @@ -96,7 +96,7 @@ TEST_F(BuildTest, GeneratePosixEnvsTest) startReq->mutable_runtimeinstanceinfo()->set_instanceid("ins-004"); startReq->mutable_runtimeinstanceinfo()->mutable_runtimeconfig()->set_language("posix-custom-runtime"); auto envMap = GeneratePosixEnvs(runtimeConfig, startReq, "21003"); - EXPECT_EQ(envMap["PYTHONUNBUFFERED"], "1"); // default + EXPECT_EQ(envMap["PYTHONUNBUFFERED"], "1"); // default } } @@ -126,7 +126,7 @@ TEST_F(BuildTest, GenerateEnvsTest) { "func-POSIX_LISTEN_ADDR", "/dcache" }); startReq->mutable_runtimeinstanceinfo()->mutable_runtimeconfig()->mutable_userenvs()->insert( { "func-NPU-DEVICE-IDS", "0,1,3" }); - auto env = GenerateEnvs(runtimeConfig, startReq, "21000", { 0, 4, 6, 7 }); + auto env = GenerateEnvs(runtimeConfig, startReq, "21000", {0, 4, 6, 7}); EXPECT_TRUE(env.customResourceEnvs["ENABLE_DS_AUTH"] == "true"); EXPECT_TRUE(env.customResourceEnvs["ENABLE_SERVER_AUTH"] == "true"); EXPECT_TRUE(env.customResourceEnvs["ENABLE_SERVER_MODE"] == "true"); @@ -135,8 +135,14 @@ TEST_F(BuildTest, GenerateEnvsTest) EXPECT_TRUE(env.posixEnvs["YR_FUNCTION_LIB_PATH"] == "/dcache/layer/func/test/test-a-b-c"); EXPECT_TRUE(env.posixEnvs["LAYER_LIB_PATH"] == "/dcache/layer/test/layer-a-b"); EXPECT_TRUE(env.userEnvs["ASCEND_RT_VISIBLE_DEVICES"] == "0,1,3"); -} + EXPECT_TRUE(env.customResourceEnvs.find("YR_LOG_PREFIX") == env.customResourceEnvs.end()); + startReq->set_logprefix("YR_123_000001"); + env = GenerateEnvs(runtimeConfig, startReq, "21000", {0, 4, 6, 7}); + EXPECT_EQ(env.customResourceEnvs["YR_LOG_PREFIX"], "YR_123_000001"); + EXPECT_EQ(env.customResourceEnvs["DATASYSTEM_CLIENT_LOG_NAME"], "YR_123_000001_ds_client"); + EXPECT_EQ(env.customResourceEnvs["DATASYSTEM_CLIENT_ACCESS_LOG_NAME"], "YR_123_000001_ds_client_access"); +} TEST_F(BuildTest, SelectRealIDsTest_CardsIDsAndEnvSizeNotTheSame) { const std::vector &cardsIDs = { 0, 4, 6, 7 }; @@ -160,5 +166,4 @@ TEST_F(BuildTest, SelectRealIDsTest_EmptyEnv) std::string result = SelectRealIDs(env, cardsIDs); EXPECT_EQ(result, ""); } - } // namespace functionsystem::test diff --git a/functionsystem/tests/unit/runtime_manager/debug/CMakeLists.txt b/functionsystem/tests/unit/runtime_manager/debug/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..be1d50089e31b8e9c47d7dea8c3b990bf7c95e2d --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/debug/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} DEBUG_SERVER_MGR_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${DEBUG_SERVER_MGR_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/debug/debug_server_mgr_test.cpp b/functionsystem/tests/unit/runtime_manager/debug/debug_server_mgr_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0f32e0f6549cbd0869c94cde7da74bdf81a7451b --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/debug/debug_server_mgr_test.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "runtime_manager/debug/debug_server_mgr.h" + +#include +#include + +#include "common/constants/actor_name.h" +#include "common/logs/logging.h" +#include "port/port_manager.h" +#include "runtime_manager/manager/runtime_manager_test_actor.h" +#include "utils/future_test_helper.h" +#include "utils/generate_info.h" + +namespace functionsystem::test { +using namespace functionsystem::runtime_manager; +using namespace ::testing; + +class DebugServerMgrTest : public testing::Test { +public: + void SetUp() override + { + auto optionEnv = litebus::os::GetEnv("PATH"); + std::string env; + if (optionEnv.IsSome()) { + env_ = optionEnv.Get(); + } + } + + void TearDown() override + { + } + +protected: + std::shared_ptr debugServerMgr_; + std::shared_ptr debugServerMgrActor_; + std::string env_; +}; + +TEST_F(DebugServerMgrTest, GdbServerNotInPath) +{ + auto optionEnv = litebus::os::GetEnv("PATH"); + std::string env; + if (optionEnv.IsSome()) { + env = optionEnv.Get(); // origin env + } + + litebus::os::SetEnv("PATH", "/"); + auto debugServerMgrActor_ = + std::make_shared(GenerateRandomName(RUNTIME_MANAGER_DEBUG_SERVER_MGR_ACTOR_NAME)); + debugServerMgr_ = std::make_shared(debugServerMgrActor_); + + // does not have debug server program in PATH + auto runtimeID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto instanceID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto future = debugServerMgr_->CreateServer(runtimeID, "12888", instanceID, "cpp"); + auto status = future.Get(); + EXPECT_EQ(status.StatusCode(), StatusCode::RUNTIME_MANAGER_DEBUG_SERVER_NOTFOUND); + + future = debugServerMgr_->DestroyServer(runtimeID); + status = future.Get(); + EXPECT_EQ(status.StatusCode(), StatusCode::RUNTIME_MANAGER_DEBUG_SERVER_NOTFOUND); + litebus::os::SetEnv("PATH", env); +} + +TEST_F(DebugServerMgrTest, CreateAndDestroyServer) +{ + litebus::os::SetEnv("PATH", litebus::os::Join("/tmp/", env_, ':')); + (void) litebus::os::Rm("/tmp/gdbserver"); + auto fd = open("/tmp/gdbserver", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + dprintf(fd, "#!/bin/bash\n"); + dprintf(fd, "sleep 2s\n"); + close(fd); + auto debugServerMgrActor_ = + std::make_shared(GenerateRandomName(RUNTIME_MANAGER_DEBUG_SERVER_MGR_ACTOR_NAME)); + debugServerMgr_ = std::make_shared(debugServerMgrActor_); + + const char *argv[] = { "./runtime-manager", "--runtime_logs_dir=/tmp/", "--host_ip=127.0.0.1" }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + debugServerMgr_->SetConfig(flags); + auto runtimeID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto instanceID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + auto future = debugServerMgr_->CreateServer(runtimeID, "12888", instanceID, "cpp"); + auto status = future.Get(); + EXPECT_EQ(status.StatusCode(), StatusCode::SUCCESS); + auto outFile = litebus::os::Join(debugServerMgrActor_->logDir_, runtimeID + "-cpp-debugserver.txt"); + EXPECT_AWAIT_TRUE([=]() { return FileExists(outFile); }); + future = debugServerMgr_->DestroyServer(runtimeID); + status = future.Get(); + EXPECT_EQ(status.StatusCode(), StatusCode::SUCCESS); + auto runtimeID2 = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + future = debugServerMgr_->DestroyServer(runtimeID2); + status = future.Get(); + EXPECT_EQ(status.StatusCode(), StatusCode::RUNTIME_MANAGER_DEBUG_SERVER_NOTFOUND); + + (void)litebus::os::Rm("/tmp/gdbserver"); + if (FileExists(outFile)) { + EXPECT_TRUE(litebus::os::Rm(outFile).IsNone()); + } +} + +TEST_F(DebugServerMgrTest, QueryDebugInstanceInfos) +{ + auto debugServerMgrActor_ = + std::make_shared(GenerateRandomName(RUNTIME_MANAGER_DEBUG_SERVER_MGR_ACTOR_NAME)); + debugServerMgr_ = std::make_shared(debugServerMgrActor_); + + const char *argv[] = { "./runtime-manager", "--runtime_logs_dir=/tmp/", "--host_ip=127.0.0.1" }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + debugServerMgr_->SetConfig(flags); + + std::vector testPids; + for (int i = 0; i < 10; ++i) { + pid_t pid = fork(); + if (pid == 0) { // child process + while (true) { + sleep(1); // keep alive + } + _exit(0); // Prevents child processes from executing code outside the loop + } else if (pid > 0) { // parent process + testPids.push_back(pid); + auto runtimeID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + debugServerMgr_->actor_->runtime2DebugServerPID_[runtimeID] = pid; + debugServerMgr_->actor_->runtime2PID_[runtimeID] = pid; + debugServerMgr_->actor_->pid2runtimeID_[pid] = runtimeID; + } + } + + // query interface + auto requestID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + messages::QueryDebugInstanceInfosResponse response = debugServerMgr_->QueryDebugInstanceInfos(requestID).Get(); + YRLOG_DEBUG("response: {}", response.ShortDebugString()); + EXPECT_TRUE(response.debuginstanceinfos_size() > 0); + + debugServerMgr_->DestroyAllServers(); +} +} \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/driver/runtime_mgr_driver_test.cpp b/functionsystem/tests/unit/runtime_manager/driver/runtime_mgr_driver_test.cpp index 407bc2037acd285c143c5220df045d7ae3e3ec99..abc9af6bfc0b06f801846610c95adb7a1c60a0e9 100644 --- a/functionsystem/tests/unit/runtime_manager/driver/runtime_mgr_driver_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/driver/runtime_mgr_driver_test.cpp @@ -55,15 +55,15 @@ TEST_F(RuntimeMgrDriverTest, DriverTest) "--kill_process_timeout_seconds=2", R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" }; - flags.ParseFlags(17, argv); + flags.ParseFlags(std::size(argv), argv); EXPECT_EQ(flags.GetRuntimeHomeDir(), litebus::os::GetEnv("HOME").Get()); std::cout << "flags.GetRuntimeHomeDir(): " << flags.GetRuntimeHomeDir() << std::endl; EXPECT_EQ(flags.GetNodeJsEntryPath(), "/home/runtime/node.js"); EXPECT_EQ(flags.GetResourceLabelPath(), "/tmp/labels"); EXPECT_EQ(flags.GetNpuDeviceInfoPath(), "/home/sn/config/topology-info.json"); EXPECT_EQ(flags.GetRuntimeDsConnectTimeout(), static_cast(10)); - EXPECT_EQ(flags.GetRuntimeLdLibraryPath(), "/tmp:/home/disk"); - RuntimeManagerDriver driver(flags); + EXPECT_EQ(flags.GetRuntimeLdLibraryPath(), "/tmp:/home/disk"); + RuntimeManagerDriver driver(flags, "runtime_manager"); EXPECT_EQ(driver.Start(), Status::OK()); EXPECT_EQ(driver.Stop(), Status::OK()); driver.Await(); @@ -91,7 +91,7 @@ TEST_F(RuntimeMgrDriverTest, DriverParseFailTest) "--kill_process_timeout_seconds=2", R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" }; - auto result = flags.ParseFlags(17, argv); + auto result = flags.ParseFlags(std::size(argv), argv); EXPECT_TRUE(result.IsSome()); EXPECT_EQ(result.Get(), "Failed to parse value for: runtime_ld_library_path"); } diff --git a/functionsystem/tests/unit/runtime_manager/executor/executor_test.cpp b/functionsystem/tests/unit/runtime_manager/executor/executor_test.cpp index 89e4216bb0a3f07ae7c9af25d290b611d8d2f4fa..166262aaf746b8331b9d5c413fcc76cf6f1da750 100644 --- a/functionsystem/tests/unit/runtime_manager/executor/executor_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/executor/executor_test.cpp @@ -14,17 +14,14 @@ * limitations under the License. */ -#include "gtest/gtest.h" +#include "runtime_manager/executor/executor.h" -#include "constants.h" -#include "status/status.h" -#include "files.h" +#include "common/utils/files.h" #include "gtest/gtest.h" #include "port/port_manager.h" -#include "runtime_manager/executor/executor.h" #include "runtime_manager/executor/runtime_executor.h" -#include "utils/os_utils.hpp" #include "runtime_manager/metrics/mock_function_agent_actor.h" +#include "utils/os_utils.hpp" using namespace functionsystem::runtime_manager; diff --git a/functionsystem/tests/unit/runtime_manager/executor/runtime_executor_test.cpp b/functionsystem/tests/unit/runtime_manager/executor/runtime_executor_test.cpp index 8caf402f2cd3b053c088ec69565d574c9450d948..ee9f7c314137fa37ad6e0eba9eaf7542d23105d4 100644 --- a/functionsystem/tests/unit/runtime_manager/executor/runtime_executor_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/executor/runtime_executor_test.cpp @@ -13,24 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include +#include "runtime_manager/executor/runtime_executor.h" +#include #include -#include "constants.h" -#include "status/status.h" -#include "files.h" +#include + +#include "common/constants/constants.h" +#include "common/status/status.h" +#include "common/utils/files.h" #include "common/utils/path.h" #include "gtest/gtest.h" +#include "mocks/mock_cmdtool.h" #include "port/port_manager.h" #include "runtime_manager/healthcheck/health_check.h" +#include "runtime_manager/metrics/mock_function_agent_actor.h" #include "utils/future_test_helper.h" #include "utils/os_utils.hpp" -#include "runtime_manager/metrics/mock_function_agent_actor.h" -#include "runtime_manager/executor/runtime_executor.h" -#include "mocks/mock_cmdtool.h" +#include "utils/port_helper.h" namespace functionsystem::runtime_manager { using namespace functionsystem::test; @@ -100,6 +102,22 @@ const std::string testCondaConfig = R"( std::shared_ptr BuildStartInstanceRequest(const std::string &language); class RuntimeExecutorTest : public ::testing::Test { public: + [[maybe_unused]] static void SetUpTestSuite() + { + if (!litebus::os::ExistPath("/tmp/cpp/bin")) { + litebus::os::Mkdir("/tmp/cpp/bin"); + } + + auto fd = open("/tmp/cpp/bin/runtime", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + + std::ofstream outfile; + outfile.open("/tmp/cpp/bin/runtime"); + outfile << "sleep 2" << std::endl; + outfile.close(); + } + void SetUp() override { PortManager::GetInstance().InitPortResource(INITIAL_PORT, PORT_NUM); @@ -195,29 +213,20 @@ TEST_F(RuntimeExecutorTest, StartInstanceTest) (void) litebus::os::Rm("/conda"); auto fd = open("/conda", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); close(fd); - (void)litebus::os::Rm("/tmp/cpp/bin/runtime"); - if (!litebus::os::ExistPath("/tmp/cpp/bin")) { - litebus::os::Mkdir("/tmp/cpp/bin"); - } - fd = open("/tmp/cpp/bin/runtime", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - EXPECT_NE(fd, -1); - close(fd); - std::ofstream outfile; - outfile.open("/tmp/cpp/bin/runtime"); - outfile << "sleep 2" << std::endl; - outfile.close(); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=2000", "--runtime_dir=/tmp" }; runtime_manager::Flags flags; - flags.ParseFlags(8, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto startRequest = std::make_shared(); @@ -252,8 +261,10 @@ TEST_F(RuntimeExecutorTest, StartInstanceTest) auto startResponse = instanceResponse.mutable_startruntimeinstanceresponse(); std::string resRuntimeID = startResponse->runtimeid(); + pid_t resPid = startResponse->pid(); EXPECT_TRUE(!resRuntimeID.empty()); EXPECT_TRUE(executor_->IsRuntimeActive(resRuntimeID)); + EXPECT_TRUE(executor_->IsRuntimeActiveByPid(resPid)); auto startRequest1 = std::make_shared(); startRequest1->set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); @@ -362,29 +373,6 @@ TEST_F(RuntimeExecutorTest, StartInstance_CondaCommandNotValid) EXPECT_THAT(response.message(), testing::HasSubstr("not valid")); } -TEST_F(RuntimeExecutorTest, StartInstance_CondaExtraCommandNotValid) -{ - litebus::os::SetEnv("PATH", litebus::os::Join("/", env_, ':')); - (void) litebus::os::Rm("/conda"); - auto fd = open("/conda", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - close(fd); - - auto request = std::make_shared(); - request->set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); - auto instanceInfo = request->mutable_runtimeinstanceinfo(); - auto runtimeConfig = instanceInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto deployOptions = instanceInfo->mutable_deploymentconfig()->mutable_deployoptions(); - deployOptions->operator[](CONDA_PREFIX) = "/usr/local/conda"; - deployOptions->operator[](CONDA_DEFAULT_ENV) = "env_name_copy"; - deployOptions->operator[](CONDA_COMMAND) = "conda; rm -rf /xxx"; - - auto future = executor_->StartInstance(request, {}); - auto response = future.Get(); - EXPECT_EQ(response.code(), RUNTIME_MANAGER_CONDA_PARAMS_INVALID); - EXPECT_THAT(response.message(), testing::HasSubstr("not valid")); -} - TEST_F(RuntimeExecutorTest, StartInstanceLanguageFailTest) { auto request = std::make_shared(); @@ -607,6 +595,7 @@ TEST_F(RuntimeExecutorTest, StopInstanceTest) auto startResponse = instanceResponse.mutable_startruntimeinstanceresponse(); std::string resRuntimeID = startResponse->runtimeid(); + pid_t resPid = startResponse->pid(); EXPECT_TRUE(!resRuntimeID.empty()); auto stopRequest = std::make_shared(); @@ -617,6 +606,7 @@ TEST_F(RuntimeExecutorTest, StopInstanceTest) auto stopResponse = executor_->StopInstance(stopRequest); EXPECT_EQ(stopResponse.StatusCode(), SUCCESS); EXPECT_FALSE(executor_->IsRuntimeActive(resRuntimeID)); + EXPECT_FALSE(executor_->IsRuntimeActiveByPid(resPid)); } TEST_F(RuntimeExecutorTest, StopInstanceFailTest) @@ -671,9 +661,16 @@ TEST_F(RuntimeExecutorTest, PosixCustomRuntimeTest) (void)litebus::os::Rmdir("/home/snuser/instances/"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto future = executor_->StartInstance(request, {}); auto response = future.Get(); @@ -702,13 +699,12 @@ TEST_F(RuntimeExecutorTest, StartInstanceWithCachePoolTest) { auto client = std::make_shared(); client->RegisterProcessExitCallback(std::bind(&RuntimeExecutor::UpdatePrestartRuntimePromise, executor_, std::placeholders::_1)); - const char *argv[] = { "./runtime-manager", - "--runtime_log_level=DEBUG", + const char *argv[] = { "./runtime-manager", "--runtime_log_level=DEBUG", "--runtime_prestart_config={\"java1.8\": {\"prestartCount\": -1, \"customArgs\": " "[\"-XX:+PrintGC\",\"-XX:+UseParallelGC\"]}, \"java11\": {\"prestartCount\": -1}, " "\"cpp11\": {\"prestartCount\": 1}, \"python3.9\": {\"prestartCount\": 1}}" }; runtime_manager::Flags flags; - flags.ParseFlags(3, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); sleep(3); auto startRequest = std::make_shared(); @@ -761,7 +757,6 @@ TEST_F(RuntimeExecutorTest, StartInstanceWithCachePoolTest) EXPECT_EQ(resRequestID, "test_requestID_monopoly"); } - /** * Feature: * Description: Start instance with prestart runtime @@ -782,7 +777,7 @@ TEST_F(RuntimeExecutorTest, StartInstanceWithPrestartRuntime) "--runtime_log_level=DEBUG", "--runtime_prestart_config={\"cpp11\": {\"prestartCount\": 1}}" }; runtime_manager::Flags flags; - flags.ParseFlags(3, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto startRequest = std::make_shared(); startRequest->set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); @@ -838,7 +833,7 @@ TEST_F(RuntimeExecutorTest, KillOtherPrestartRuntimeProcessTest) "--runtime_log_level=DEBUG", "--runtime_prestart_config={\"python3.9\": {\"prestartCount\": 1}}" }; runtime_manager::Flags flags; - flags.ParseFlags(3, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto startRequest = std::make_shared(); startRequest->set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); @@ -1004,7 +999,7 @@ TEST_F(RuntimeExecutorTest, GetJavaBuildArgsTest) const char *argv[] = { "./runtime-manager", "--runtime_log_level=DEBUG","--runtime_prestart_config={}","--proc_metrics_memory=1000"}; runtime_manager::Flags flags; - flags.ParseFlags(4, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto prestartArgs = executor_->GetBuildArgsForPrestart("runtime11", "java1.8", "8080"); EXPECT_TRUE(std::count(prestartArgs.begin(), prestartArgs.end(), "-XX:+CMSClassUnloadingEnabled") == 1); @@ -1073,7 +1068,7 @@ TEST_F(RuntimeExecutorTest, GetJava11BuildArgsTest) deployConfig->set_storagetype("s3"); const char *argv[] = { "./runtime-manager", "--runtime_log_level=DEBUG","--runtime_prestart_config={}","--proc_metrics_memory=1000"}; runtime_manager::Flags flags; - flags.ParseFlags(4, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto prestartArgs = executor_->GetBuildArgsForPrestart("runtime11", "java11", "8080"); EXPECT_TRUE(std::count(prestartArgs.begin(), prestartArgs.end(), "-XX:+UseG1GC") == 1); @@ -1130,7 +1125,7 @@ TEST_F(RuntimeExecutorTest, GetJava17BuildArgsTest) deployConfig->set_storagetype("s3"); const char *argv[] = { "./runtime-manager", "--runtime_log_level=DEBUG","--runtime_prestart_config={}","--proc_metrics_memory=1000"}; runtime_manager::Flags flags; - flags.ParseFlags(4, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); std::vector args; @@ -1187,7 +1182,7 @@ TEST_F(RuntimeExecutorTest, GetJava21BuildArgsTest) deployConfig->set_storagetype("s3"); const char *argv[] = { "./runtime-manager", "--runtime_log_level=DEBUG","--runtime_prestart_config={}","--proc_metrics_memory=1000"}; runtime_manager::Flags flags; - flags.ParseFlags(4, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); std::vector args; @@ -1244,14 +1239,25 @@ TEST_F(RuntimeExecutorTest, GetNoneExistedGoExecPathTest) deployConfig->set_bucketid("test_bucketID"); deployConfig->set_deploydir(testDeployDir); deployConfig->set_storagetype("s3"); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); - const char *argv[]={"/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", "--runtime_initial_port=500", "--port_num=2000", - "--runtime_dir=/tmp", "--agent_address=127.0.0.1:8080", - "--runtime_ld_library_path=/tmp", "--proc_metrics_cpu=2000", "--proc_metrics_memory=2000", - R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})"}; + const char *argv[] = { + "/runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--port_num=2000", + "--runtime_dir=/tmp", + "--agent_address=127.0.0.1:8080", + "--runtime_ld_library_path=/tmp", + "--proc_metrics_cpu=2000", + "--proc_metrics_memory=2000", + R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" + }; functionsystem::runtime_manager::Flags flags; - flags.ParseFlags(13, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto future = executor_->StartInstance(startRequest, {}); @@ -1295,13 +1301,24 @@ TEST_F(RuntimeExecutorTest, GetNoneExistedCPPExecPathTest) deployConfig->set_deploydir(testDeployDir); deployConfig->set_storagetype("s3"); - const char *argv[]={"/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", "--runtime_initial_port=500", "--port_num=2000", - "--runtime_dir=/tmp", "--agent_address=127.0.0.1:8080", - "--runtime_ld_library_path=/tmp", "--proc_metrics_cpu=2000", "--proc_metrics_memory=2000", - R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})"}; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { + "/runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--port_num=2000", + "--runtime_dir=/tmp", + "--agent_address=127.0.0.1:8080", + "--runtime_ld_library_path=/tmp", + "--proc_metrics_cpu=2000", + "--proc_metrics_memory=2000", + R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" + }; functionsystem::runtime_manager::Flags flags; - flags.ParseFlags(13, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto future = executor_->StartInstance(startRequest, {}); @@ -1418,7 +1435,7 @@ TEST_F(RuntimeExecutorTest, GetValgrindMassifBuildArgs) "/runtime_manager", "--massif_enable=true", }; - flags.ParseFlags(2, argv); + flags.ParseFlags(std::size(argv), argv); executor_->Executor::SetRuntimeConfig(flags); auto startRequest = BuildStartInstanceRequest(GO_LANGUAGE); auto future = executor_->StartInstance(startRequest, {}); @@ -1444,9 +1461,16 @@ TEST_F(RuntimeExecutorTest, SetLD_LIBRARY_PATH) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1489,9 +1513,15 @@ TEST_F(RuntimeExecutorTest, SetEmptyLD_LIBRARY_PATH) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1530,9 +1560,15 @@ TEST_F(RuntimeExecutorTest, SetErrorLD_LIBRARY_PATH) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1580,9 +1616,15 @@ TEST_F(RuntimeExecutorTest, StartPosixCustomInstanceTest) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto startRequest = std::make_shared(); startRequest->set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); @@ -1653,11 +1695,13 @@ TEST_F(RuntimeExecutorTest, InheritEnvTest) env.posixEnvs.insert({CONDA_PREFIX, "/usr/local/conda"}); env.posixEnvs.insert({CONDA_DEFAULT_ENV, "env_name_file"}); litebus::os::SetEnv("PATH", "/inherit/path"); + litebus::os::SetEnv("YR_FRONTEND_ADDRESS", "127.0.0.1:0"); auto combineEnv = executor_->CombineEnvs(env); EXPECT_EQ(combineEnv["Inherit_env"], "123456"); EXPECT_EQ(combineEnv["user_env1"], "user_env1_value"); EXPECT_EQ(combineEnv["user_env2"], "user_env2_value"); EXPECT_EQ(combineEnv["LD_LIBRARY_PATH"], "/usr/posix/path"); + EXPECT_EQ(combineEnv["YR_FRONTEND_ADDRESS"], "127.0.0.1:0"); // append UNZIPPED_WORKING_DIR Into PYTHONPATH EXPECT_EQ( combineEnv["PYTHONPATH"], "/path/to/python_runtime:/python/path:/home/sn/function/package/xxx/working_dir/:/userdefined/pythonpath"); @@ -1689,6 +1733,17 @@ TEST_F(RuntimeExecutorTest, InheritEnvTest) EXPECT_TRUE(combineEnv.find("ASCEND_RT_VISIBLE_DEVICES") == combineEnv.end()); } +TEST_F(RuntimeExecutorTest, InheritEnvFalseTest) +{ + executor_->config_.inheritEnv = false; + auto env = Envs{}; + litebus::os::SetEnv("YR_FRONTEND_ADDRESS", "127.0.0.1:0"); + litebus::os::SetEnv("POD_FRONTEND_ADDRESS", "127.0.0.1:0"); + auto combineEnv = executor_->CombineEnvs(env); + EXPECT_EQ(combineEnv["YR_FRONTEND_ADDRESS"], "127.0.0.1:0"); + EXPECT_EQ(combineEnv["POD_FRONTEND_ADDRESS"], ""); +} + TEST_F(RuntimeExecutorTest, SeparatedRuntimeStdRedirected) { litebus::ExecIO stdOut = litebus::ExecIO::CreatePipeIO(); @@ -1733,9 +1788,15 @@ TEST_F(RuntimeExecutorTest, StartJobEntrypoint_WithoutWorkingDirTest) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances"}; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1763,12 +1824,14 @@ TEST_F(RuntimeExecutorTest, StartJobEntrypointInWorkingDirTest) CreatePythonEnvInfoScript(entrypointPath); runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", "--node_id=node1", "--runtime_ld_library_path=/tmp", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--agent_address=127.0.0.1:1234", "--runtime_std_log_dir=instances", @@ -1838,18 +1901,20 @@ TEST_F(RuntimeExecutorTest, StartPythonConda_WithWorkingDirTest) CreatePythonEnvInfoScript("/home/snuser/python/fnruntime/server.py"); runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", "--node_id=node1", "--runtime_ld_library_path=/tmp", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--agent_address=127.0.0.1:1234", "--runtime_std_log_dir=instances", "--data_system_port=24560", "--proxy_grpc_server_port=20258" }; - auto ret = flags.ParseFlags(11, argv); + auto ret = flags.ParseFlags(std::size(argv), argv); ASSERT_TRUE(ret.IsNone()) << ret.Get(); executor_->SetRuntimeConfig(flags); @@ -1903,9 +1968,16 @@ TEST_F(RuntimeExecutorTest, StartJobEntrypoint_InvalidWorkingDirTest) (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances"}; - flags.ParseFlags(7, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1940,9 +2012,17 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_false (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_direct_connection_enable=false"}; - flags.ParseFlags(8, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances", + "--runtime_direct_connection_enable=false" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -1980,9 +2060,17 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_serve (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_direct_connection_enable=true"}; - flags.ParseFlags(8, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances", + "--runtime_direct_connection_enable=true" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -2022,9 +2110,17 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_tls_s (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; - const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_direct_connection_enable=true"}; - flags.ParseFlags(8, argv); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + + const char *argv[] = { "/runtime_manager", + "--node_id=", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--runtime_std_log_dir=instances", + "--runtime_direct_connection_enable=true" }; + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); auto request1 = std::make_shared(); @@ -2065,16 +2161,18 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_tls_s (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=10", "--runtime_std_log_dir=instances", "--runtime_direct_connection_enable=true" }; - flags.ParseFlags(9, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); for (int k = 0; k < 11; k++) { @@ -2120,16 +2218,18 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_tls_s (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=10", "--runtime_std_log_dir=instances", "--runtime_direct_connection_enable=true" }; - flags.ParseFlags(9, argv); + flags.ParseFlags(std::size(argv), argv); executor_->SetRuntimeConfig(flags); for (int k = 0; k < 11; k++) { @@ -2149,11 +2249,12 @@ TEST_F(RuntimeExecutorTest, SetRuntimeEnv_runtime_direct_connection_enable_tls_s auto response = executor_->StartInstance(request1, {}).Get(); ASSERT_AWAIT_TRUE([k]() { - auto output = litebus::os::Read("/home/snuser/instances/-user_func_std.log"); + const auto output = litebus::os::Read("/home/snuser/instances/-user_func_std.log"); if (k < 11) { // 11th ok return output.IsSome() && (output.Get().find("RUNTIME_DIRECT_CONNECTION_ENABLE=true") != std::string::npos); } + return false; }); } (void)litebus::os::Rm("/home/snuser/instances/-user_func_std.log"); diff --git a/functionsystem/tests/unit/runtime_manager/healthcheck/health_check_test.cpp b/functionsystem/tests/unit/runtime_manager/healthcheck/health_check_test.cpp index ef0d6be6435705cd4aaec985ec457fa282424186..33b48025ba7f397c5a622e27f869595803a68d5a 100644 --- a/functionsystem/tests/unit/runtime_manager/healthcheck/health_check_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/healthcheck/health_check_test.cpp @@ -22,9 +22,9 @@ #include -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" +#include "common/utils/files.h" #include "utils/future_test_helper.h" -#include "files.h" namespace functionsystem::test { @@ -113,6 +113,12 @@ TEST_F(HealthCheckTest, HealthCheckWithKill) .WillOnce(FutureArg<2>(&msgValue)); litebus::Spawn(functionAgent); + std::string captureStr; + auto func = [&](const std::string runtimeID){ + captureStr = runtimeID; + }; + client->RegisterHandleLogPrefixExit(func); + auto execPtr = litebus::Exec::CreateExec("sleep 10", litebus::None(), litebus::ExecIO::CreatePipeIO(), litebus::ExecIO::CreatePipeIO(), litebus::ExecIO::CreatePipeIO(), {}, {}, false); @@ -126,6 +132,7 @@ TEST_F(HealthCheckTest, HealthCheckWithKill) auto info = req.instancestatusinfo(); EXPECT_EQ(0, execPtr->GetStatus().Get().Get()); EXPECT_EQ(info.instanceid(), "Instance-ID"); + EXPECT_EQ(captureStr, "runtime-ID"); EXPECT_TRUE(info.instancemsg().find("exitState:0 exitStatus:0") != std::string::npos); litebus::Terminate(functionAgent->GetAID()); @@ -148,7 +155,7 @@ TEST_F(HealthCheckTest, HealthCheckWithRuntimeMemoryExceedLimit) runtime_manager::Flags flags; const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", "--memory_detection_interval=200", "--oom_kill_enable=true" }; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); client->SetConfig(flags); const std::string instanceID = "Instance-ID"; const std::string requestID = "Request-ID"; @@ -200,7 +207,7 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExceptionExitWithExceptionLog) .WillOnce(FutureArg<2>(&msgValue)); litebus::Spawn(functionAgent); - const std::string &exceptionLog = "/home/snuser/exception"; + const std::string &exceptionLog = "/home/snuser/health_check/exception"; if (!litebus::os::ExistPath(exceptionLog)) { litebus::os::Mkdir(exceptionLog); } @@ -214,7 +221,7 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExceptionExitWithExceptionLog) outfile << "runtime ID backtrace log. This is a Test." << std::endl; outfile.close(); - const std::string &stdLog = "/home/snuser/instances/"; + const std::string &stdLog = "/home/snuser/health_check/instances/"; if (!litebus::os::ExistPath(stdLog)) { litebus::os::Mkdir(stdLog); } @@ -235,8 +242,8 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExceptionExitWithExceptionLog) auto client = std::make_shared(); runtime_manager::Flags flags; const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_logs_dir=/home/snuser/health_check" }; + flags.ParseFlags(std::size(argv), argv); client->SetConfig(flags); client->AddRuntimeRecord(functionAgent->GetAID(), execPtr->GetPid(), "Instance-ID", "runtime-ID", "runtime-ID"); @@ -270,7 +277,7 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExceptionExitWithStdLog) .WillOnce(FutureArg<2>(&msgValue)); litebus::Spawn(functionAgent); - const std::string &stdLog = "/home/snuser/instances"; + const std::string &stdLog = "/home/snuser/health_check/instances"; if (!litebus::os::ExistPath(stdLog)) { litebus::os::Mkdir(stdLog); } @@ -292,8 +299,8 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExceptionExitWithStdLog) auto client = std::make_shared(); runtime_manager::Flags flags; const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_logs_dir=/home/snuser/health_check" }; + flags.ParseFlags(std::size(argv), argv); client->SetConfig(flags); client->AddRuntimeRecord(functionAgent->GetAID(), execPtr->GetPid(), "Instance-ID", "runtime-ID", "runtime-ID"); @@ -317,7 +324,7 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExit) .WillOnce(FutureArg<2>(&msgValue)); litebus::Spawn(functionAgent); - const std::string &stdLog = "/home/snuser/instances"; + const std::string &stdLog = "/home/snuser/health_check/instances"; if (!litebus::os::ExistPath(stdLog)) { litebus::os::Mkdir(stdLog); } @@ -339,8 +346,8 @@ TEST_F(HealthCheckTest, HealthCheckWhenRuntimeExit) runtime_manager::Flags flags; client->SetConfig(flags); const char *argv[] = { "/runtime_manager", "--node_id=", "--ip=127.0.0.1","--host_ip=127.0.0.1", "--port=32233", - "--runtime_initial_port=500", "--runtime_std_log_dir=instances" }; - flags.ParseFlags(7, argv); + "--runtime_initial_port=500", "--runtime_std_log_dir=instances", "--runtime_logs_dir=/home/snuser/health_check" }; + flags.ParseFlags(std::size(argv), argv); client->SetConfig(flags); client->AddRuntimeRecord(functionAgent->GetAID(), execPtr->GetPid(), "Instance-ID", "runtime-ID", "runtime-ID"); diff --git a/functionsystem/tests/unit/runtime_manager/log/log_manager_test.cpp b/functionsystem/tests/unit/runtime_manager/log/log_manager_test.cpp index edc43c4489c02f4fe5117f3cfeaadabd2deb92e6..41fcc1b4704a407b9bcd595a99242df6f32f6b81 100644 --- a/functionsystem/tests/unit/runtime_manager/log/log_manager_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/log/log_manager_test.cpp @@ -16,18 +16,18 @@ #include "runtime_manager/log/log_manager.h" +#include #include #include #include -#include #include -#include "utils/os_utils.hpp" -#include "logs/logging.h" -#include "../manager/runtime_manager_test_actor.h" +#include "common/logs/logging.h" +#include "runtime_manager/manager/runtime_manager_test_actor.h" #include "utils/future_test_helper.h" #include "utils/generate_info.h" +#include "utils/os_utils.hpp" namespace functionsystem::test { using namespace functionsystem::runtime_manager; @@ -42,9 +42,11 @@ const std::string STD_LOG_DIR = "/tmp/snuser/log/instances/"; class LogManagerActorHelper : public runtime_manager::LogManagerActor { public: - explicit LogManagerActorHelper(const std::string &name, const litebus::AID &runtimeManagerAID) - : LogManagerActor(name, runtimeManagerAID){}; + explicit LogManagerActorHelper(const std::string &name, const litebus::AID &runtimeManagerAID, + bool logReuse = false) + : LogManagerActor(name, runtimeManagerAID, logReuse) {}; MOCK_METHOD(litebus::Future, IsRuntimeActive, (const std::string &runtimeID), (const)); + MOCK_METHOD(litebus::Future, IsRuntimeActiveByPid, (const pid_t &pid), (const)); }; class LogManagerTest : public testing::Test { @@ -133,6 +135,32 @@ public: outfile.close(); } + void MockCreateDsClientLogs() + { + std::vector logFiles = { + "ds_client_12345.DEBUG", + "ds_client_12345.DEBUG.log", + "ds_client_12345.INFO", + "ds_client_12345.INFO.log", + "ds_client_12345.WARNING", + "ds_client_12345.WARNING.log", + "ds_client_12345.ERROR", + "ds_client_12345.ERROR.log", + "ds_client_access_12345.log" + }; + (void)litebus::os::Mkdir(LOG_BASE_DIR); + for (const auto& logFile : logFiles) { + auto logPath = litebus::os::Join(LOG_BASE_DIR, logFile); + auto fd = open(logPath.c_str(), O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + std::ofstream outfile; + outfile.open(logPath.c_str()); + outfile << logPath << " This is a Test." << std::endl; + outfile.close(); + } + } + void MockCreateCppRuntimeRollingLogsWithCompression() { auto jobId = litebus::uuid_generator::UUID::GetRandomUUID().ToString().substr(0, 8); @@ -255,10 +283,59 @@ public: MockCreateRuntimeStdLogs(); } + void MockCreatePrefixLogs() + { + (void)litebus::os::Mkdir(LOG_BASE_DIR); + std::vector logNameList = { + "YR_123_000001_abc_libruntime.log", + "YR_123_000001_abc_libruntime.2025081617281825.log", + "YR_123_000001_abc_libruntime.2025081801463525.log.gz", + "YR_123_000001_abc_libruntime.2025082901463525.log.gz.1", + "YR_123_000002_runtime.log", + "YR_123_000002_runtime.1.log", + "YR_123_000002_runtime.log.gz", + "YR_123_000002_runtime.log.gz.10086", + "YR_123_000003_runtime.log", + "YR_123_000003_runtime.1755842461755699.log.gz", + "YR_123_000003_runtime.1755971816292456.log.gz.1123", + "YR_123_000003_libruntime.log", + "YR_123_000003_libruntime.1755842461755699.log.gz", + }; + for (auto logName : logNameList) { + std::string logFileName = litebus::os::Join(LOG_BASE_DIR, logName); + + int fd = open(logFileName.c_str(), O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + + std::ofstream outfile; + outfile.open(logFileName); + outfile << "runtime log #" << logFileName << ". This is a Test." << std::endl; + outfile.close(); + YRLOG_DEBUG("Creating log files for: {}", logFileName); + } + } + + void MockCreateManyPrefixLogs(int cnt = 100) + { + (void)litebus::os::Mkdir(LOG_BASE_DIR); + for (int i = 0; i < cnt; i++) { + std::string logFileName = litebus::os::Join(LOG_BASE_DIR, "YR_123_000005_xxx." + std::to_string(i) + ".log"); + int fd = open(logFileName.c_str(), O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + + std::ofstream outfile; + outfile.open(logFileName); + outfile << "runtime log #" << logFileName << ". This is a Test." << std::endl; + outfile.close(); + YRLOG_DEBUG("Creating log files for: {}", logFileName); + } + } + void SetUp() override { (void)litebus::os::Rmdir(LOG_BASE_DIR); - testActor_ = std::make_shared(GenerateRandomName("randomRuntimeManagerTestActor")); litebus::Spawn(testActor_, true); @@ -274,6 +351,8 @@ public: litebus::Terminate(testActor_->GetAID()); litebus::Await(testActor_->GetAID()); + helper_ = nullptr; + testActor_ = nullptr; } protected: @@ -294,7 +373,7 @@ TEST_F(LogManagerTest, EmptyLogDir) const char *argv[] = { "./runtime-manager", "--runtime_logs_dir=/tmp/snuser/log", "--log_expiration_enable=true", "--log_expiration_cleanup_interval=0", "--log_expiration_max_file_count=100" }; runtime_manager::Flags flags; - flags.ParseFlags(5, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); helper_->ScanLogsRegularly(); @@ -311,7 +390,7 @@ TEST_F(LogManagerTest, LogFileExpirationNotExpired1) const char *argv[] = { "./runtime-manager", "--runtime_logs_dir=/tmp/snuser/log", "--log_expiration_enable=true", "--log_expiration_cleanup_interval=0", "--log_expiration_max_file_count=100" }; runtime_manager::Flags flags; - flags.ParseFlags(5, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); helper_->ScanLogsRegularly(); @@ -344,7 +423,7 @@ TEST_F(LogManagerTest, LogFileExpirationNotExpired2) "--runtime_std_log_dir=instances" }; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait for log expiration @@ -375,7 +454,7 @@ TEST_F(LogManagerTest, LogFileExpirationNotExpired3) "--log_expiration_max_file_count=10", "--runtime_std_log_dir=instances"}; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(1500)); // wait for log expiration @@ -409,7 +488,7 @@ TEST_F(LogManagerTest, LogFileExpirationExpired1) "--runtime_std_log_dir=instances" }; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(1500)); // wait for log expiration @@ -437,7 +516,7 @@ TEST_F(LogManagerTest, LogFileExpirationExpired2) "--runtime_std_log_dir=instances" }; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(1500)); // wait for log expiration @@ -469,7 +548,7 @@ TEST_F(LogManagerTest, LogFileExpirationExpired3) "--log_expiration_max_file_count=0" // delete all expired log }; runtime_manager::Flags flags; - flags.ParseFlags(6, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(500)); // wait for log expiration @@ -522,7 +601,7 @@ TEST_F(LogManagerTest, LogFileExpirationExpiredAsync) "--runtime_std_log_dir=instances" }; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); helper_->ScanLogsRegularly(); @@ -558,13 +637,13 @@ TEST_F(LogManagerTest, LogFileExpirationComplexCaseWithRollingCompressionTest) "./runtime-manager", "--runtime_logs_dir=/tmp/snuser/log", "--log_expiration_enable=true", - "--log_expiration_cleanup_interval=10", // execute once in this ut case + "--log_expiration_cleanup_interval=1", "--log_expiration_time_threshold=1", "--log_expiration_max_file_count=1", // keep 1 expired log "--runtime_std_log_dir=instances" }; runtime_manager::Flags flags; - flags.ParseFlags(7, argv); + flags.ParseFlags(std::size(argv), argv); helper_->SetConfig(flags); std::this_thread::sleep_for(std::chrono::milliseconds(2000)); // wait for log expiration @@ -573,5 +652,325 @@ TEST_F(LogManagerTest, LogFileExpirationComplexCaseWithRollingCompressionTest) auto files = litebus::os::Ls(LOG_BASE_DIR); return files.Get().size() == static_cast(3); // expect 'exception' and 'instances' dir + 1 }); + auto files = litebus::os::Ls(LOG_BASE_DIR); +} + +TEST_F(LogManagerTest, DsClientLogFileExpirationNotExpired) +{ + MockCreateDsClientLogs(); + + // Set runtime inActive + EXPECT_CALL(*helper_, IsRuntimeActiveByPid(_)).WillRepeatedly(Return(true)); + + const char *argv[] = { + "./runtime-manager", + "--runtime_logs_dir=/tmp/snuser/log", + "--log_expiration_enable=true", + "--log_expiration_cleanup_interval=10", // execute once in this ut case + "--log_expiration_time_threshold=1", + "--log_expiration_max_file_count=0", // delete all expired log + "--runtime_std_log_dir=instances" + }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + helper_->SetConfig(flags); + + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); // wait for log expiration + helper_->ScanLogsRegularly(); + + EXPECT_AWAIT_TRUE([=]() -> bool { + auto files = litebus::os::Ls(LOG_BASE_DIR); + return files.Get().size() == static_cast(9); // nothing deleted + }); +} + +TEST_F(LogManagerTest, DsClientLogFileExpirationExpired) +{ + MockCreateDsClientLogs(); + + // Set runtime inActive + EXPECT_CALL(*helper_, IsRuntimeActiveByPid(_)).WillRepeatedly(Return(false)); + + const char *argv[] = { + "./runtime-manager", + "--runtime_logs_dir=/tmp/snuser/log", + "--log_expiration_enable=true", + "--log_expiration_cleanup_interval=10", // execute once in this ut case + "--log_expiration_time_threshold=1", + "--log_expiration_max_file_count=0", // delete all expired log + "--runtime_std_log_dir=instances" + }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + helper_->SetConfig(flags); + + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); // wait for log expiration + helper_->ScanLogsRegularly(); + + EXPECT_AWAIT_TRUE([=]() -> bool { + auto files = litebus::os::Ls(LOG_BASE_DIR); + return files.Get().size() == static_cast(0); // everything is deleted + }); +} + +TEST_F(LogManagerTest, AcquireLogPrefix) +{ + auto pid = std::to_string(getpid()); + // 0. mock actor_->logCount_ > MaxCount + helper_->logCount_ = 999998; + auto res1 = helper_->AcquireLogPrefix("runtime1"); + auto exp1 = "YR_" + pid + "_000000"; + EXPECT_EQ(res1.Get(), exp1); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp1], "runtime1"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime1"], exp1); + + // 1. get and generate + auto res2 = helper_->AcquireLogPrefix("runtime2"); + auto res3 = helper_->AcquireLogPrefix("runtime3"); + + auto exp2 = "YR_" + pid + "_000001"; + auto exp3 = "YR_" + pid + "_000002"; + auto exp4 = "YR_" + pid + "_000003"; + + EXPECT_EQ(res2.Get(), exp2); + EXPECT_EQ(res3.Get(), exp3); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp2], "runtime2"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime2"], exp2); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp3], "runtime3"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime3"], exp3); + + // 2. get from queue and then generate + helper_->ReleaseLogPrefix("runtime2"); + helper_->ReleaseLogPrefix("runtime3"); + + res1 = helper_->AcquireLogPrefix("runtime4"); + res2 = helper_->AcquireLogPrefix("runtime5"); + + EXPECT_EQ(res1.Get(), exp3); + EXPECT_EQ(res2.Get(), exp2); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp3], "runtime4"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime4"], exp3); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp2], "runtime5"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime5"], exp2); + + res2 = helper_->AcquireLogPrefix("runtime6"); + EXPECT_EQ(res2.Get(), exp4); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp4], "runtime6"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime6"], exp4); +} + +TEST_F(LogManagerTest, ReleaseLogPrefix) +{ + auto pid = std::to_string(getpid()); + + auto res1 = helper_->AcquireLogPrefix("runtime1"); + auto exp1 = "YR_" + pid + "_000001"; + EXPECT_EQ(res1.Get(), exp1); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp1], "runtime1"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime1"], exp1); + + // try to release failed + helper_->ReleaseLogPrefix("runtime2"); + EXPECT_EQ(helper_->logPrefix2RuntimeID_[exp1], "runtime1"); + EXPECT_EQ(helper_->runtimeID2LogPrefix_["runtime1"], exp1); + + // try to release success + helper_->ReleaseLogPrefix("runtime1"); + EXPECT_EQ(helper_->logPrefix2RuntimeID_.find(exp1), helper_->logPrefix2RuntimeID_.end()); + EXPECT_EQ(helper_->runtimeID2LogPrefix_.find("runtime1"), helper_->runtimeID2LogPrefix_.end()); + EXPECT_EQ(helper_->logPrefixDequeue_.front(), exp1); + + // try to release but not match + helper_->runtimeID2LogPrefix_["runtime2"] = "log1111"; + helper_->logPrefix2RuntimeID_["log2222"] = "runtime2"; + helper_->runtimeID2LogPrefix_["runtime3"] = "log2222"; + + helper_->ReleaseLogPrefix("runtime2"); + EXPECT_TRUE(helper_->logPrefix2RuntimeID_.find("log2222") != helper_->logPrefix2RuntimeID_.end()); + + helper_->ReleaseLogPrefix("runtime3"); + EXPECT_TRUE(helper_->logPrefix2RuntimeID_.find("log2222") != helper_->logPrefix2RuntimeID_.end()); +} + +TEST_F(LogManagerTest, AddLogAndPopLogTest) +{ + int toDeleteCount = 200; + for (int i = 0; i < toDeleteCount; i++) { + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_"+std::to_string(i), 1756216109, false), + false); + } + std::vector> toBeDeleteFiles; + for (size_t i = 0; i < toDeleteCount; ++i) { + auto file = helper_->expiredLogQueue_->PopLogFile(); + if (file == nullptr){ + break; + } + toBeDeleteFiles.emplace_back(file); + } + EXPECT_TRUE(toBeDeleteFiles.size() == toDeleteCount); + +} + +TEST_F(LogManagerTest, AddAndPopLogTest) +{ + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_0", 1756216109, false), + true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/java/file_"+std::to_string(0), 1756216110, false), + true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/java/file_"+std::to_string(1), 1756216110, false), + true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/java/file_"+std::to_string(2), 1756216110, false), + true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/javaDir/", 1756216110, true), + true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_100", 1756216111, false), + true); + std::vector expectDir = {false, false, false, false, true, false}; + std::vector expectStr = {"/tmp/xxx_0", "/java/file_", "/java/file_", "/java/file_", "/javaDir/", "/tmp/xxx_100",}; + for (size_t i = 0; i < 6; ++i) { + auto file = helper_->expiredLogQueue_->PopLogFile(); + EXPECT_TRUE(file->GetFilePath().substr(0, expectStr[i].size()) == expectStr[i]); + EXPECT_EQ(file->IsDir(), expectDir[i]); + } + ASSERT_TRUE(helper_->expiredLogQueue_->PopLogFile() == nullptr); + + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_1", 1756216109, false), true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_2", 1756216109, false), true); + helper_->expiredLogQueue_->AddLogFile(std::make_shared("", "/tmp/xxx_3", 1756216109, false), true); + helper_->expiredLogQueue_->Reset(); + ASSERT_TRUE(helper_->expiredLogQueue_->PopLogFile() == nullptr); +} + +TEST_F(LogManagerTest, RecycleReuseLogTest) +{ + MockCreatePrefixLogs(); + MockCreateDsClientLogs(); + MockCreateManyPrefixLogs(200); + helper_->logReuse_ = true; + + // Set runtime inActive + EXPECT_CALL(*helper_, IsRuntimeActive(_)).WillRepeatedly(Return(false)); + EXPECT_CALL(*helper_, IsRuntimeActiveByPid(_)).WillRepeatedly(Return(false)); + + const char *argv[] = { + "./runtime-manager", + "--runtime_logs_dir=/tmp/snuser/log", + "--log_expiration_enable=true", + "--log_expiration_cleanup_interval=10", + "--log_expiration_time_threshold=0", + "--log_expiration_max_file_count=0" // delete all expired log + }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + helper_->SetConfig(flags); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for log expiration + helper_->ScanLogsRegularly(); + + EXPECT_AWAIT_TRUE([=]() -> bool { + auto files = litebus::os::Ls(LOG_BASE_DIR); + return files.Get().size() == static_cast(0); + }); + auto files = litebus::os::Ls(LOG_BASE_DIR); +} + +TEST_F(LogManagerTest, RecycleReuseLogWhileReusingTest) +{ + MockCreatePrefixLogs(); + MockCreateDsClientLogs(); + MockCreateManyPrefixLogs(200); + helper_->logReuse_ = true; + helper_->logPrefix2RuntimeID_["YR_123_000003"] = "runtime3"; + helper_->logPrefix2RuntimeID_["YR_123_000002"] = "runtime2"; + + // Set runtime inActive + EXPECT_CALL(*helper_, IsRuntimeActive(_)) + .WillRepeatedly([](const std::string &runtimeID) { + if (runtimeID == "runtime3") { + return litebus::Future(true); + } + return litebus::Future(false); + }); + EXPECT_CALL(*helper_, IsRuntimeActiveByPid(_)).WillOnce(Return(true)).WillRepeatedly(Return(false)); + + const char *argv[] = { + "./runtime-manager", + "--runtime_logs_dir=/tmp/snuser/log", + "--log_expiration_enable=true", + "--log_expiration_cleanup_interval=30", // execute once in this ut case + "--log_expiration_time_threshold=0", + "--log_expiration_max_file_count=3" // reverse 3 + }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + helper_->SetConfig(flags); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for log expiration + helper_->ScanLogsRegularly(); + + EXPECT_AWAIT_TRUE_FOR([=]() -> bool { + auto files = litebus::os::Ls(LOG_BASE_DIR); + return files.Get().size() == static_cast(6); // 2 reuse log + 1 ds log + reverse 3 + }, 5000); + auto files = litebus::os::Ls(LOG_BASE_DIR); +} + +TEST_F(LogManagerTest, RecycleReuseLogWhileReusingAsyncTest) +{ + MockCreatePrefixLogs(); + MockCreateDsClientLogs(); + MockCreateManyPrefixLogs(200); + helper_->logReuse_ = true; + helper_->logPrefix2RuntimeID_["YR_123_000003"] = "runtime3"; + helper_->logPrefix2RuntimeID_["YR_123_000002"] = "runtime2"; + + // Set runtime inActive + litebus::Promise promise1; + litebus::Promise promise2; + EXPECT_CALL(*helper_, IsRuntimeActive(_)) + .WillRepeatedly([javaRuntimeID(javaRuntimeID_), cppRuntimeID(cppRuntimeID_), &promise1, + &promise2](const std::string &runtimeID) { + if (runtimeID == "runtime3") { + return promise1.GetFuture(); + } else if (runtimeID == "runtime2") { + return promise2.GetFuture(); + } + return litebus::Future(true); + }); + EXPECT_CALL(*helper_, IsRuntimeActiveByPid(_)).WillOnce(Return(true)).WillRepeatedly(Return(false)); + + + // async IsRuntimeActive + std::thread runtimeThread1([&promise1]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + promise1.SetValue(true); + }); + + std::thread runtimeThread2([&promise2]() { + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + promise2.SetValue(false); + }); + + const char *argv[] = { + "./runtime-manager", + "--runtime_logs_dir=/tmp/snuser/log", + "--log_expiration_enable=true", + "--log_expiration_cleanup_interval=30", + "--log_expiration_time_threshold=0", + "--log_expiration_max_file_count=3" // reverse 3 + }; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + helper_->SetConfig(flags); + helper_->ScanLogsRegularly(); + // wait async threads done + runtimeThread1.join(); + runtimeThread2.join(); + + EXPECT_AWAIT_TRUE_FOR([=]() -> bool { + auto files = litebus::os::Ls(LOG_BASE_DIR); + return files.Get().size() == static_cast(6); // 2 reuse log + 1 ds log + reverse 3 + }, 5000); + auto files = litebus::os::Ls(LOG_BASE_DIR); } } \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test.cpp b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test.cpp index bc709bdbb30dbdc896d2c4e0838d673e20756a70..2de3dca50632935611c13730cadb31b93043dafd 100644 --- a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test.cpp @@ -14,25 +14,117 @@ * limitations under the License. */ -#include "files.h" +#include "runtime_manager/manager/runtime_manager.h" + +#include +#include "common/utils/files.h" #include "exec/exec.hpp" #include "gtest/gtest.h" #include "runtime_manager/port/port_manager.h" #include "runtime_manager_test_actor.h" #include "utils/future_test_helper.h" -#include "utils/port_helper.h" #include "utils/generate_info.h" +#include "utils/port_helper.h" -#include "runtime_manager/manager/runtime_manager.h" using namespace functionsystem::test; namespace functionsystem::runtime_manager { const uint32_t MAX_REGISTER_TEST_TIMES = 5; -const uint32_t INITIAL_PORT = 600; +const uint32_t INITIAL_PORT = 700; const uint32_t PORT_NUM = 800; const std::string testDeployDir = "/tmp/layer/func/bucket-test-log1/yr-test-runtime-manager"; const std::string funcObj = testDeployDir + "/" + "funcObj"; +const std::string POST_START_EXEC_REGEX = R"(^(uv )?pip3.[0-9]* install [a-zA-Z0-9\-\s:/\.=_]* && pip3.[0-9]* check$)"; + +const std::vector debugServerScriptContent = { + "import socket", + "import threading", + "import os", + "import signal", + "import time", + "import argparse", + + "def forward_data(source, destination):", + " try:", + " while True:", + " data = source.recv(4096) # ���ն��������ݣ������н���", + " if not data: # ���ӹر�", + " break", + " destination.sendall(data) # ����ԭʼ����������", + " except (ConnectionResetError, BrokenPipeError, socket.error) as e:", + " print(f\"Connection error in forward_data: {e}\")", + " finally:", + " source.shutdown(socket.SHUT_RDWR)", + " source.close()", + " destination.shutdown(socket.SHUT_RDWR)", + " destination.close()", + + "def process_b(port):", + " # ����socket������", + " server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)", + " server_socket.bind(('localhost', port))", + " server_socket.listen(2) # ��Ҫ����A��C��������", + + " while True:", + " print(\"B�������������ȴ�����...\")", + + " # ���Ƚ���C���̵����ӺͶ˿���Ϣ", + " conn_c, addr_c = server_socket.accept()", + " print(f\"B����: �յ�����C���̵����� {addr_c}\")", + " # ��C���̽��ն˿���Ϣ", + " recved = conn_c.recv(1024).strip().decode()", + " c_port, c_pid = recved.split(' ')", + " c_port, c_pid = int(c_port), int(c_pid)", + " print(f\"B����: C����ʵ�ʶ˿�Ϊ {c_port}�����̺� {c_pid}\")", + " conn_c.close()", + + " # Ȼ�����A���̵�����", + " conn_a, addr_a = server_socket.accept()", + " print(f\"B����: �յ�����A���̵����� {addr_a}\")", + + " # ����SIGCONT������C", + " os.kill(c_pid, signal.SIGCONT)", + " time.sleep(0.5)", + + " # ���ӵ�C���̵�ʵ�ʶ˿�", + " conn_c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)", + " conn_c.connect(('localhost', c_port))", + " print(\"B����: �����ӵ�C���̵�ʵ�ʶ˿�\")", + + " t1 = threading.Thread(target=forward_data, args=(conn_c, conn_a))", + " t2 = threading.Thread(target=forward_data, args=(conn_a, conn_c))", + + " t1.start()", + " t2.start()", + " t1.join()", + " t2.join()", + + " server_socket.close()", + + "if __name__ == \"__main__\":", + " parser = argparse.ArgumentParser(description=\"Proxy server\")", + " parser.add_argument('--port', type=int, help=\"Port number to listen on\", default=5555)", + " args = parser.parse_args()", + " process_b(args.port)" +}; + class RuntimeManagerTest : public ::testing::Test { public: + [[maybe_unused]] static void SetUpTestSuite() + { + if (!litebus::os::ExistPath("/tmp/cpp/bin")) { + litebus::os::Mkdir("/tmp/cpp/bin"); + } + + auto fd = open("/tmp/cpp/bin/runtime", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + EXPECT_NE(fd, -1); + close(fd); + + std::ofstream outfile; + outfile.open("/tmp/cpp/bin/runtime"); + outfile << "sleep 2" << std::endl; + outfile.close(); + } + void SetUp() override { PortManager::GetInstance().InitPortResource(INITIAL_PORT, PORT_NUM); @@ -63,6 +155,29 @@ public: sigReceived_.SetValue(true); } + messages::StartInstanceRequest GenStartInstanceRequest() + { + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("cpp"); + auto userEnvs = runtimeConfig->mutable_userenvs(); + userEnvs->insert({ "user_env1", "user_env1_value" }); + userEnvs->insert({ "user_env2", "user_env2_value" }); + + auto deployConfig = runtimeInfo->mutable_deploymentconfig(); + deployConfig->set_objectid("test_objectID"); + deployConfig->set_bucketid("test_bucketID"); + deployConfig->set_deploydir(testDeployDir); + deployConfig->set_storagetype("s3"); + return startRequest; + } + protected: std::string runtimeManagerActorName_; std::shared_ptr manager_; @@ -70,42 +185,100 @@ protected: inline static litebus::Future sigReceived_; }; +class RuntimeManagerDebugServerTest : public RuntimeManagerTest { +public: + void SetUp() override + { + auto optionEnv = litebus::os::GetEnv("PATH"); + if (optionEnv.IsSome()) { + env_ = optionEnv.Get(); + } + + RuntimeManagerTest::SetUp(); + litebus::os::SetEnv("PATH", litebus::os::Join("/tmp", env_, ':')); + (void)litebus::os::Rm("/tmp/gdbserver"); + auto fd = open("/tmp/gdbserver", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + close(fd); + EXPECT_AWAIT_TRUE([=]() { return FileExists("/tmp/gdbserver"); }); + (void)litebus::os::Rm("/tmp/python3"); + fd = open("/tmp/python3", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + close(fd); + EXPECT_AWAIT_TRUE([=]() { return FileExists("/tmp/python3"); }); + + // python debug server script + (void)litebus::os::Rm("/tmp/python/fnruntime/debug_server.py"); + if (!litebus::os::ExistPath("/tmp/python/fnruntime")) { + litebus::os::Mkdir("/tmp/python/fnruntime"); + } + std::string debugServerScriptPath = litebus::os::Join("/tmp/python/fnruntime", "debug_server.py"); + (void)litebus::os::Rm(debugServerScriptPath); + TouchFile(debugServerScriptPath); + std::ofstream outfile(debugServerScriptPath); + if (outfile.is_open()) { + for (const auto& line : debugServerScriptContent) { + outfile << line << std::endl; + } + outfile.close(); + } else { + std::cerr << "cannot open file" << std::endl; + } + + functionsystem::runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { + "/runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--port_num=2000", + "--runtime_dir=/tmp", + const_cast("123.123.123.123:80"), + "--runtime_ld_library_path=/tmp", + "--proc_metrics_cpu=2000", + "--proc_metrics_memory=2000", + "--runtime_instance_debug_enable=true", + R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" + }; + flags.ParseFlags(std::size(argv), argv); + manager_->SetRegisterHelper(std::make_shared("node1-RuntimeManagerSrv")); + manager_->SetConfig(flags); + } + + void TearDown() override + { + litebus::os::SetEnv("PATH", env_); + (void)litebus::os::Rm("/tmp/gdbserver"); + (void)litebus::os::Rm("/tmp/python3"); + RuntimeManagerTest::TearDown(); + } + + static inline std::string env_; +}; + TEST_F(RuntimeManagerTest, StartInstanceTest) { - (void)litebus::os::Rm("/tmp/cpp/bin/runtime"); - if (!litebus::os::ExistPath("/tmp/cpp/bin")) { - litebus::os::Mkdir("/tmp/cpp/bin"); - } - auto fd = open("/tmp/cpp/bin/runtime", O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - EXPECT_NE(fd, -1); - close(fd); - std::ofstream outfile; - outfile.open("/tmp/cpp/bin/runtime"); - outfile << "sleep 2" << std::endl; - outfile.close(); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); const char *argv[] = { "/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=2000", "--runtime_dir=/tmp" }; functionsystem::runtime_manager::Flags flags; - flags.ParseFlags(8, argv); + flags.ParseFlags(std::size(argv), argv); manager_->SetConfig(flags); testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto startRequest = GenStartInstanceRequest(); auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); auto resources = runtimeInfo->mutable_runtimeconfig()->mutable_resources()->mutable_resources(); resource_view::Resource cpuResource; cpuResource.set_type(::resources::Value_Type::Value_Type_SCALAR); @@ -115,28 +288,16 @@ TEST_F(RuntimeManagerTest, StartInstanceTest) memResource.set_type(::resources::Value_Type::Value_Type_SCALAR); memResource.mutable_scalar()->set_value(500.0); (*resources)["Memory"] = memResource; - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); // lost connection with function agent manager_->connected_ = false; testActor_->StartInstance(manager_->GetAID(), startRequest); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(testActor_->GetIsReceiveStartInstanceResponse(), false); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStartInstanceResponse(); }); + manager_->connected_ = true; // repeat testActor_->ResetStartInstanceTimes(); - std::unordered_set receivedStartingReq{ "repeat-123" }; - manager_->receivedStartingReq_ = receivedStartingReq; + manager_->receivedStartingReq_ = std::unordered_set{ "repeat-123" }; messages::StartInstanceRequest repeatRequest; startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); repeatRequest.mutable_runtimeinstanceinfo()->set_requestid("repeat-123"); @@ -177,25 +338,7 @@ TEST_F(RuntimeManagerTest, StartInstanceWithPreStartSuccessTest) testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); - auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); - + auto startRequest = GenStartInstanceRequest(); testActor_->StartInstance(manager_->GetAID(), startRequest); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); @@ -218,25 +361,10 @@ TEST_F(RuntimeManagerTest, StartInstance_PosixCustomRuntime_WithEntryfileEmpty) testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto startRequest = GenStartInstanceRequest(); auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); runtimeConfig->set_language("posix-custom-runtime"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); - testActor_->StartInstance(manager_->GetAID(), startRequest); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); @@ -255,26 +383,10 @@ TEST_F(RuntimeManagerTest, StartInstanceWithPreStartFailedTest) testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto startRequest = GenStartInstanceRequest(); auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); runtimeConfig->mutable_posixenvs()->insert({ "POST_START_EXEC", "/usr/bin/cp a b;" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); - testActor_->StartInstance(manager_->GetAID(), startRequest); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); @@ -302,13 +414,8 @@ TEST_F(RuntimeManagerTest, StartInstance_PosixCustomRuntime_POST_START_EXEC_pip_ testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto startRequest = GenStartInstanceRequest(); auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); runtimeConfig->set_language("posix-custom-runtime"); runtimeConfig->set_entryfile("echo hello"); @@ -320,14 +427,6 @@ TEST_F(RuntimeManagerTest, StartInstance_PosixCustomRuntime_POST_START_EXEC_pip_ // both UNZIPPED_WORKING_DIR and YR_WORKING_DIR are required. runtimeConfig->mutable_posixenvs()->insert({ "UNZIPPED_WORKING_DIR", "/tmp" }); runtimeConfig->mutable_posixenvs()->insert({ "YR_WORKING_DIR", "file:///tmp/file.zip" }); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); testActor_->StartInstance(manager_->GetAID(), startRequest); @@ -429,25 +528,8 @@ TEST_F(RuntimeManagerTest, StartInstanceWithInvalidExecutorTypeTest) testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; + auto startRequest = GenStartInstanceRequest(); startRequest.set_type(static_cast(EXECUTOR_TYPE::UNKNOWN)); - auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); - testActor_->StartInstance(manager_->GetAID(), startRequest); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); @@ -534,28 +616,11 @@ TEST_F(RuntimeManagerTest, StopInstanceTest) testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); litebus::Spawn(testActor_, true); - messages::StartInstanceRequest startRequest; - startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); - auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); - runtimeInfo->set_requestid("test_requestID"); - runtimeInfo->set_instanceid("test_instanceID"); - runtimeInfo->set_traceid("test_traceID"); - - auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); - runtimeConfig->set_language("cpp"); - auto userEnvs = runtimeConfig->mutable_userenvs(); - userEnvs->insert({ "user_env1", "user_env1_value" }); - userEnvs->insert({ "user_env2", "user_env2_value" }); - - auto deployConfig = runtimeInfo->mutable_deploymentconfig(); - deployConfig->set_objectid("test_objectID"); - deployConfig->set_bucketid("test_bucketID"); - deployConfig->set_deploydir(testDeployDir); - deployConfig->set_storagetype("s3"); - + auto startRequest = GenStartInstanceRequest(); testActor_->StartInstance(manager_->GetAID(), startRequest); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); + EXPECT_FALSE(manager_->runtimeInstanceDebugEnable_); auto response = testActor_->GetStartInstanceResponse(); EXPECT_EQ(StatusCode::SUCCESS, response->code()); @@ -575,8 +640,7 @@ TEST_F(RuntimeManagerTest, StopInstanceTest) // lost connection with function agent manager_->connected_ = false; testActor_->StopInstance(manager_->GetAID(), request); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(testActor_->GetIsReceiveStopInstanceResponse(), false); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStopInstanceResponse(); }); manager_->connected_ = true; // success @@ -654,12 +718,13 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentFailedTest) functionsystem::runtime_manager::Flags flags; std::string agentAddress = "--agent_address=" + testActor_->GetAID().Url(); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); const char *argv[] = { "/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=2000", "--runtime_dir=/tmp", @@ -669,7 +734,7 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentFailedTest) "--proc_metrics_memory=2000", R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" }; - flags.ParseFlags(13, argv); + flags.ParseFlags(std::size(argv), argv); manager_->SetRegisterHelper(std::make_shared("node1-RuntimeManagerSrv")); manager_->SetConfig(flags); messages::RegisterRuntimeManagerResponse registerRuntimeManagerResponse; @@ -678,8 +743,8 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentFailedTest) manager_->Start(); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetRegisterRuntimeManagerRequest()->address().empty(); }); - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); - EXPECT_TRUE(("127.0.0.1:" + std::to_string(port)) == testActor_->GetRegisterRuntimeManagerRequest()->address()); + uint16_t lport = GetPortEnv("LITEBUS_PORT", 8080); + EXPECT_TRUE(("127.0.0.1:" + std::to_string(lport)) == testActor_->GetRegisterRuntimeManagerRequest()->address()); EXPECT_TRUE(runtimeManagerActorName_ == testActor_->GetRegisterRuntimeManagerRequest()->name()); EXPECT_TRUE(testActor_->GetRegisterRuntimeManagerRequest()->mutable_runtimeinstanceinfos()->size() == 0); @@ -703,12 +768,13 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentUnknownErrorTest) functionsystem::runtime_manager::Flags flags; std::string agentAddress = "--agent_address=" + testActor_->GetAID().Url(); + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); const char *argv[] = { "/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=2000", "--runtime_dir=/tmp", @@ -718,7 +784,7 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentUnknownErrorTest) "--proc_metrics_memory=2000", R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" }; - flags.ParseFlags(13, argv); + flags.ParseFlags(std::size(argv), argv); manager_->SetRegisterHelper(std::make_shared("node1-RuntimeManagerSrv")); manager_->SetConfig(flags); messages::RegisterRuntimeManagerResponse registerRuntimeManagerResponse; @@ -727,8 +793,8 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentUnknownErrorTest) manager_->Start(); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetRegisterRuntimeManagerRequest()->address().empty(); }); - uint16_t port = GetPortEnv("LITEBUS_PORT", 8080); - EXPECT_TRUE(("127.0.0.1:" + std::to_string(port)) == testActor_->GetRegisterRuntimeManagerRequest()->address()); + uint16_t lport = GetPortEnv("LITEBUS_PORT", 8080); + EXPECT_TRUE(("127.0.0.1:" + std::to_string(lport)) == testActor_->GetRegisterRuntimeManagerRequest()->address()); EXPECT_TRUE(runtimeManagerActorName_ == testActor_->GetRegisterRuntimeManagerRequest()->name()); EXPECT_TRUE(testActor_->GetRegisterRuntimeManagerRequest()->mutable_runtimeinstanceinfos()->size() == 0); @@ -741,12 +807,13 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentTimeoutTest) manager_->isUnitTestSituation_ = false; functionsystem::runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); const char *argv[] = { "/runtime_manager", "--node_id=node1", "--ip=127.0.0.1", "--host_ip=127.0.0.1", - "--port=32233", + port, "--runtime_initial_port=500", "--port_num=2000", "--runtime_dir=/tmp", @@ -756,7 +823,7 @@ TEST_F(RuntimeManagerTest, RegisterToFunctionAgentTimeoutTest) "--proc_metrics_memory=2000", R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" }; - flags.ParseFlags(13, argv); + flags.ParseFlags(std::size(argv), argv); manager_->SetRegisterHelper(std::make_shared("node1-RuntimeManagerSrv")); manager_->SetConfig(flags); manager_->SetRegisterInterval(5); @@ -788,8 +855,8 @@ TEST_F(RuntimeManagerTest, QueryInstanceStatusInfoTest) // lost connection with function agent manager_->connected_ = false; testActor_->QueryInstanceStatusInfo(manager_->GetAID(), request); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_EQ(testActor_->GetIsReceiveQueryInstanceStatusInfoResponse(), false); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveQueryInstanceStatusInfoResponse(); }); + manager_->connected_ = true; testActor_->QueryInstanceStatusInfo(manager_->GetAID(), request); ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetQueryInstanceStatusResponse()->requestid().empty(); }); @@ -811,18 +878,15 @@ TEST_F(RuntimeManagerTest, CleanStatusTest) litebus::Spawn(testActor_, true); testActor_->Send(manager_->GetAID(), "CleanStatus", "invalid msg&&"); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_FALSE(testActor_->GetIsReceiveCleanStatusResponse()); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveCleanStatusResponse(); }); messages::CleanStatusRequest cleanStatusRequest; cleanStatusRequest.set_name("invalid RuntimeManagerID"); testActor_->Send(manager_->GetAID(), "CleanStatus", cleanStatusRequest.SerializeAsString()); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_TRUE(testActor_->GetIsReceiveCleanStatusResponse()); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetIsReceiveCleanStatusResponse(); }); testActor_->ResetIsReceiveCleanStatusResponse(); cleanStatusRequest.set_name(manager_->runtimeManagerID_); testActor_->Send(manager_->GetAID(), "CleanStatus", cleanStatusRequest.SerializeAsString()); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - EXPECT_TRUE(testActor_->GetIsReceiveCleanStatusResponse()); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetIsReceiveCleanStatusResponse(); }); litebus::Terminate(testActor_->GetAID()); litebus::Await(testActor_->GetAID()); @@ -859,4 +923,307 @@ TEST_F(RuntimeManagerTest, UpdateTokenTest) litebus::Terminate(testActor_->GetAID()); litebus::Await(testActor_->GetAID()); } + +TEST_F(RuntimeManagerTest, CollectCpuType) +{ + manager_->CollectCpuType(); + + EXPECT_FALSE(manager_->GetCpuType().empty()); +} + +TEST_F(RuntimeManagerTest, GetCpuTypeByProc) +{ + EXPECT_FALSE(manager_->GetCpuTypeByProc().empty()); +} + +TEST_F(RuntimeManagerTest, GetCpuTypeByCommand) +{ + EXPECT_FALSE(manager_->GetCpuTypeByCommand().empty()); +} + +TEST_F(RuntimeManagerTest, EnableDebugInstanceIDTest_NotFound_Gdbserver) +{ + auto optionEnv = litebus::os::GetEnv("PATH"); + std::string env; + if (optionEnv.IsSome()) { + env = optionEnv.Get(); // origin env + } + + litebus::os::SetEnv("PATH", "/tmp/"); + functionsystem::runtime_manager::Flags flags; + const char *port = ("--port=" + std::to_string(FindAvailablePort())).c_str(); + const char *argv[] = { + "/runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + port, + "--runtime_initial_port=500", + "--port_num=2000", + "--runtime_dir=/tmp", + const_cast("127.0.0.1:80"), + "--runtime_ld_library_path=/tmp", + "--proc_metrics_cpu=2000", + "--proc_metrics_memory=2000", + "--runtime_instance_debug_enable=true", + R"(--log_config={"filepath": "/home/yr/log", "level": "DEBUG", "rolling": {"maxsize": 100, "maxfiles": 1},"alsologtostderr":true})" + }; + flags.ParseFlags(std::size(argv), argv); + manager_->SetRegisterHelper(std::make_shared("node1-RuntimeManagerSrv")); + manager_->SetConfig(flags); + + testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); + litebus::Spawn(testActor_, true); + + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("cpp"); + auto posixEnvs = runtimeConfig->mutable_posixenvs(); + posixEnvs->insert({ YR_DEBUG_CONFIG, R"({"enable": "true"})" }); + + testActor_->StartInstance(manager_->GetAID(), startRequest); + + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); + + auto response = testActor_->GetStartInstanceResponse(); + EXPECT_EQ(StatusCode::RUNTIME_MANAGER_DEBUG_SERVER_NOTFOUND, response->code()); + EXPECT_EQ("Debug server components for language cpp not found.", response->message()); + EXPECT_EQ("test_requestID", response->requestid()); + + auto instanceResponse = response->mutable_startruntimeinstanceresponse(); + auto resRuntimeID = instanceResponse->runtimeid(); + EXPECT_TRUE(resRuntimeID.empty()); + EXPECT_TRUE(manager_->runtimeInstanceDebugEnable_); + EXPECT_FALSE(manager_->debugServerMgr_->actor_->languageDebugConfigs_["cpp"].isFound); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStopInstanceResponse(); }); + + const std::string stopRequestID = "test_requestID"; + messages::StopInstanceRequest request; + request.set_runtimeid(resRuntimeID); + request.set_requestid(stopRequestID); + request.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + testActor_->StopInstance(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetStopInstanceResponse()->requestid() == stopRequestID; }); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->code(), + static_cast(StatusCode::RUNTIME_MANAGER_RUNTIME_PROCESS_NOT_FOUND)); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->message(), "stop instance failed"); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->runtimeid(), resRuntimeID); + + litebus::Terminate(testActor_->GetAID()); + litebus::Await(testActor_->GetAID()); + litebus::os::SetEnv("PATH", env); +} + +TEST_F(RuntimeManagerDebugServerTest, QueryDebugInstanceInfosTest) +{ + testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); + litebus::Spawn(testActor_, true); + + messages::QueryDebugInstanceInfosRequest request; + request.set_requestid("request_id"); + + // lost connection with function agent + manager_->connected_ = false; + testActor_->QueryDebugInstanceInfos(manager_->GetAID(), request); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_EQ(testActor_->isReceiveQueryDebugInstanceInfosResponse_, false); + + // resume connection + manager_->connected_ = true; + testActor_->QueryDebugInstanceInfos(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->isReceiveQueryDebugInstanceInfosResponse_; }); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetQueryDebugInstanceResponse()->requestid() == "request_id"; }); + EXPECT_EQ(testActor_->GetQueryDebugInstanceResponse()->debuginstanceinfos_size(), 0); + testActor_->ResetMessage(); + + // stub pid data + std::vector testPids; + pid_t pid = fork(); + std::string port; + if (pid == 0) { // child process + while (true) { + sleep(2); // keep alive + } + _exit(0); // Prevents child processes from executing code outside the loop + } else if (pid > 0) { // parent process + testPids.push_back(pid); + auto runtimeID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + manager_->debugServerMgr_->actor_->runtime2PID_[runtimeID] = pid; + manager_->debugServerMgr_->actor_->runtime2DebugServerPID_[runtimeID] = pid; + auto instanceID = litebus::uuid_generator::UUID::GetRandomUUID().ToString(); + manager_->debugServerMgr_->actor_->runtime2instanceID_[runtimeID] = instanceID; + + // stub port for debug server + port = PortManager::GetInstance().RequestPort(runtimeID); + YRLOG_DEBUG("port: {}", port); + manager_->debugServerMgr_->actor_->hostIP_ = "127.0.0.1"; + manager_->debugServerMgr_->actor_->pid2runtimeID_[pid] = runtimeID; + manager_->debugServerMgr_->actor_->runtime2DebugServerPort_[runtimeID] = port; + } + + // success + testActor_->QueryDebugInstanceInfos(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE( + [=]() -> bool { return testActor_->GetQueryDebugInstanceResponse()->requestid() == "request_id"; }); + ASSERT_EQ(testActor_->GetQueryDebugInstanceResponse()->debuginstanceinfos_size(), 1); + EXPECT_EQ(testActor_->GetQueryDebugInstanceResponse()->debuginstanceinfos(0).pid(), pid); + EXPECT_EQ(testActor_->GetQueryDebugInstanceResponse()->debuginstanceinfos(0).debugserver(), "127.0.0.1:" + port); + + litebus::Terminate(testActor_->GetAID()); + litebus::Await(testActor_->GetAID()); +} + +TEST_F(RuntimeManagerDebugServerTest, DisableDebugInstanceIDTest) +{ + testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); + litebus::Spawn(testActor_, true); + + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("cpp"); + auto posixEnvs = runtimeConfig->mutable_posixenvs(); + posixEnvs->insert({ YR_DEBUG_CONFIG, R"({"enable": "false"})" }); + + testActor_->StartInstance(manager_->GetAID(), startRequest); + + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); + + auto response = testActor_->GetStartInstanceResponse(); + EXPECT_EQ(StatusCode::SUCCESS, response->code()); + EXPECT_EQ("start instance success", response->message()); + EXPECT_EQ("test_requestID", response->requestid()); + + auto instanceResponse = response->mutable_startruntimeinstanceresponse(); + auto resRuntimeID = instanceResponse->runtimeid(); + EXPECT_TRUE(!resRuntimeID.empty()); + EXPECT_EQ(std::to_string(INITIAL_PORT), instanceResponse->port()); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStopInstanceResponse(); }); + + const std::string stopRequestID = "test_requestID"; + messages::StopInstanceRequest request; + request.set_runtimeid(resRuntimeID); + request.set_requestid(stopRequestID); + request.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + + testActor_->StopInstance(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetStopInstanceResponse()->requestid() == stopRequestID; }); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->code(), static_cast(StatusCode::SUCCESS)); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->message(), "stop instance success"); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->runtimeid(), resRuntimeID); + + litebus::Terminate(testActor_->GetAID()); + litebus::Await(testActor_->GetAID()); +} + +TEST_F(RuntimeManagerDebugServerTest, EnableDebugInstanceIDTest) +{ + testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); + litebus::Spawn(testActor_, true); + + + auto startRequest = GenStartInstanceRequest(); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + auto posixEnvs = runtimeConfig->mutable_posixenvs(); + posixEnvs->insert({ YR_DEBUG_CONFIG, R"({"enable": "true"})" }); + testActor_->StartInstance(manager_->GetAID(), startRequest); + + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); + + auto response = testActor_->GetStartInstanceResponse(); + EXPECT_EQ(StatusCode::SUCCESS, response->code()); + EXPECT_EQ("start instance success", response->message()); + EXPECT_EQ("test_requestID", response->requestid()); + + auto instanceResponse = response->mutable_startruntimeinstanceresponse(); + auto resRuntimeID = instanceResponse->runtimeid(); + EXPECT_TRUE(!resRuntimeID.empty()); + EXPECT_EQ(std::to_string(INITIAL_PORT + 1), instanceResponse->port()); + EXPECT_TRUE(manager_->runtimeInstanceDebugEnable_); + EXPECT_TRUE(manager_->debugServerMgr_->actor_->languageDebugConfigs_["cpp"].isFound); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStopInstanceResponse(); }); + + const std::string stopRequestID = "test_requestID"; + messages::StopInstanceRequest request; + request.set_runtimeid(resRuntimeID); + request.set_requestid(stopRequestID); + request.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + + testActor_->StopInstance(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetStopInstanceResponse()->requestid() == stopRequestID; }); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->code(), static_cast(StatusCode::SUCCESS)); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->message(), "stop instance success"); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->runtimeid(), resRuntimeID); + + litebus::Terminate(testActor_->GetAID()); + litebus::Await(testActor_->GetAID()); +} + +TEST_F(RuntimeManagerDebugServerTest, EnablePythonDebugInstanceIDTest) +{ + testActor_ = std::make_shared(GenerateRandomName("RuntimeManagerTestActor")); + litebus::Spawn(testActor_, true); + + messages::StartInstanceRequest startRequest; + startRequest.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + auto runtimeInfo = startRequest.mutable_runtimeinstanceinfo(); + runtimeInfo->set_requestid("test_requestID"); + runtimeInfo->set_instanceid("test_instanceID"); + runtimeInfo->set_traceid("test_traceID"); + + auto runtimeConfig = runtimeInfo->mutable_runtimeconfig(); + runtimeConfig->set_language("python3"); + auto posixEnvs = runtimeConfig->mutable_posixenvs(); + posixEnvs->insert({ YR_DEBUG_CONFIG, R"({"enable": "true"})" }); + auto deployConfig = runtimeInfo->mutable_deploymentconfig(); + deployConfig->set_objectid("test_objectID"); + deployConfig->set_bucketid("test_bucketID"); + deployConfig->set_deploydir(testDeployDir); + deployConfig->set_storagetype("local"); + + testActor_->StartInstance(manager_->GetAID(), startRequest); + + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetStartInstanceResponse()->message().empty(); }); + + auto response = testActor_->GetStartInstanceResponse(); + EXPECT_EQ(StatusCode::SUCCESS, response->code()); + EXPECT_EQ("start instance success", response->message()); + EXPECT_EQ("test_requestID", response->requestid()); + + auto instanceResponse = response->mutable_startruntimeinstanceresponse(); + auto resRuntimeID = instanceResponse->runtimeid(); + EXPECT_TRUE(!resRuntimeID.empty()); + EXPECT_EQ(std::to_string(INITIAL_PORT + 1), instanceResponse->port()); + EXPECT_TRUE(manager_->runtimeInstanceDebugEnable_); + EXPECT_TRUE(manager_->debugServerMgr_->actor_->languageDebugConfigs_["python"].isFound); + ASSERT_AWAIT_TRUE([=]() -> bool { return !testActor_->GetIsReceiveStopInstanceResponse(); }); + + const std::string stopRequestID = "test_requestID"; + messages::StopInstanceRequest request; + request.set_runtimeid(resRuntimeID); + request.set_requestid(stopRequestID); + request.set_type(static_cast(EXECUTOR_TYPE::RUNTIME)); + + testActor_->StopInstance(manager_->GetAID(), request); + ASSERT_AWAIT_TRUE([=]() -> bool { return testActor_->GetStopInstanceResponse()->requestid() == stopRequestID; }); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->code(), static_cast(StatusCode::SUCCESS)); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->message(), "stop instance success"); + EXPECT_EQ(testActor_->GetStopInstanceResponse()->runtimeid(), resRuntimeID); + + litebus::Terminate(testActor_->GetAID()); + litebus::Await(testActor_->GetAID()); +} + } // namespace functionsystem::runtime_manager \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.cpp b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.cpp index ae9f5be31cd6cfea0ac0d6f8c0268cac1b3e9cc2..be67a738667efb2cf5d5fb4f11d9253f53d33d8d 100644 --- a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.cpp +++ b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.cpp @@ -17,8 +17,8 @@ #include "runtime_manager_test_actor.h" #include "common/constants/actor_name.h" -#include "heartbeat/ping_pong_driver.h" -#include "logs/logging.h" +#include "common/heartbeat/heartbeat_client.h" +#include "common/logs/logging.h" #include "utils/port_helper.h" namespace functionsystem::runtime_manager { @@ -107,13 +107,6 @@ uint32_t RuntimeManagerTestActor::GetReceiveTimes() { return receiveTimes_; } -void RuntimeManagerTestActor::SendPingOnce(const litebus::AID &to) -{ - uint16_t port = functionsystem::test::GetPortEnv("LITEBUS_PORT", 8080); - litebus::AID dst(RUNTIME_MANAGER_PINGPONG_ACTOR_NAME + PINGPONG_BASENAME, "127.0.0.1:" + std::to_string(port)); - dst.SetProtocol(litebus::BUS_UDP); - Send(dst, "Ping", ""); -} void RuntimeManagerTestActor::QueryInstanceStatusInfo(const litebus::AID &to, const messages::QueryInstanceStatusRequest &request) { diff --git a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.h b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.h index 76b930055a1f4ad7871333207efaab3119e806a0..c8a66b054f5a2d84341f593c5c038f93539af17e 100644 --- a/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.h +++ b/functionsystem/tests/unit/runtime_manager/manager/runtime_manager_test_actor.h @@ -18,7 +18,7 @@ #define RUNTIME_MANAGER_MANAGER_RUNTIME_MANAGER_TEST_ACTOR_H #include "actor/actor.hpp" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::runtime_manager { class RuntimeManagerTestActor : public litebus::ActorBase { public: @@ -40,8 +40,6 @@ public: void Register(const litebus::AID &from, std::string &&, std::string &&msg); - void SendPingOnce(const litebus::AID &to); - void QueryInstanceStatusInfo(const litebus::AID &to, const messages::QueryInstanceStatusRequest &request); void QueryInstanceStatusInfoResponse(const litebus::AID &from, std::string &&, std::string &&msg); diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/disk_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/disk_collector_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..72e5a815768973d3c5ede1f31e3e4141d61ae73d --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/disk_collector_test.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "runtime_manager/metrics/collector/disk_collector.h" + +namespace functionsystem::test { + +using namespace testing; +using namespace functionsystem::runtime_manager; + +struct ExpectedDisk { + std::string name; + int size; + std::string mountpoints; +}; + +class DiskCollectorTest : public ::testing::Test {}; + +void ValidateDiskExtensions(const DevClusterMetrics &devMetrics, const std::vector &expectedDisks) +{ + + EXPECT_EQ(devMetrics.extensionInfo.size(), expectedDisks.size()); + EXPECT_EQ(devMetrics.intsInfo.count(resource_view::DISK_RESOURCE_NAME) > 0, true); + EXPECT_EQ(devMetrics.intsInfo.at(resource_view::DISK_RESOURCE_NAME).size(), expectedDisks.size()); + + for (size_t i = 0; i < expectedDisks.size(); ++i) { + const auto& diskExt = devMetrics.extensionInfo[i].disk(); + const auto& expected = expectedDisks[i]; + + EXPECT_EQ(diskExt.name(), expected.name); + EXPECT_EQ(diskExt.size(), expected.size); + EXPECT_EQ(diskExt.mountpoints(), expected.mountpoints); + + if (devMetrics.intsInfo.count(resource_view::DISK_RESOURCE_NAME) > 0) { + EXPECT_EQ(diskExt.size(), devMetrics.intsInfo.at(resource_view::DISK_RESOURCE_NAME)[i]); + } + } +} + +/* +Test for parsing valid disk configurations from JSON input +1. JSON contains multiple disks with valid name, size and mount points -> should successfully parse all disks +2. Each parsed disk should have correct metrics including size conversion from "G" to numeric value +3. Mount paths should be accurately captured for each disk device +*/ +TEST_F(DiskCollectorTest, ParseValidDisks) { + std::string json = R"( + [{"name": "disk1", "size": "100G", "mountPoints": "/tmp/disk1/"}, + {"name": "disk2", "size": "200G", "mountPoints": "/tmp/disk2a/"}] + )"; + + auto collector = std::make_shared(json); + auto usage = collector->GetUsage().Get(); + + auto devMetrics = usage.devClusterMetrics.Get(); + std::vector expectedDisks = { + {"disk1", 100, "/tmp/disk1/"}, + {"disk2", 200, "/tmp/disk2a/"} + }; + ValidateDiskExtensions(devMetrics, expectedDisks); +} + +/* + * Test for disk configs with mixed valid and invalid entries + * 1. JSON array contains both properly formatted disk and malformed config + * 2. Valid disk config should be parsed normally while invalid entry is silently skipped + */ +TEST_F(DiskCollectorTest, ParseMixedConfigWithPartialValidity) { + std::string json = R"( + [{"name": "invalid_disk1", "mountPoints": "/mnt/"}, + {"name": "invalid_disk2", "size": "100G", "mountPoints": "/mnt——a/"}, + {"name": "disk1", "size": "100G", "mountPoints": "/tmp/disk1/a_b-c.d/"}] + )"; + + auto collector = std::make_shared(json); + auto usage = collector->GetUsage().Get(); + + auto devMetrics = usage.devClusterMetrics.Get(); + std::vector expectedDisks = { + {"disk1", 100, "/tmp/disk1/a_b-c.d/"} + }; + ValidateDiskExtensions(devMetrics, expectedDisks); +} + +/* + * Test for disk config validation with missing mandatory fields + * JSON disk config missing "size" field -> validation fails + */ +TEST_F(DiskCollectorTest, RejectConfigWithMissingSizeField) { + std::string json = R"( + [{"name": "invalid_disk", "mountPoints": "/mnt/"}] + )"; + + DiskCollector collector(json); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for disk size format validation + * Invalid size value -> rejected by regex check + */ +TEST_F(DiskCollectorTest, RejectInvalidSizeFormat) { + std::string json = R"( + [{"name": "disk3", "size": "10K", "mountPoints": "/mnt/disk3/"}, + {"name": "disk3", "size": "10GB", "mountPoints": "/mnt/disk3/"}, + {"name": "disk3", "size": "10g", "mountPoints": "/mnt/disk3/"}, + {"name": "disk3", "size": "1_00g", "mountPoints": "/mnt/disk3/"},] + )"; + + DiskCollector collector(json); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for detecting path traversal attacks + * Mount path containing '..' sequence -> blocked by security check + */ +TEST_F(DiskCollectorTest, BlockPathTraversalInMountPoint) { + std::string json = R"( + [{"name": "unsafe_disk", "size": "50G", "mountPoints": "/var/../unsafe/"}] + )"; + + DiskCollector collector(json); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for absolute path format enforcement + * Mount point missing trailing slash -> violates path regex + */ +TEST_F(DiskCollectorTest, RequireTrailingSlashInPath) { + std::string json = R"( + [{"name": "bad_path", "size": "30G", "mountPoints": "/tmp"}] + )"; + + DiskCollector collector(json); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for special character filtering in paths + * Path contains invalid characters '()' -> rejected by regex + */ +TEST_F(DiskCollectorTest, FilterSpecialCharsInMountPoint) { + std::string json = R"( + [{"name": "weird_path", "size": "40G", "mountPoints": "/tmp/()}"] + )"; + + DiskCollector collector(json); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for empty disk configuration JSON input handling + * Provide empty JSON array as disk config -> config validation fails + */ +TEST_F(DiskCollectorTest, HandleEmptyJSONConfig) { + DiskCollector collector("[]"); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +/* + * Test for JSON structure validation at root level + * Provide JSON object instead of array as root element -> config validation fails + */ +TEST_F(DiskCollectorTest, RejectNonArrayJSONInput) { + DiskCollector collector(R"({"name": "invalid_root"})"); + EXPECT_FALSE(collector.GetUsage().Get().devClusterMetrics.IsSome()); +} + +} diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_cpu_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_cpu_collector_test.cpp index d95987e1776b66f1a2b0877f9ab5e8c259a6f1fb..65b441c06889a2f01a0a4ac928e85985ceead408 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_cpu_collector_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_cpu_collector_test.cpp @@ -69,10 +69,111 @@ TEST_F(InstanceCPUCollectorTest, GetLimit) TEST_F(InstanceCPUCollectorTest, GetUsage) { auto tools = std::make_shared(); + EXPECT_CALL(*tools.get(), Read("/proc/stat")) + .WillOnce(testing::Return(litebus::Option{ + R"(cpu 258231303 589 11088715 8195531331 15244 2754827 848459 0 0 0 +cpu0 11923702 41 1087644 515738274 2847 210938 279500 0 0 0 +cpu1 20057109 15 291565 508745688 604 141915 104390 0 0 0 +cpu2 6874749 48 1122999 520663754 1549 199736 75274 0 0 0 +cpu3 24848059 22 188129 504152845 194 147783 12873 0 0 0 +cpu4 18860652 37 1106445 509009860 982 229940 43436 0 0 0 +cpu5 13826623 14 284263 515088686 176 114049 28281 0 0 0 +cpu6 19189567 40 1127859 508655781 1034 232963 43336 0 0 0 +cpu7 13439629 18 266736 515512787 199 110400 15170 0 0 0 +cpu8 18788454 72 1130358 509053729 1080 232841 43926 0 0 0 +cpu9 13702977 11 264410 515252647 182 111011 14882 0 0 0 +cpu10 18793791 72 1149681 509029680 844 232674 44180 0 0 0 +cpu11 13555453 8 248708 515425993 144 108081 12689 0 0 0 +cpu12 18626769 78 1175226 509185741 893 233971 43553 0 0 0 +cpu13 13681432 11 240006 515307595 137 107740 12482 0 0 0 +cpu14 19112432 62 1147838 508693565 938 234448 56686 0 0 0 +cpu15 12949897 33 256839 516014697 3433 106331 17795 0 0 0 +intr 7392494251 0 9 0 0 0 0 0 0 0 0 529358 29 15 0 0 0 0 0 0 0 0 0 0 0 0 5659358 0 22 0 55474 0 9004793 4035290 8719258 4140521 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 5334914153 +btime 1755739399 +processes 64962782 +procs_running 5 +procs_blocked 0 +softirq 2896746553 1 202731347 1 59350651 5590608 0 135535 1076900957 1 1552037452)" + })).WillOnce(testing::Return(litebus::Option{ + R"(cpu 258231644 589 11088718 8195532805 15244 2754830 848460 0 0 0 +cpu0 11923703 41 1087645 515738386 2847 210938 279500 0 0 0 +cpu1 20057222 15 291565 508745688 604 141915 104390 0 0 0 +cpu2 6874750 48 1123000 520663867 1549 199736 75274 0 0 0 +cpu3 24848059 22 188129 504152959 194 147783 12873 0 0 0 +cpu4 18860653 37 1106445 509009974 982 229940 43436 0 0 0 +cpu5 13826650 14 284263 515088772 176 114049 28281 0 0 0 +cpu6 19189568 40 1127859 508655894 1034 232963 43336 0 0 0 +cpu7 13439630 18 266737 515512900 199 110400 15170 0 0 0 +cpu8 18788455 72 1130359 509053843 1080 232841 43926 0 0 0 +cpu9 13703016 11 264410 515252721 182 111011 14882 0 0 0 +cpu10 18793791 72 1149681 509029792 844 232674 44180 0 0 0 +cpu11 13555453 8 248708 515426107 144 108081 12689 0 0 0 +cpu12 18626882 78 1175226 509185741 893 233971 43553 0 0 0 +cpu13 13681432 11 240006 515307709 137 107740 12482 0 0 0 +cpu14 19112477 62 1147839 508693634 938 234448 56686 0 0 0 +cpu15 12949897 33 256839 516014810 3433 106331 17795 0 0 0 +intr 7392500979 0 9 0 0 0 0 0 0 0 0 529359 29 15 0 0 0 0 0 0 0 0 0 0 0 0 5659358 0 22 0 55474 0 9004793 4035290 8719267 4140525 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 5334917357 +btime 1755739399 +processes 64962801 +procs_running 3 +procs_blocked 0 +softirq 2896749124 1 202731472 1 59350683 5590608 0 135535 1076902277 1 1552038546)" + })); - auto collector = std::make_shared(1, "id", 1000.0, "urn"); + EXPECT_CALL(*tools.get(), Read("/proc/1/stat")) + .WillOnce(testing::Return(litebus::Option{ + R"(1 (monitor) S 3142022 3142022 2132682 34817 2132682 4194368 483 0 0 0 16072344 16340 0 0 20 0 2 0 470263721 109170688 2385 18446744073709551615 94672056987648 94672056987989 140733336937424 0 0 0 0 16781313 2 0 0 0 17 4 0 0 0 0 0 94672056999304 94672056999952 94672087425024 140733336942591 140733336942615 140733336942615 140733336944615 0)" + })) + .WillOnce(testing::Return(litebus::Option{ + R"(1 (monitor) S 3142022 3142022 2132682 34817 2132682 4194368 483 0 0 0 16072345 16340 0 0 20 0 2 0 470263721 109170688 2385 18446744073709551615 94672056987648 94672056987989 140733336937424 0 0 0 0 16781313 2 0 0 0 17 4 0 0 0 0 0 94672056999304 94672056999952 94672087425024 140733336942591 140733336942615 140733336942615 140733336944615 0)" + })); + + auto collector = std::make_shared(1, "id", 1000.0, "urn", tools); + auto usage = collector->GetUsage().Get(); + YRLOG_DEBUG("usage.value is {}", usage.value.Get()); + EXPECT_NEAR(usage.value.Get(), 0.05, 0.09); + EXPECT_EQ(usage.instanceID.Get(), "id"); +} + +/** + * Feature: InstanceCPUCollector + * Description: Get Usage + * Steps: read file is empty + * Expectation: + * {} + */ +#if 0 +TEST_F(InstanceCPUCollectorTest, GetUsageWithEmptyContent) +{ + auto tools = std::make_shared(); + EXPECT_CALL(*tools.get(), Read) + .WillOnce(::testing::Return(litebus::Option{})); + + auto collector = std::make_shared(1, "id", 1000.0, "urn", tools); + auto usage = collector->GetUsage().Get(); + EXPECT_EQ(usage.value.IsNone(), true); + EXPECT_EQ(usage.instanceID.Get(), "id"); +} + +/** + * Feature: InstanceCPUCollector + * Description: Get Usage + * Steps: read file content is not correct + * Expectation: + * {} + */ +TEST_F(InstanceCPUCollectorTest, GetUsageWithInvaldContent) +{ + auto tools = std::make_shared(); + EXPECT_CALL(*tools.get(), Read) + .WillOnce(::testing::Return(litebus::Option{"1"})); + + auto collector = std::make_shared(1, "id", 1000.0, "urn", tools); auto usage = collector->GetUsage().Get(); - EXPECT_EQ(usage.value.Get(), 0); + EXPECT_EQ(usage.value.IsNone(), true); EXPECT_EQ(usage.instanceID.Get(), "id"); } +#endif } \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_xpu_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_xpu_collector_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8bde1c0fef6c83ede4d224446cffd6f65a986b97 --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/instance_xpu_collector_test.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include + +#include "mocks/mock_cmdtool.h" +#include "common/utils/files.h" +#include "utils/future_test_helper.h" +#include "runtime_manager/metrics/collector/heterogeneous_collector/gpu_probe.h" +#include "runtime_manager/metrics/collector/heterogeneous_collector/npu_probe.h" +#include "runtime_manager/metrics/collector/heterogeneous_collector/topo_info.h" + +#define private public +#include "runtime_manager/metrics/collector/instance_xpu_collector.h" + +using ::testing::MatchesRegex; + +namespace functionsystem::test { +class MockProcFSTools : public ProcFSTools { +public: + MOCK_METHOD(litebus::Option, Read, (const std::string &path), (override)); +}; + +// npu-smi info 910B +const std::string npuSmiInfo910B = R"( ++------------------------------------------------------------------------------------------------+ +| npu-smi 25.2.0 Version: 25.2.0 | ++---------------------------+---------------+----------------------------------------------------+ +| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page)| +| Chip | Bus-Id | AICore(%) Memory-Usage(MB) HBM-Usage(MB) | ++===========================+===============+====================================================+ +| 0 910B4 | OK | 84.7 38 0 / 0 | +| 0 | 0000:C1:00.0 | 0 0 / 0 2894 / 32768 | ++===========================+===============+====================================================+ +| 1 910B4 | OK | 88.9 39 0 / 0 | +| 0 | 0000:01:00.0 | 0 0 / 0 2653 / 32768 | ++===========================+===============+====================================================+ +| 2 910B4 | OK | 85.3 38 0 / 0 | +| 0 | 0000:C2:00.0 | 0 0 / 0 2836 / 32768 | ++===========================+===============+====================================================+ +| 3 910B4 | OK | 88.7 40 0 / 0 | +| 0 | 0000:02:00.0 | 0 0 / 0 2842 / 32768 | ++===========================+===============+====================================================+ +| 4 910B4 | OK | 86.0 39 0 / 0 | +| 0 | 0000:81:00.0 | 0 0 / 0 2653 / 32768 | ++===========================+===============+====================================================+ +| 5 910B4 | OK | 90.0 40 0 / 0 | +| 0 | 0000:41:00.0 | 0 0 / 0 2836 / 32768 | ++===========================+===============+====================================================+ +| 6 910B4 | OK | 88.6 39 0 / 0 | +| 0 | 0000:82:00.0 | 0 0 / 0 2659 / 32768 | ++===========================+===============+====================================================+ +| 7 910B4 | OK | 84.9 40 0 / 0 | +| 0 | 0000:42:00.0 | 0 0 / 0 2659 / 32768 | ++===========================+===============+====================================================+ ++---------------------------+---------------+----------------------------------------------------+ +| NPU Chip | Process id | Process name | Process memory(MB) | ++===========================+===============+====================================================+ +| 0 0 | 472206 | python3.9 | 113 | ++===========================+===============+====================================================+ +| No running processes found in NPU 1 | ++===========================+===============+====================================================+ +| No running processes found in NPU 2 | ++===========================+===============+====================================================+ +| No running processes found in NPU 3 | ++===========================+===============+====================================================+ +| No running processes found in NPU 4 | ++===========================+===============+====================================================+ +| No running processes found in NPU 5 | ++===========================+===============+====================================================+ +| No running processes found in NPU 6 | ++===========================+===============+====================================================+ +| No running processes found in NPU 7 | ++===========================+===============+====================================================+ +)"; + +std::vector npuinfoStrToVector(const std::string &input) +{ + std::vector vec; + std::istringstream stream(input); + std::string line; + while (std::getline(stream, line)) { + vec.push_back(line); + } + return vec; +} + +class InstanceXpuCollectorTest : public ::testing::Test {}; + +TEST_F(InstanceXpuCollectorTest, InstanceNPUCollectorGetUsage) +{ + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); + auto params = std::make_shared(); + params->ldLibraryPath = ""; + params->deviceInfoPath = "/home/sn/config/topology-info.json"; + params->collectMode = runtime_manager::NPU_COLLECT_HBM; + auto probe = std::make_shared("co200", tool, cmdTool, params); + EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info")).WillRepeatedly(testing::Return(npuinfoStrToVector(npuSmiInfo910B))); + EXPECT_CALL(*cmdTool.get(), GetCmdResult("pip3 list")).WillRepeatedly(testing::Return(npuinfoStrToVector(""))); + + auto deployDir = "/home/sn/function/package/xxxz"; + auto startReq = std::make_shared(); + startReq->mutable_runtimeinstanceinfo()->mutable_runtimeconfig()->mutable_userenvs()->insert({"func-NPU-DEVICE-IDS", "0"}); + runtime_manager::InstInfoWithXPU info = {472206, "instanceID1", deployDir, startReq->runtimeinstanceinfo(), "co200"}; + + auto instNpuCollector = + std::make_shared(info, runtime_manager::metrics_type::NPU, tool, params); + + instNpuCollector->probe_ = probe; + ASSERT_EQ(instNpuCollector->logicIDs_.size(), 1); + litebus::Future future = instNpuCollector->GetUsage(); + runtime_manager::Metric metric = future.Get(); + ASSERT_EQ(metric.value, 8.0); + runtime_manager::DevClusterMetrics devClusterMetrics = metric.devClusterMetrics.Get(); + + std::vector expectUseIDs = std::vector{ 0 }; + std::vector expectRealIDs = std::vector{ 0, 1, 2, 3, 4, 5, 6, 7 }; + std::vector expectUseHBM = std::vector{ 2894, 2653, 2836, 2842, 2653, 2836, 2659, 2659 }; + + auto usedIDs = devClusterMetrics.intsInfo.at(resource_view::USED_IDS_KEY); + auto ids = devClusterMetrics.intsInfo.at(resource_view::IDS_KEY); + auto usedHBM = devClusterMetrics.intsInfo.at(runtime_manager::dev_metrics_type::USED_HBM_KEY); + + for (size_t i = 0; i < expectRealIDs.size(); i++) { + ASSERT_EQ(expectRealIDs[i], ids[i]); + ASSERT_EQ(expectUseHBM[i], usedHBM[i]); + } + ASSERT_EQ(expectUseIDs[0], usedIDs[0]); +} + +} diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/node_cpu_utilization_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_cpu_utilization_collector_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1e7b9b40e945a4548d1ac2cad388808aa879c727 --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_cpu_utilization_collector_test.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "runtime_manager/metrics/collector/node_cpu_utilization_collector.h" + +namespace functionsystem::test { +class MockProcFSTools : public ProcFSTools { +public: + MOCK_METHOD(litebus::Option, Read, (const std::string &path), (override)); +}; + +class NodeCPUUtilizationCollectorTest : public ::testing::Test {}; + + +TEST_F(NodeCPUUtilizationCollectorTest, GetUsage) +{ + auto tools = std::make_shared(); + EXPECT_CALL(*tools.get(), Read) + .WillOnce(testing::Return(litebus::Option{ + R"(cpu 75516619 39 2153191 297728831 6311 1721359 360091 0 0 0 +cpu0 3384272 0 139385 19797097 846 121505 124819 0 0 0 +cpu1 5836240 0 66362 17585075 313 81257 41791 0 0 0 +cpu2 7357656 1 137150 15894497 332 150990 17509 0 0 0 +cpu3 1889326 10 137052 21499020 355 67661 17826 0 0 0 +cpu4 5454478 0 165505 17792773 502 141432 16498 0 0 0 +cpu5 4092877 0 106571 19330296 253 78039 11926 0 0 0 +cpu6 5298449 11 170382 17943279 389 142069 15950 0 0 0 +cpu7 4249886 11 105856 19175645 210 77997 9827 0 0 0 +cpu8 5217513 0 171257 18024660 327 141772 15927 0 0 0 +cpu9 4332041 0 106459 19092948 211 78630 9521 0 0 0 +cpu10 5223854 0 172906 18016599 431 140731 15875 0 0 0 +cpu11 4275180 0 107497 19149022 195 78247 9279 0 0 0 +cpu12 5060223 0 176147 18179345 345 138770 15940 0 0 0 +cpu13 4445408 0 104966 18981743 145 77514 9138 0 0 0 +cpu14 5260870 0 174289 17980399 338 134536 17333 0 0 0 +cpu15 4138338 0 111400 19286425 1111 70202 10926 0 0 0 +intr 1290728296 0 9 0 0 0 0 0 0 0 0 23784 31 15 0 0 0 0 0 0 0 0 0 0 0 0 306095 0 1 0 1908 0 744513 264294 534304 255984 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 578202814 +btime 1756458064 +processes 2910938 +procs_running 1 +procs_blocked 0 +softirq 453474464 2 21784575 1 4296442 291444 0 6370 180610881 0 246484749)" + })).WillOnce( + testing::Return(litebus::Option{ + R"(cpu 75518250 39 2153238 297731563 6311 1721402 360098 0 0 0 +cpu0 3384365 0 139387 19797282 846 121506 124820 0 0 0 +cpu1 5836387 0 66363 17585198 313 81258 41792 0 0 0 +cpu2 7357686 1 137153 15894746 332 150991 17510 0 0 0 +cpu3 1889450 10 137055 21499160 355 67665 17827 0 0 0 +cpu4 5454640 0 165506 17792878 502 141442 16498 0 0 0 +cpu5 4092884 0 106579 19330557 253 78040 11926 0 0 0 +cpu6 5298599 11 170383 17943400 389 142073 15950 0 0 0 +cpu7 4250027 11 105862 19175778 210 77998 9827 0 0 0 +cpu8 5217689 0 171258 18024765 327 141773 15927 0 0 0 +cpu9 4332054 0 106460 19093212 211 78632 9521 0 0 0 +cpu10 5224003 0 172909 18016718 431 140735 15876 0 0 0 +cpu11 4275193 0 107501 19149285 195 78248 9279 0 0 0 +cpu12 5060335 0 176150 18179508 345 138771 15940 0 0 0 +cpu13 4445541 0 104967 18981888 145 77517 9139 0 0 0 +cpu14 5261024 0 174292 17980517 338 134542 17334 0 0 0 +cpu15 4138365 0 111405 19286664 1111 70203 10926 0 0 0 +intr 1290752326 0 9 0 0 0 0 0 0 0 0 23784 31 15 0 0 0 0 0 0 0 0 0 0 0 0 306099 0 1 0 1908 0 744582 264388 534349 256064 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 578209657 +btime 1756458064 +processes 2910988 +procs_running 9 +procs_blocked 0 +softirq 453483073 2 21784959 1 4296888 291448 0 6383 180613657 0 246489735)"})); + + auto collector = std::make_shared(tools); + auto metric = collector->GetUsage().Get(); + YRLOG_INFO("result is {}", metric.value.Get()); + EXPECT_NEAR(metric.value.Get(), 38.74, 0.9); // Solving the precision issue in floating-point number comparison +} +} + diff --git a/functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.h b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_disk_collector_test.cpp similarity index 48% rename from functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.h rename to functionsystem/tests/unit/runtime_manager/metrics/collector/node_disk_collector_test.cpp index a085c21b0b7ebe294314c31b69f4323faaa25734..db2add399655d92681eddc5f9879d106e6a6249a 100644 --- a/functionsystem/tests/unit/common/heartbeat/ChildHeartbeatObserver.h +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_disk_collector_test.cpp @@ -14,23 +14,27 @@ * limitations under the License. */ -#ifndef YUANRONG_KERNEL_CHILDHEARTBEATOBSERVER_H -#define YUANRONG_KERNEL_CHILDHEARTBEATOBSERVER_H +#include +#include +#include -#endif // YUANRONG_KERNEL_CHILDHEARTBEATOBSERVER_H +#include "runtime_manager/metrics/collector/node_disk_collector.h" -#include "heartbeat/heartbeat_observer.h" -namespace functionsystem { -class ChildHeartbeatObserver : public HeartbeatObserver { +namespace functionsystem::test { +class MockProcFSTools : public ProcFSTools { public: - [[maybe_unused]] explicit ChildHeartbeatObserver(const std::string &name, const litebus::AID &dst, - const HeartbeatObserver::TimeOutHandler &handler); + MOCK_METHOD(litebus::Option, Read, (const std::string &path), (override)); +}; - void Exited(const litebus::AID &actor) override - { - HeartbeatObserver::Exited(actor); - } +class NodeDiskCollectorTest : public ::testing::Test {}; - ~ChildHeartbeatObserver() override = default; -}; -} \ No newline at end of file +TEST_F(NodeDiskCollectorTest, GetUsage) +{ + auto tools = std::make_shared(); + + auto collector = std::make_shared(tools); + + auto usage = collector->GetUsage(); + EXPECT_GT(usage.Get().value.Get(), 0); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/node_memory_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_memory_collector_test.cpp index 494e80891643e22a8e422bfe505397019790ee4e..0fc7cde19c8d5c3dab069813689bba71eaac1aef 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/collector/node_memory_collector_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/node_memory_collector_test.cpp @@ -47,7 +47,7 @@ TEST_F(NodeMemoryCollectorTest, GenFilter) * Steps: * Expectation: */ -TEST_F(NodeMemoryCollectorTest, GetLimit) +TEST_F(NodeMemoryCollectorTest, GetLimitAndGetUsage) { auto tools = std::make_shared(); EXPECT_CALL(*tools.get(), Read) @@ -107,6 +107,8 @@ DirectMap1G: 84934656 kB)" EXPECT_LT(limit.value.Get(), 190042.3); EXPECT_GT(limit.value.Get(), 190042.2); EXPECT_EQ(limit.instanceID.IsNone(), true); + auto usage = collector->GetUsage(); + EXPECT_LT(usage.Get().value.Get(), 61020.9); + EXPECT_GT(usage.Get().value.Get(), 61020.1); } - } \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/metrics/collector/system_xpu_collector_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/collector/system_xpu_collector_test.cpp index 366be2d162b97df561af98fea34a2b030f645ca9..1d6fbb44b530596bbb270e928b51e0a3a627f545 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/collector/system_xpu_collector_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/metrics/collector/system_xpu_collector_test.cpp @@ -21,7 +21,7 @@ #include #include "mocks/mock_cmdtool.h" -#include "files.h" +#include "common/utils/files.h" #include "utils/future_test_helper.h" #include "runtime_manager/metrics/collector/heterogeneous_collector/gpu_probe.h" #include "runtime_manager/metrics/collector/heterogeneous_collector/npu_probe.h" @@ -514,21 +514,71 @@ const std::string npuSmiInfo910C = R"( +===========================+===============+====================================================+ )"; +const std::string npuSmiInfo310P3 = R"( ++--------------------------------------------------------------------------------------------------------+ +| npu-smi 24.1.0.1 Version: 24.1.0.1 | ++-------------------------------+-----------------+------------------------------------------------------+ +| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | +| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | ++===============================+=================+======================================================+ +| 1536 310P3 | OK | NA 51 0 / 0 | +| 0 0 | 0000:06:00.0 | 0 1596 / 44280 | ++-------------------------------+-----------------+------------------------------------------------------+ +| 1536 310P3 | OK | NA 50 0 / 0 | +| 1 1 | 0000:06:00.0 | 0 1325 / 43693 | ++===============================+=================+======================================================+ +| 1792 310P3 | OK | NA 51 0 / 0 | +| 0 2 | 0000:07:00.0 | 0 1452 / 44280 | ++-------------------------------+-----------------+------------------------------------------------------+ +| 1792 310P3 | OK | NA 51 0 / 0 | +| 1 3 | 0000:07:00.0 | 0 1444 / 43693 | ++===============================+=================+======================================================+ +| 2048 310P3 | OK | NA 50 0 / 0 | +| 0 4 | 0000:08:00.0 | 0 1353 / 44280 | ++-------------------------------+-----------------+------------------------------------------------------+ +| 2048 310P3 | OK | NA 49 0 / 0 | +| 1 5 | 0000:08:00.0 | 0 1529 / 43693 | ++===============================+=================+======================================================+ +| 2304 310P3 | OK | NA 52 0 / 0 | +| 0 6 | 0000:09:00.0 | 0 1656 / 44280 | ++-------------------------------+-----------------+------------------------------------------------------+ +| 2304 310P3 | OK | NA 50 0 / 0 | +| 1 7 | 0000:09:00.0 | 0 1278 / 43693 | ++===============================+=================+======================================================+ ++-------------------------------+-----------------+------------------------------------------------------+ +| NPU Chip | Process id | Process name | Process memory(MB) | ++===============================+=================+======================================================+ +| No running processes found in NPU 1536 | ++===============================+=================+======================================================+ +| No running processes found in NPU 1792 | ++===============================+=================+======================================================+ +| No running processes found in NPU 2048 | ++===============================+=================+======================================================+ +| No running processes found in NPU 2304 | ++===============================+=================+======================================================+ +)"; + // npu-smi info const std::string wrongNpuSmiInfo1 = R"( +===========================+===============+====================================================+ | 0 910B4 | OK | 85.0 36 0 / 0 | +| 0 | 0000:C1:00.0 | 0 0 / 0 22283/ 32768 | ++===========================+===============+====================================================+ +| 0 910B4 | OK | 85.0 36 0 / 0 | )"; const std::string wrongNpuSmiInfo2 = R"( -+===========================+===============+====================================================+ -| 0 910 | | 85.0 36 0 / 0 | -| 0 | 0000:C1:00.0 | 0 0 / 0 | -+===========================+===============+====================================================+ +| 0 Ascend910 | OK | 182.0 36 0 / 0 | +| 0 0 | 0000:9D:00.0 | 0 0 / 0 3402 / 65536 | ++------------------------------------------------------------------------------------------------+ +| 0 Ascend910 | OK | - 35 0 / 0 | )"; const std::string wrongNpuSmiInfo3 = R"( +===========================+===============+====================================================+ +| 0 910B4 | OK | 85.0 36 0 / 0 | +| 0 | 0000:C1:00.0 | 0 0 / 0 22283/ 32768 | ++===========================+===============+====================================================+ | 0 910B4 | OK | 85.0 36 0 / 0 | | 0 | 0000:C1:00.0 | 0 0 / 0 | +===========================+===============+====================================================+ @@ -538,10 +588,19 @@ const std::string wrongNpuSmiInfo3 = R"( const std::string wrongNpuSmiInfo4 = R"( +===========================+===============+====================================================+ | 0 910B4 | OK | 85.0 36 0 / 0 | +| 0 | 0000:C1:00.0 | 0 0 / 0 22283/ 32768 | ++===========================+===============+====================================================+ +| 0 910B4 | OK | 85.0 36 0 / 0 | | 0 | 0000:C1:00.0 | 0 4 / 5.s 30759/ 32768 | +===========================+===============+====================================================+ )"; +const std::string wrongNpuSmiInfo5 = R"( +| 1536 310P3 | OK | NA 51 0 / 0 | +| 0 0 | 0000:06:00.0 | 0 1596 / 44280 | ++-------------------------------+-----------------+------------------------------------------------------+ +| 1536 310P3 | OK | NA 50 0 / 0 | +)"; // mock ls /dev | grep davinci 910C 1Chip2Npu const std::string devDavinciInfo = R"(davinci0 @@ -769,6 +828,11 @@ std::vector expectUseHBM = std::vector{ 22283, 22267, 2818, 2819, 2829 std::vector expectLimitHBM = std::vector{ 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768 }; std::vector expectUseHBM16 = std::vector{ 3402, 3200, 3396, 3205, 3395, 3203, 3395, 3203, 52553, 52358, 52567, 52345, 52552, 52358, 52554, 52358 }; + +std::vector expectUseMemory310 = std::vector{ 1596, 1325, 1452, 1444, 1353, 1529, 1656, 1278}; +std::vector expectTotalMemory310 = std::vector{ 44280, 43693, 44280, 43693, 44280, 43693, 44280, 43693}; + + const int expectLimitHBM16 = 65536; class XpuCollectorTest : public ::testing::Test {}; @@ -788,7 +852,6 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetCountNPUInfo) // case 1.1 get from /dev successfully { EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(devDavinciInfo))); - EXPECT_CALL(*cmdTool.get(), GetCmdResult("pip3 list")).WillRepeatedly(testing::Return(PIP_LIST_INFO)); auto status = probe->OnGetNPUInfo(true); EXPECT_TRUE(status.IsOk()); auto devInfo = probe->GetClusterInfo(); @@ -803,7 +866,6 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetCountNPUInfo) { probe = std::make_shared("co200", tool, cmdTool, params); EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(""))).WillOnce(testing::Return(stringToVector(npuSmiInfo910B))); - EXPECT_CALL(*cmdTool.get(), GetCmdResult("pip3 list")).WillRepeatedly(testing::Return(PIP_LIST_INFO)); auto status = probe->OnGetNPUInfo(true); EXPECT_TRUE(status.IsOk()); auto devInfo = probe->GetClusterInfo(); @@ -832,7 +894,7 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetCountNPUInfo) EXPECT_CALL(*tool.get(), Read).WillRepeatedly(testing::Return(litebus::Option{})); status = probe->OnGetNPUInfo(true); EXPECT_TRUE(status.IsError()); - EXPECT_EQ(status.RawMessage(), "can not get npu info from npu-smi info"); + EXPECT_EQ(status.RawMessage(), "failed to parse npu smi info!"); } } @@ -850,7 +912,6 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetNPUSmiInfo) // case 2.1 success get from npu-smi info { EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(npuSmiInfo910C))); - EXPECT_CALL(*cmdTool.get(), GetCmdResult("pip3 list")).WillRepeatedly(testing::Return(PIP_LIST_INFO)); auto status = probe->OnGetNPUInfo(false); EXPECT_TRUE(status.IsOk()); auto devInfo = probe->GetClusterInfo(); @@ -866,6 +927,37 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetNPUSmiInfo) EXPECT_EQ("Ascend910", devInfo->devProductModel); } + // case 2.2 success get from npu-smi info for 310P3 + { + EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(npuSmiInfo310P3))); + auto status = probe->OnGetNPUInfo(false); + EXPECT_TRUE(status.IsOk()); + auto devInfo = probe->GetClusterInfo(); + ASSERT_EQ(devInfo->devIDs.size(), expectID8.size()); + for (size_t i = 0; i < expectID8.size(); i++) { + EXPECT_EQ(expectID8[i], devInfo->devIDs[i]); + EXPECT_EQ(expectUseMemory310[i], devInfo->devUsedMemory[i]); + EXPECT_EQ(expectTotalMemory310[i], devInfo->devTotalMemory[i]); + EXPECT_EQ(expectUseMemory310[i], devInfo->devUsedHBM[i]); + EXPECT_EQ(expectTotalMemory310[i], devInfo->devLimitHBMs[i]); + EXPECT_EQ(0, devInfo->health[i]); + } + EXPECT_EQ("310P3", devInfo->devProductModel); + } +} + + +// case 2.1: test for get wrong npu smi info +TEST_F(XpuCollectorTest, TestNpuProbeOnGetNPUSmiInfoFailed) +{ + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); + auto params = std::make_shared(); + params->ldLibraryPath = emptyLdLibraryPath; + params->deviceInfoPath = "/home/sn/config/topology-info.json"; + params->collectMode = runtime_manager::NPU_COLLECT_COUNT; + auto probe = std::make_shared("co200", tool, cmdTool, params); + // case 2.2 failed get from npu-smi info and get from json { EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector("AAAAA"))); @@ -873,17 +965,17 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetNPUSmiInfo) EXPECT_CALL(*cmdTool.get(), GetCmdResult("pip3 list")).WillRepeatedly(testing::Return(PIP_LIST_INFO)); auto status = probe->OnGetNPUInfo(false); EXPECT_TRUE(status.IsError()); - EXPECT_EQ(status.RawMessage(), "can not get npu info from npu-smi info"); + EXPECT_EQ(status.RawMessage(), "failed to parse npu smi info!"); auto devInfo = probe->GetClusterInfo(); EXPECT_EQ(devInfo->devIDs.size(), 6); EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(wrongNpuSmiInfo1))); status = probe->OnGetNPUInfo(false); - EXPECT_EQ(status.RawMessage(), "parse npu basic info failed, no chip info in following line."); + EXPECT_EQ(status.RawMessage(), "parse npu chip info failed."); EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(wrongNpuSmiInfo2))); status = probe->OnGetNPUInfo(false); - EXPECT_EQ(status.RawMessage(), "can not get npu info from npu-smi info"); + EXPECT_EQ(status.RawMessage(), "parse npu chip info failed."); EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(wrongNpuSmiInfo3))); status = probe->OnGetNPUInfo(false); @@ -892,6 +984,10 @@ TEST_F(XpuCollectorTest, TestNpuProbeOnGetNPUSmiInfo) EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(wrongNpuSmiInfo4))); status = probe->OnGetNPUInfo(false); EXPECT_EQ(status.RawMessage(), "parse npu chip info failed."); + + EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).WillOnce(testing::Return(stringToVector(wrongNpuSmiInfo5))); + status = probe->OnGetNPUInfo(false); + EXPECT_EQ(status.RawMessage(), "parse npu chip info failed."); } } @@ -1068,14 +1164,14 @@ TEST_F(XpuCollectorTest, TestGetNPUTopoInfo) TEST_F(XpuCollectorTest, TestRefreshTopoInfo) { - auto tool = std::make_shared(); std::string nodeID = "co200"; - auto cmdTool = std::make_shared(); auto params = std::make_shared(); params->ldLibraryPath = emptyLdLibraryPath; params->deviceInfoPath = "/home/sn/config/topology-info.json"; // case1: don't get any npu info { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = "false"; EXPECT_CALL(*cmdTool.get(), GetCmdResult(testing::_)).Times(0); EXPECT_CALL(*tool.get(), Read(testing::_)).Times(0); @@ -1086,6 +1182,8 @@ TEST_F(XpuCollectorTest, TestRefreshTopoInfo) // case2: count scene, success get { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = runtime_manager::NPU_COLLECT_COUNT; EXPECT_CALL(*cmdTool.get(), GetCmdResult("ls /dev | grep davinci")).WillOnce(testing::Return(stringToVector(devDavinciInfo))); auto probe = std::make_shared(nodeID, tool, cmdTool, params); @@ -1095,6 +1193,8 @@ TEST_F(XpuCollectorTest, TestRefreshTopoInfo) // case3: hbm scene, success get { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = runtime_manager::NPU_COLLECT_HBM; EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info")).WillOnce(testing::Return(stringToVector(npuSmiInfo910B))); auto probe = std::make_shared(nodeID, tool, cmdTool, params); @@ -1104,6 +1204,8 @@ TEST_F(XpuCollectorTest, TestRefreshTopoInfo) // case4: sfmd scene, success get { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = runtime_manager::NPU_COLLECT_SFMD; EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info")).WillOnce(testing::Return(stringToVector(npuSmiInfo910C))); EXPECT_CALL(*tool.get(), Read("/etc/hccn.conf")).WillOnce(testing::Return(litebus::Option{hccn16NpuConf})); @@ -1114,6 +1216,8 @@ TEST_F(XpuCollectorTest, TestRefreshTopoInfo) // case5: topo scene, success get { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = runtime_manager::NPU_COLLECT_TOPO; EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info")).WillOnce(testing::Return(stringToVector(npuSmiInfo910B))); EXPECT_CALL(*cmdTool.get(), GetCmdResultWithError("npu-smi info -t topo")).WillOnce(testing::Return(stringToVector(npuSminTopoInfo))); @@ -1122,8 +1226,10 @@ TEST_F(XpuCollectorTest, TestRefreshTopoInfo) EXPECT_TRUE(status.IsOk()); } - // case5: off scene or other scene, failed get + // case6: off scene or other scene, failed get { + auto tool = std::make_shared(); + auto cmdTool = std::make_shared(); params->collectMode = "off"; auto probe = std::make_shared(nodeID, tool, cmdTool, params); auto status = probe->RefreshTopo(); @@ -1238,7 +1344,8 @@ TEST_F(XpuCollectorTest, TestUpdateInfo) EXPECT_EQ(devInfo->devLimitHBMs.size(), 8); EXPECT_EQ(devInfo->devIPs.size(), 8); probe->npuNum_ = 4; - EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info -t topo")).WillRepeatedly(testing::Return(topoInfo)); + EXPECT_CALL(*cmdTool.get(), GetCmdResult("npu-smi info -t topo")).WillOnce(testing::Return(stringToVector(""))).WillRepeatedly(testing::Return(topoInfo)); + probe->UpdateDevTopo(); probe->UpdateDevTopo(); EXPECT_EQ(devInfo->devTopo.size(), 4); } @@ -1315,7 +1422,7 @@ TEST_F(XpuCollectorTest, TestNpuCollectorByCmd) params->ldLibraryPath = emptyLdLibraryPath; params->deviceInfoPath = "/home/sn/config/topology-info.json"; params->collectMode = runtime_manager::NPU_COLLECT_COUNT; - EXPECT_CALL(*cmdTool.get(), GetCmdResult("ls /dev | grep davinci")).WillOnce(testing::Return(stringToVector(devDavinciInfo))); + EXPECT_CALL(*cmdTool.get(), GetCmdResult("ls /dev | grep davinci")).WillRepeatedly(testing::Return(stringToVector(devDavinciInfo))); auto probe = std::make_shared(nodeID, tool, cmdTool, params); auto npuCollector = runtime_manager::SystemXPUCollector(nodeID, runtime_manager::metrics_type::NPU, tool, params); npuCollector.probe_ = probe; diff --git a/functionsystem/tests/unit/runtime_manager/metrics/metrics_actor_test.cpp b/functionsystem/tests/unit/runtime_manager/metrics/metrics_actor_test.cpp index bddffb8ba1625ebcb2eb7dac9e042ad964bb470d..3da23124e67f0990c7cc6bc034a63b507dc5dbd8 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/metrics_actor_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/metrics/metrics_actor_test.cpp @@ -13,20 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "runtime_manager/metrics/metrics_actor.h" + #include #include "common/utils/exec_utils.h" -#include "runtime_manager/metrics/collector/base_metrics_collector.h" -#include "utils/future_test_helper.h" -#include "runtime_manager/config/flags.h" #include "mock_function_agent_actor.h" -#include "runtime_manager/metrics/metrics_actor.h" -#include "mocks/mock_instance_memory_collector.h" -#include "runtime_manager/manager/runtime_manager.h" -#include "runtime_manager/executor/runtime_executor.h" #include "mocks/mock_local_sched_srv.h" -#include "mocks/mock_proc_fs_tools.h" -#include "runtime_manager/metrics/collector/system_memory_collector.h" +#include "runtime_manager/config/flags.h" +#include "runtime_manager/executor/runtime_executor.h" +#include "runtime_manager/manager/runtime_manager.h" +#include "runtime_manager/metrics/collector/base_metrics_collector.h" +#include "utils/future_test_helper.h" #include "utils/port_helper.h" namespace functionsystem::test { @@ -54,7 +52,7 @@ public: class MetricsActorTest : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { port_ = GetPortEnv("LITEBUS_PORT", 8080); } @@ -84,6 +82,31 @@ protected: std::shared_ptr runtimeManager_; }; +struct DiskInfo { + std::string name; + uint64_t size; + std::string mountPoints; +}; + +void CheckDiskResourceMatch(const Resource& resource, const std::vector& expected) +{ + EXPECT_EQ(resource.extensions_size(), expected.size()); + EXPECT_TRUE(resource.vectors().values().count(metrics_type::DISK)); + EXPECT_EQ(resource.vectors().values().at(metrics_type::DISK).vectors().size(), 1); + auto diskVector = resource.vectors().values().at(metrics_type::DISK).vectors().begin()->second; + for (size_t i = 0; i < expected.size(); ++i) { + const auto& ext = resource.extensions(i); + EXPECT_TRUE(ext.has_disk()); + const auto& disk = ext.disk(); + const auto& exp = expected[i]; + + EXPECT_EQ(diskVector.values(i), exp.size); + EXPECT_EQ(disk.name(), exp.name); + EXPECT_EQ(disk.size(), exp.size); + EXPECT_EQ(disk.mountpoints(), exp.mountPoints); + } +} + /** * Feature: MetricsActor * Description: Build UpdateMetricsRequest @@ -96,6 +119,11 @@ TEST_F(MetricsActorTest, BuildUpdateMetricsRequest) runtimeManager_->metricsClient_->StartDiskUsageMonitor(); runtimeManager_->metricsClient_->StartRuntimeMemoryLimitMonitor(); + const char *argv[] = { "./runtime-manager", "--node_id=node1"}; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + metricsActor_->SetConfig(flags); + // stub instanceInfos messages::RuntimeInstanceInfo runtimeInstanceInfo; runtimeInstanceInfo.set_runtimeid("runtimeID1"); @@ -107,6 +135,16 @@ TEST_F(MetricsActorTest, BuildUpdateMetricsRequest) metricsActor_->AddInstance(runtimeInstanceInfo, testPid, cpuLimit, memoryLimit); EXPECT_AWAIT_TRUE([&]() -> bool { return metricsActor_->instanceInfos_.size() >= 1; }); + // parse disk resources + std::string diskResources = R"( + [{"name": "disk1", "size": "100G", "mountPoints": "/tmp/disk1/"}, + {"name": "disk2", "size": "200G", "mountPoints": "/tmp/disk2a/"}] + )"; + metricsActor_->ResolveDiskResourceMetricsCollector(diskResources); + auto iter = metricsActor_->filter_.find("system-disk"); + EXPECT_NE(iter, metricsActor_->filter_.end()); + auto diskMetrics = iter->second->GetMetrics().Get(); + // given auto given = std::vector>{ {runtime_manager::Metrics{100.0, 120.0, {}, {}, runtime_manager::metrics_type::CPU}}, @@ -115,22 +153,27 @@ TEST_F(MetricsActorTest, BuildUpdateMetricsRequest) {runtime_manager::Metrics{140.0, 160.0, {"id-1"}, {}, runtime_manager::metrics_type::MEMORY}}, {runtime_manager::Metrics{180.0, 200.0, {"id-2"}, {}, runtime_manager::metrics_type::CPU}}, {runtime_manager::Metrics{220.0, 240.0, {"id-2"}, {}, runtime_manager::metrics_type::MEMORY}}, + {diskMetrics} }; // got - auto actor = std::make_shared(); - auto got = actor->BuildUpdateMetricsRequest(given); + auto got = metricsActor_->BuildUpdateMetricsRequest(given); // want + // 1.validate cpu/memory resource messages::UpdateResourcesRequest req; EXPECT_TRUE(req.ParseFromString(got) == 1); auto unit = req.resourceunit(); + std::cout << unit.ShortDebugString() << std::endl; EXPECT_EQ(unit.capacity().resources().find("CPU")->second.scalar().value(), 120.0); EXPECT_EQ(unit.actualuse().resources().find("CPU")->second.scalar().value(), 100.0); EXPECT_EQ(unit.allocatable().resources().find("CPU")->second.scalar().value(), 120.0); + EXPECT_EQ(unit.capacity().resources().find("Memory")->second.scalar().value(), 130.0); EXPECT_EQ(unit.actualuse().resources().find("Memory")->second.scalar().value(), 110.0); EXPECT_EQ(unit.allocatable().resources().find("Memory")->second.scalar().value(), 130.0); + + // 2.validate instance info EXPECT_EQ(unit.instances().find("id-1")->second.actualuse().resources().find("CPU")->second.scalar().value(), 120.0); EXPECT_EQ(unit.instances().find("id-1")->second.actualuse().resources().find("Memory")->second.scalar().value(), @@ -140,6 +183,23 @@ TEST_F(MetricsActorTest, BuildUpdateMetricsRequest) EXPECT_EQ(unit.instances().find("id-2")->second.actualuse().resources().find("Memory")->second.scalar().value(), 220.0); + // 3.validate disk resource + std::vector expectedDisks = { + {"disk1", 100, "/tmp/disk1/"}, + {"disk2", 200, "/tmp/disk2a/"}, + }; + auto diskResourceIter = unit.capacity().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.capacity().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); + + diskResourceIter = unit.actualuse().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.actualuse().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); + + diskResourceIter = unit.allocatable().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.allocatable().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); + runtimeManager_->metricsClient_->StopUpdateResource(); runtimeManager_->metricsClient_->StopDiskUsageMonitor(); runtimeManager_->metricsClient_->StopRuntimeMemoryLimitMonitor(); @@ -153,16 +213,33 @@ TEST_F(MetricsActorTest, BuildUpdateMetricsRequest) */ TEST_F(MetricsActorTest, BuildResourceUnit) { - DevClusterMetrics metrics{ "a1b2c3d4", 4, {}, { { resource_view::IDS_KEY, { 85, 90, 78, 92 } } } }; + const char *argv[] = { "./runtime-manager", "--node_id=node1"}; + runtime_manager::Flags flags; + flags.ParseFlags(std::size(argv), argv); + metricsActor_->SetConfig(flags); + + // parse disk resources + std::string diskResources = R"( + [{"name": "disk1", "size": "100G", "mountPoints": "/tmp/disk1/"}, + {"name": "disk2", "size": "200G", "mountPoints": "/tmp/disk2a/"}] + )"; + metricsActor_->ResolveDiskResourceMetricsCollector(diskResources); + auto iter = metricsActor_->filter_.find("system-disk"); + EXPECT_NE(iter, metricsActor_->filter_.end()); + auto diskMetrics = iter->second->GetMetrics().Get(); + // given + DevClusterMetrics npuMetrics{ "a1b2c3d4", 4, {}, { { resource_view::IDS_KEY, { 85, 90, 78, 92 } } } }; auto given = std::vector>{ { runtime_manager::Metrics{ - 100.0, 120.0, {}, {}, runtime_manager::metrics_type::CPU, collector_type::INSTANCE, { metrics } } }, + 4.0, 4.0, {}, {}, runtime_manager::metrics_type::NPU, collector_type::SYSTEM, { npuMetrics } } }, + { runtime_manager::Metrics{ 100.0, 120.0, {}, {}, runtime_manager::metrics_type::CPU,} }, { runtime_manager::Metrics{ 110.0, 130.0, {}, {}, runtime_manager::metrics_type::MEMORY } }, { runtime_manager::Metrics{ 120.0, 140.0, { "id-1" }, {}, runtime_manager::metrics_type::CPU } }, { runtime_manager::Metrics{ 140.0, 160.0, { "id-1" }, {}, runtime_manager::metrics_type::MEMORY } }, { runtime_manager::Metrics{ 180.0, 200.0, { "id-2" }, {}, runtime_manager::metrics_type::CPU } }, { runtime_manager::Metrics{ 220.0, 240.0, { "id-2" }, {}, runtime_manager::metrics_type::MEMORY } }, + {diskMetrics}, }; // got @@ -171,12 +248,15 @@ TEST_F(MetricsActorTest, BuildResourceUnit) // want auto unit = got; + // 1.validate cpu/memory resource EXPECT_EQ(unit.capacity().resources().find("CPU")->second.scalar().value(), 120.0); EXPECT_EQ(unit.actualuse().resources().find("CPU")->second.scalar().value(), 100.0); EXPECT_EQ(unit.allocatable().resources().find("CPU")->second.scalar().value(), 120.0); EXPECT_EQ(unit.capacity().resources().find("Memory")->second.scalar().value(), 130.0); EXPECT_EQ(unit.actualuse().resources().find("Memory")->second.scalar().value(), 110.0); EXPECT_EQ(unit.allocatable().resources().find("Memory")->second.scalar().value(), 130.0); + + // 2.validate instance info EXPECT_EQ(unit.instances().find("id-1")->second.actualuse().resources().find("CPU")->second.scalar().value(), 120.0); EXPECT_EQ(unit.instances().find("id-1")->second.actualuse().resources().find("Memory")->second.scalar().value(), @@ -185,8 +265,27 @@ TEST_F(MetricsActorTest, BuildResourceUnit) 180.0); EXPECT_EQ(unit.instances().find("id-2")->second.actualuse().resources().find("Memory")->second.scalar().value(), 220.0); + + + // 3.validate disk resource + std::vector expectedDisks = { + {"disk1", 100, "/tmp/disk1/"}, + {"disk2", 200, "/tmp/disk2a/"}, + }; + auto diskResourceIter = unit.capacity().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.capacity().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); + + diskResourceIter = unit.actualuse().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.actualuse().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); + + diskResourceIter = unit.allocatable().resources().find(metrics_type::DISK); + ASSERT_NE(diskResourceIter, unit.allocatable().resources().end()); + CheckDiskResourceMatch(diskResourceIter->second, expectedDisks); } +#if 0 /** * Feature: MetricsActorTest MonitorDiskUsageTest * Description: monitor disk usage @@ -197,7 +296,7 @@ TEST_F(MetricsActorTest, BuildResourceUnit) * Expectation: * 3. Report error to function agent */ -TEST_F(MetricsActorTest, DISABLED_MonitorDiskUsageTest) +TEST_F(MetricsActorTest, MonitorDiskUsageTest) { auto mockFuncAgentActor = std::make_shared(); litebus::Spawn(mockFuncAgentActor); @@ -206,7 +305,7 @@ TEST_F(MetricsActorTest, DISABLED_MonitorDiskUsageTest) litebus::os::Rmdir("/diskMonitorTestDir"); litebus::os::Rmdir("/home/snuser/testdir"); - litebus::os::Rmdir("/tmp/testdir"); + litebus::os::Rmdir("/tmp/MetricsActorTest/testdir"); std::string dir = "/diskMonitorTestDir"; litebus::os::Mkdir(dir); std::string portOption = "--port=" + std::to_string(port_); @@ -222,7 +321,7 @@ TEST_F(MetricsActorTest, DISABLED_MonitorDiskUsageTest) "--disk_usage_limit=1", "--runtime_home_dir=/home/snuser", "--snuser_disk_usage_limit=1", - "--tmp_disk_usage_limit=600", + "--tmp_disk_usage_limit=500", "--disk_usage_monitor_duration=50", "--disk_usage_monitor_notify_failure_enable=false", }; @@ -251,10 +350,10 @@ TEST_F(MetricsActorTest, DISABLED_MonitorDiskUsageTest) mockFuncAgentActor->requestArray_.clear(); litebus::os::Rmdir(dir); // write tmp file - dir = "/tmp/testdir"; + dir = "/tmp/MetricsActorTest/testdir"; litebus::os::Rmdir(dir); litebus::os::Mkdir(dir); - ExecuteCommand("dd if=/dev/zero of=" + dir + "/test.txt bs=600M count=1"); + ExecuteCommand("dd if=/dev/zero of=" + dir + "/test.txt bs=500M count=1"); EXPECT_AWAIT_TRUE([&]() -> bool { return mockFuncAgentActor->requestArray_.size() >= 1; }); request = mockFuncAgentActor->requestArray_[mockFuncAgentActor->requestArray_.size()-1]; EXPECT_TRUE(request->message().find("tmp dir") != std::string::npos); @@ -266,6 +365,54 @@ TEST_F(MetricsActorTest, DISABLED_MonitorDiskUsageTest) litebus::Await(mockFuncAgentActor->GetAID()); litebus::os::Rmdir(dir); } +#endif + +TEST_F(MetricsActorTest, MonitorDiskUsageErrorTest) +{ + auto mockFuncAgentActor = std::make_shared(); + litebus::Spawn(mockFuncAgentActor); + mockFuncAgentActor->needAutoSendResp_ = false; + litebus::Async(metricsActor_->GetAID(), &TestMetricsActor::UpdateAgentInfo, mockFuncAgentActor->GetAID()); + + litebus::os::Rmdir("/diskMonitorTestDir"); + litebus::os::Rmdir("/home/snuser/testdir"); + litebus::os::Rmdir("/tmp/MetricsActorTest/testdir"); + std::string dir = "/diskMonitorTestDir"; + litebus::os::Mkdir(dir); + std::string portOption = "--port=" + std::to_string(port_); + const char *argv[] = { "./runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--host_ip=127.0.0.1", + "--runtime_ld_library_path=/home/sn/runtime", + portOption.c_str(), + "--agent_address=127.0.0.1:8081", + "--runtime_initial_port=20000", + "--disk_usage_monitor_path=/diskMonitorTestDir1;;/diskMonitorTestDir", + "--disk_usage_limit=1", + "--runtime_home_dir=/home/snuser", + "--snuser_disk_usage_limit=1", + "--tmp_disk_usage_limit=500", + "--disk_usage_monitor_duration=50", + "--disk_usage_monitor_notify_failure_enable=false", + }; + runtime_manager::Flags flags; + ASSERT_TRUE(flags.ParseFlags(15, argv).IsNone()); + + metricsActor_->SetConfig(flags); + litebus::Async(metricsActor_->GetAID(), &TestMetricsActor::StartDiskUsageMonitor); + ExecuteCommand("mv /usr/bin/du /usr/bin/du1"); + dir = "/tmp/MetricsActorTest/testdir"; + litebus::os::Rmdir(dir); + litebus::os::Mkdir(dir); + ExecuteCommand("dd if=/dev/zero of=" + dir + "/test.txt bs=500M count=1"); + EXPECT_EQ(mockFuncAgentActor->requestArray_.size(), 0); + litebus::os::Rmdir(dir); + ExecuteCommand("mv /usr/bin/du1 /usr/bin/du"); + litebus::Terminate(mockFuncAgentActor->GetAID()); + litebus::Await(mockFuncAgentActor->GetAID()); + litebus::os::Rmdir(dir); +} TEST_F(MetricsActorTest, OomMonitor_ExceedControlLimit_Trigger_OomKillInstance) { @@ -285,7 +432,7 @@ TEST_F(MetricsActorTest, OomMonitor_ExceedControlLimit_Trigger_OomKillInstance) "--oom_consecutive_detection_count=2", }; runtime_manager::Flags flags; - auto ret = flags.ParseFlags(13, argv); + auto ret = flags.ParseFlags(std::size(argv), argv); ASSERT_TRUE(ret.IsNone()) << ret.Get(); runtimeManager_->SetConfig(flags); @@ -352,7 +499,7 @@ TEST_F(MetricsActorTest, OomMonitor_ExceedControlLimit_Trigger_OomKillInstance) runtimeManager_->healthCheckClient_->AddRuntimeRecord(mockFuncAgentActor->GetAID(), testPid, instanceID, runtimeID, runtimeID); - metricsActor_->RuntimeMemoryMetricsProcess({metrics1, metrics2, metrics3, metrics4}); + metricsActor_->RuntimeMemoryMetricsProcess({metrics1, metrics2, metrics3, metrics4, metrics4}); // expected runtime killed EXPECT_AWAIT_TRUE([&]() -> bool { return executor->runtime2PID_.find(runtimeID) == executor->runtime2PID_.end(); }); @@ -409,4 +556,52 @@ TEST_F(MetricsActorTest, CustomResourceTest) iter = metricsActor_->filter_.find("system-CustomResource222"); EXPECT_EQ(iter, metricsActor_->filter_.end()); } + +/* + * Verifies disk resource parsing handles both valid and invalid config entries: + * - disk1 (valid config) + * - disk2 (invalid mount point) + */ +TEST_F(MetricsActorTest, DiskResourceTestWithMixedConfig) +{ + std::string portOption = "--port=" + std::to_string(port_); + + const char* disk_json = R"([{"name":"disk1","size":"100G","mountPoints":"/tmp/disk1/"}, + {"name":"disk2","size":"200G","mountPoints":"/tmp/invalidDisk"}])"; + std::string disk_arg = "--disk_resources=" + std::string(disk_json); + const char *argv[] = { "./runtime_manager", + "--node_id=node1", + "--ip=127.0.0.1", + "--proxy_ip=127.0.0.1", + "--host_ip=127.0.0.1", + "--runtime_ld_library_path=/home/sn/runtime", + portOption.c_str(), + "--agent_address=127.0.0.1:8081", + "--runtime_initial_port=20000", + disk_arg.c_str(), + + }; + runtime_manager::Flags flags; + ASSERT_TRUE(flags.ParseFlags(10, argv).IsNone()); + + EXPECT_EQ(flags.GetDiskResources(), std::string(disk_json)); + metricsActor_->ResolveDiskResourceMetricsCollector(flags.GetDiskResources()); + auto iter = metricsActor_->filter_.find("system-disk"); + EXPECT_NE(iter, metricsActor_->filter_.end()); + auto metrics = iter->second->GetMetrics().Get(); + EXPECT_EQ(metrics.collectorType, runtime_manager::collector_type::SYSTEM); + EXPECT_EQ(metrics.metricsType, "disk"); + + auto devMetrics = metrics.devClusterMetrics.Get(); + + EXPECT_EQ(devMetrics.intsInfo.count(resource_view::DISK_RESOURCE_NAME), 1); + EXPECT_EQ(devMetrics.intsInfo.at(resource_view::DISK_RESOURCE_NAME).size(), 1); + EXPECT_EQ(devMetrics.intsInfo.at(resource_view::DISK_RESOURCE_NAME)[0], 100); + + EXPECT_EQ(devMetrics.extensionInfo.size(), 1); + const auto& diskExt = devMetrics.extensionInfo[0].disk(); + EXPECT_EQ(diskExt.name(), "disk1"); + EXPECT_EQ(diskExt.size(), 100); + EXPECT_EQ(diskExt.mountpoints(), "/tmp/disk1/"); +} } // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.cpp b/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.cpp index c394c25a038ec5a755fd8b370c2f998b15287e28..bec4e060d887396e96cccaac601f242681ce4ef7 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.cpp +++ b/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.cpp @@ -16,8 +16,8 @@ #include "mock_function_agent_actor.h" -#include "status/status.h" -#include "logs/logging.h" +#include "common/status/status.h" +#include "common/logs/logging.h" namespace functionsystem::runtime_manager::test { void MockFunctionAgentActor::Init() diff --git a/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.h b/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.h index 36753f76d9cc687df50c6731a5f89ccc578c9a7f..02d6700a97c0d6a666dec79aaea645750370528d 100644 --- a/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.h +++ b/functionsystem/tests/unit/runtime_manager/metrics/mock_function_agent_actor.h @@ -22,7 +22,7 @@ #include "actor/actor.hpp" #include "async/future.hpp" -#include "proto/pb/message_pb.h" +#include "common/proto/pb/message_pb.h" namespace functionsystem::runtime_manager::test { class MockFunctionAgentActor : public litebus::ActorBase { diff --git a/functionsystem/tests/unit/runtime_manager/utils/runtime_utils_test.cpp b/functionsystem/tests/unit/runtime_manager/utils/runtime_utils_test.cpp index 69b9d01eb70a5787b004da85d41f2614c8e43426..2c531fd79add71c9d83556a613b570b96ad3c240 100644 --- a/functionsystem/tests/unit/runtime_manager/utils/runtime_utils_test.cpp +++ b/functionsystem/tests/unit/runtime_manager/utils/runtime_utils_test.cpp @@ -15,7 +15,7 @@ */ #include -#include "files.h" +#include "common/utils/files.h" #include "common/utils/exec_utils.h" #include "exec/exec.hpp" #include "gtest/gtest.h" @@ -71,6 +71,7 @@ class RuntimeStdRedirectorTest : public ::testing::Test { public: void SetUp() override { + litebus::os::Rm("/tmp/stdout.log"); } void TearDown() override @@ -125,51 +126,34 @@ TEST_F(RuntimeStdRedirectorTest, RedirectorLogTest) EXPECT_TRUE(permission.Get().others == 0); litebus::Try> s = litebus::Exec::CreateExec( - "echo output1; /usr/bin/cp a b; /usr/bin/cp a b; /usr/bin/cp a b; /usr/bin/cp a b;", litebus::None(), + "echo output1; sleep 1; /usr/bin/cp a b; /usr/bin/cp a b; /usr/bin/cp a b; /usr/bin/cp a b;", litebus::None(), litebus::ExecIO::CreateFDIO(STDIN_FILENO), litebus::ExecIO::CreatePipeIO(), litebus::ExecIO::CreatePipeIO()); litebus::Async(redirector->GetAID(), &StdRedirector::StartRuntimeStdRedirection, "runtimeID", "instanceID", s.Get()->GetOut(), s.Get()->GetErr()); ASSERT_AWAIT_TRUE([=]() { return !s.Get()->GetStatus().IsInit(); }); ASSERT_AWAIT_TRUE([=]() { - sleep(1); auto output = litebus::os::Read("/tmp/stdout.log"); if (output.IsNone()) { return false; } - const auto &msg = output.Get(); - return msg.find(INFO_LEVEL) != msg.npos && msg.find(ERROR_LEVEL) != msg.npos; + auto lines = RemoveEmptyLines("/tmp/stdout.log"); + return lines.size() >= 5; }); auto info = StdRedirector::GetStdLog("/tmp/stdout.log", "runtimeID", INFO_LEVEL, 1); - auto err = StdRedirector::GetStdLog("/tmp/stdout.log", "runtimeID", ERROR_LEVEL, 2); + auto err = StdRedirector::GetStdLog("/tmp/stdout.log", "runtimeID", ERROR_LEVEL, 20, 3); EXPECT_EQ(info.find("runtimeID") != info.npos, true); EXPECT_EQ(info.find(INFO_LEVEL) != info.npos, true); EXPECT_EQ(err.find("runtimeID") != info.npos, true); EXPECT_EQ(err.find(ERROR_LEVEL) != info.npos, true); - - // lines to read is default value 1000 - err = StdRedirector::GetStdLog("/tmp/stdout.log", "runtimeID", ERROR_LEVEL, 20); - EXPECT_EQ(err.find("runtimeID") != info.npos, true); - EXPECT_EQ(err.find(ERROR_LEVEL) != info.npos, true); - auto errLines = litebus::strings::Split(err, "\n"); - errLines.erase(std::remove(errLines.begin(), errLines.end(), ""), errLines.end()); - EXPECT_EQ(static_cast(errLines.size()), 4); - - // lines to read is 2 - err = StdRedirector::GetStdLog("/tmp/stdout.log", "runtimeID", ERROR_LEVEL, 20, 3); - EXPECT_EQ(err.find("runtimeID") != info.npos, true); - EXPECT_EQ(err.find(ERROR_LEVEL) != info.npos, true); EXPECT_TRUE(litebus::strings::Split(err, "\n").size() < 4); auto output = litebus::os::Read("/tmp/stdout.log"); EXPECT_EQ(output.IsNone(), false); auto msg = output.Get(); std::cout << "msg: \n" << msg << std::endl; - auto lines = RemoveEmptyLines("/tmp/stdout.log"); - EXPECT_EQ(lines.size(), size_t(5)); litebus::Terminate(redirector->GetAID()); litebus::Await(redirector->GetAID()); - litebus::os::Rm("/tmp/stdout.log"); umask(origin); } diff --git a/functionsystem/tests/unit/runtime_manager/utils/volume_mount_test.cpp b/functionsystem/tests/unit/runtime_manager/utils/volume_mount_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5eb2d4a118edeb01f3c855489243cef03a38580a --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/utils/volume_mount_test.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "runtime_manager/utils/volume_mount.h" + +#include +#include +#include +#include +#include + +#include + +#include "common/logs/logging.h" +#include "mocks/mock_volume_mount.h" +#include "utils/os_utils.hpp" + +using namespace functionsystem::runtime_manager; + +namespace functionsystem::test { + +class VolumeMountTest : public ::testing::Test { +public: + void SetUp() override + { + mockMounter_ = std::make_shared(); + mounter_ = std::make_shared(); + } + + void TearDown() override + { + } + +protected: + std::shared_ptr mockMounter_; + std::shared_ptr mounter_; +}; + +TEST_F(VolumeMountTest, VolumeMountTestEmptyFuncMounts) +{ + messages::FuncMountConfig mountConfig; + messages::FuncMountUser mountUser; + mountUser.set_userid(1004); + mountUser.set_groupid(1004); + mountConfig.mutable_funcmountuser()->CopyFrom(mountUser); + + auto mountResultStatus = mounter_->MountVolumesForFunction(mountConfig); + YRLOG_INFO("mountResultStatus: {}", mountResultStatus.ToString()); + EXPECT_EQ(mountResultStatus.IsOk(), true); +} + +TEST_F(VolumeMountTest, VolumeMountTestLocalPathExists) +{ + messages::FuncMountConfig mountConfig; + messages::FuncMountUser mountUser; + mountUser.set_userid(1004); + mountUser.set_groupid(1004); + mountConfig.mutable_funcmountuser()->CopyFrom(mountUser); + + auto funcMountSFS = mountConfig.add_funcmounts(); + funcMountSFS->set_mounttype("sfs"); + funcMountSFS->set_mountsharepath("10.244.134.153:/home/disk/lzt/nfs-share"); + funcMountSFS->set_localmountpath("/testMount/testSFS"); + + (void)litebus::os::Mkdir("/testMount/testSFS"); + auto mountResultStatus = mockMounter_->MountVolumesForFunction(mountConfig); + YRLOG_INFO("mountResultStatus: {}", mountResultStatus.ToString()); + EXPECT_EQ(mountResultStatus.StatusCode(), StatusCode::SUCCESS); + + funcMountSFS->set_status("disabled"); + mountResultStatus = mockMounter_->MountVolumesForFunction(mountConfig); + EXPECT_EQ(mountResultStatus.StatusCode(), StatusCode::SUCCESS); + (void)litebus::os::Rmdir("/testMount/testSFS"); +} + +TEST_F(VolumeMountTest, VolumeMountTestExecMountFailed) +{ + messages::FuncMountConfig mountConfig; + messages::FuncMountUser mountUser; + mountUser.set_userid(1004); + mountUser.set_groupid(1004); + mountConfig.mutable_funcmountuser()->CopyFrom(mountUser); + + messages::FuncMount funcMountECS; + funcMountECS.set_mounttype("ecs"); + funcMountECS.set_mountsharepath("10.244.134.153:/home/disk/lzt/ecs-share"); + funcMountECS.set_localmountpath("/testMount/testECS"); + mountConfig.add_funcmounts()->CopyFrom(funcMountECS); + + (void)litebus::os::Rmdir("/testMount/testECS"); + EXPECT_CALL(*mockMounter_, Connect).WillOnce(testing::Return(0)); + EXPECT_CALL(*mockMounter_, ExecMount(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(Status(StatusCode::RUNTIME_MANAGER_MOUNT_VOLUME_FAILED, "mount failed"))); + auto mountResultStatus = mockMounter_->MountVolumesForFunction(mountConfig); + YRLOG_INFO("mountResultStatus: {}", mountResultStatus.ToString()); + EXPECT_EQ(mountResultStatus.StatusCode(), StatusCode::RUNTIME_MANAGER_MOUNT_VOLUME_FAILED); + (void)litebus::os::Rmdir("/testMount/testECS"); +} + +TEST_F(VolumeMountTest, InvalidSharedPath) +{ + messages::FuncMount funcMount; + funcMount.set_mounttype("ecs"); + funcMount.set_mountsharepath("1000.244.134.153:/home/disk/lzt/ecs-share"); + funcMount.set_localmountpath("/testMount/testECS"); + EXPECT_EQ(mounter_->MountVolume(funcMount).StatusCode(), StatusCode::RUNTIME_MANAGER_MOUNT_VOLUME_FAILED); + + funcMount.set_mountsharepath("invalidIP:/home/disk/lzt/ecs-share"); + EXPECT_EQ(mounter_->MountVolume(funcMount).StatusCode(), StatusCode::RUNTIME_MANAGER_MOUNT_VOLUME_FAILED); +} + +/** + * Feature: + * Description: Test ExecMount + * Steps: + * 1. Call ExecMount + * Expectation: + * 1. Receive RUNTIME_MANAGER_MOUNT_VOLUME_FAILED status. + */ +TEST_F(VolumeMountTest, ExecMountTest) +{ + std::string host = "localhost"; + std::string mountSharePath = "/testMount"; + std::string localPath = "/testMount"; + + auto mounter = std::make_shared(); + auto status = mounter->ExecMount(host, mountSharePath, localPath); + EXPECT_EQ(status, StatusCode::RUNTIME_MANAGER_MOUNT_VOLUME_FAILED); +} +} // namespace functionsystem::test diff --git a/functionsystem/tests/unit/runtime_manager/virtual_env_manager/CMakeLists.txt b/functionsystem/tests/unit/runtime_manager/virtual_env_manager/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..13fc4f28f9fa2928094be2f91c873cc140813038 --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/virtual_env_manager/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. + +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} VIRTUAL_ENV_MANAGER_TEST_SRCS) + +target_sources(${UNIT_TEST_MODULE} PRIVATE ${VIRTUAL_ENV_MANAGER_TEST_SRCS}) \ No newline at end of file diff --git a/functionsystem/tests/unit/runtime_manager/virtual_env_manager/virtual_env_manager_actor_test.cpp b/functionsystem/tests/unit/runtime_manager/virtual_env_manager/virtual_env_manager_actor_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..30ecd65c10b581a0f43f376cd7bc870cb54171d3 --- /dev/null +++ b/functionsystem/tests/unit/runtime_manager/virtual_env_manager/virtual_env_manager_actor_test.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "function_agent/agent_service/agent_service_test_actor.h" +#include "manager/runtime_manager.h" +#include "mocks/mock_cmdtool.h" +#include "virtual_env_manager/virtual_env_mgr_actor.h" + +namespace functionsystem::test { +using namespace functionsystem::runtime_manager; +const std::vector envRecycleResult{ "Remove all packages in environment" }; +class VirtualEnvMgrActorTest : public testing::Test { +public: + void SetUp() override + { + actor_ = std::make_shared(RUNTIME_MANAGER_VIRTUAL_ENV_MGR_ACTOR_NAME, 0); + actor_->SetRecycleUnusedEnvsInterval(0); + cmdTool_ = std::make_shared(); + litebus::Spawn(actor_); + } + + void TearDown() override + { + litebus::Terminate(actor_->GetAID()); + litebus::Await(actor_->GetAID()); + } + +protected: + std::shared_ptr actor_; + std::shared_ptr cmdTool_; +}; + +TEST_F(VirtualEnvMgrActorTest, AddEnvReferInfos) +{ + EXPECT_CALL(*cmdTool_.get(), GetCmdResultWithError).WillRepeatedly(testing::Return(envRecycleResult)); + + std::string envName = "env-1"; + std::string runtimeId = "runtimeId-1"; + + actor_->AddEnvReferInfos(envName, runtimeId); + + ASSERT_EQ(actor_->GetEnvReferInfos().size(), 1); + for (const auto &info : actor_->GetEnvReferInfos()) { + ASSERT_EQ(info.first, envName); + ASSERT_TRUE(info.second.runtimeIds.count(runtimeId)); + } +} + +TEST_F(VirtualEnvMgrActorTest, RmEnvReferInfosOnSameEnvs) +{ + EXPECT_CALL(*cmdTool_.get(), GetCmdResultWithError).WillRepeatedly(testing::Return(envRecycleResult)); + + std::string envName1 = "env-1"; + std::string runtimeId1 = "runtimeId-1"; + actor_->AddEnvReferInfos(envName1, runtimeId1); + + std::string runtimeId2 = "runtimeId-2"; + actor_->AddEnvReferInfos(envName1, runtimeId2); + actor_->RmEnvReferInfos(runtimeId1); + + actor_->RecycleUnusedEnvs(); + + ASSERT_EQ(actor_->GetEnvReferInfos().size(), 1); +} +} // namespace functionsystem::test \ No newline at end of file diff --git a/functionsystem/tests/unit/utils/future_test_helper.h b/functionsystem/tests/unit/utils/future_test_helper.h index f188812c5cbd2e4cea3bea198a81801327df9e8c..d5e3566b3bedc7b436f725dce7b6ff9b095f0f5f 100644 --- a/functionsystem/tests/unit/utils/future_test_helper.h +++ b/functionsystem/tests/unit/utils/future_test_helper.h @@ -21,6 +21,7 @@ #include #include "async/future.hpp" +#include "common/utils/actor_worker.h" namespace functionsystem::test { @@ -111,6 +112,26 @@ PromiseArgActionP>> FutureArg(litebus return PromiseArg(promise); } +template +litebus::Future AsyncReturn(T val) +{ + auto promise = std::make_shared>(); + auto actor = std::make_shared(); + (void)actor->AsyncWork([promise, val]() { promise->SetValue(val); }) + .OnComplete([actor](const litebus::Future &) { actor->Terminate(); }); + return promise->GetFuture(); +} + +template +litebus::Future AsyncReturn(litebus::Future val) +{ + auto promise = std::make_shared>(); + auto actor = std::make_shared(); + (void)actor->AsyncWork([promise, val]() { promise->SetValue(val); }) + .OnComplete([actor](const litebus::Future &) { actor->Terminate(); }); + return promise->GetFuture(); +} + } // namespace functionsystem::test #endif // LLT_UTILS_FUTURE_TEST_HELPER_H diff --git a/functionsystem/tests/unit/utils/future_test_helper_example.cpp b/functionsystem/tests/unit/utils/future_test_helper_example.cpp index 2e167abf734c1bd0e2e72552949610907da9725b..67163bab2c4f9ed579c37485e809f2887d8c0ea3 100644 --- a/functionsystem/tests/unit/utils/future_test_helper_example.cpp +++ b/functionsystem/tests/unit/utils/future_test_helper_example.cpp @@ -94,10 +94,10 @@ private: class FutureHelperExample : public ::testing::Test { protected: - static void SetUpTestCase() + [[maybe_unused]] static void SetUpTestSuite() { } - static void TearDownTestCase() + [[maybe_unused]] static void TearDownTestSuite() { } void SetUp() diff --git a/functionsystem/tests/unit/utils/generate_info.h b/functionsystem/tests/unit/utils/generate_info.h index 4ac1103e4e7ebf5e6c542d8a01b21732c3872a2f..9513a8eaa79c8ff814017da6a21011864359ea53 100644 --- a/functionsystem/tests/unit/utils/generate_info.h +++ b/functionsystem/tests/unit/utils/generate_info.h @@ -20,8 +20,8 @@ #include #include -#include "proto/pb/posix_pb.h" -#include "resource_type.h" +#include "common/proto/pb/posix_pb.h" +#include "common/resource_view/resource_type.h" #include "common/types/instance_state.h" #include "common/explorer/explorer.h" diff --git a/functionsystem/tests/unit/utils/grpc_client_helper.h b/functionsystem/tests/unit/utils/grpc_client_helper.h index 6918b646f704595b73ae166f6803638226bb4140..11c53dc7e6c7ccdeaccee7fa804dff3151d5e995 100644 --- a/functionsystem/tests/unit/utils/grpc_client_helper.h +++ b/functionsystem/tests/unit/utils/grpc_client_helper.h @@ -17,7 +17,7 @@ #ifndef TEST_UNIT_UTILS_GRPC_CLIENT_HELPER_H #define TEST_UNIT_UTILS_GRPC_CLIENT_HELPER_H -#include "rpc/client/grpc_client.h" +#include "common/rpc/client/grpc_client.h" #include "etcd/api/etcdserverpb/rpc.grpc.pb.h" #include "etcd/server/etcdserver/api/v3election/v3electionpb/v3election.grpc.pb.h"