From 793e7d79d76a068dba28931a77a8ae30080cb396 Mon Sep 17 00:00:00 2001 From: dingjiahuichina Date: Fri, 18 Jul 2025 17:16:27 +0800 Subject: [PATCH] feat: init basic modules of backend --- backend/artifacts/__init__.py | 0 backend/artifacts/apps.py | 6 + backend/artifacts/migrations/__init__.py | 0 backend/artifacts/models.py | 96 ++++ backend/artifacts/serializers.py | 167 ++++++ backend/artifacts/tasks/__init__.py | 0 backend/artifacts/tasks/install_mcp_task.py | 32 ++ backend/artifacts/tasks/uninstall_mcp_task.py | 13 + backend/artifacts/views.py | 504 ++++++++++++++++++ backend/constants/__init__.py | 0 backend/constants/choices.py | 19 + backend/constants/configs/__init__.py | 0 backend/constants/configs/mariadb_config.py | 71 +++ .../configs/task_scheduler_config.py | 34 ++ backend/constants/paths.py | 37 ++ 15 files changed, 979 insertions(+) create mode 100644 backend/artifacts/__init__.py create mode 100644 backend/artifacts/apps.py create mode 100644 backend/artifacts/migrations/__init__.py create mode 100644 backend/artifacts/models.py create mode 100644 backend/artifacts/serializers.py create mode 100644 backend/artifacts/tasks/__init__.py create mode 100644 backend/artifacts/tasks/install_mcp_task.py create mode 100644 backend/artifacts/tasks/uninstall_mcp_task.py create mode 100644 backend/artifacts/views.py create mode 100644 backend/constants/__init__.py create mode 100644 backend/constants/choices.py create mode 100644 backend/constants/configs/__init__.py create mode 100644 backend/constants/configs/mariadb_config.py create mode 100644 backend/constants/configs/task_scheduler_config.py create mode 100644 backend/constants/paths.py diff --git a/backend/artifacts/__init__.py b/backend/artifacts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/artifacts/apps.py b/backend/artifacts/apps.py new file mode 100644 index 0000000..0951455 --- /dev/null +++ b/backend/artifacts/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ArtifactsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'artifacts' diff --git a/backend/artifacts/migrations/__init__.py b/backend/artifacts/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/artifacts/models.py b/backend/artifacts/models.py new file mode 100644 index 0000000..2450ca3 --- /dev/null +++ b/backend/artifacts/models.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +from django.db import models +from django.db.models import UniqueConstraint + + +class Plugin(models.Model): + + class Meta: + ordering = ["-updated_at"] + constraints = [ + UniqueConstraint(fields=['name', 'version'], name='unique_plugin_name_version') + ] + + class Type(models.TextChoices): + APP = "app", "application" + + def __str__(self): + return self.name + + name = models.CharField( + "插件名称", + max_length=1024, + help_text="允许插件名称中带有版本号，表示软件本身的版本，而非插件的版本。" + ) + version = models.CharField( + "插件版本", + max_length=256, + help_text="注意该版本为插件版本，非部署软件版本" + ) + updated_at = models.DateTimeField("插件更新时间") + description = models.CharField("插件描述", max_length=2048) + type = models.CharField( + "插件类型", + max_length=16, + choices=Type.choices, + default=Type.APP, + help_text="保留字段，暂不生效" + ) + sha256sum = models.CharField( + "插件校验码", + max_length=1024, + help_text="插件 sha256sum 校验码" + ) + size = models.PositiveBigIntegerField( + "插件大小", + help_text="插件大小，单位为 Bytes" + ) + author = models.CharField("插件作者", max_length=256, blank=True, null=True) + can_be_deployed_local = models.BooleanField( + "是否支持本地单节点部署", + default=False, + help_text="该插件是否支持单节点部署以及是否支持和 oeDeploy 部署在同一节点" + ) + repo = models.CharField("插件代码仓库链接", max_length=2048, blank=True, null=True) + readme = models.CharField("README 文件链接", max_length=2048, blank=True, null=True) + icon = models.CharField("插件图标链接", max_length=2048, blank=True, null=True) + download_url = models.CharField("插件下载链接", max_length=2048, blank=True, null=True) + + +class MCPService(models.Model): + + class Meta: + ordering = ["-updated_at"] + constraints = [ + UniqueConstraint(fields=['name', 'version'], name='unique_mcp_name_version') + ] + + def __str__(self): + return self.name + + name = models.CharField("MCP 服务名称", max_length=1024) + package_name = models.CharField("MCP 服务软件包名称", max_length=1024) + version = models.CharField("MCP 服务软件包版本", max_length=256) + updated_at = models.DateTimeField('MCP 服务更新时间') + author = models.CharField("MCP 服务发布者", max_length=256, blank=True, null=True) + description = models.CharField("MCP 服务描述", max_length=2048) + size = models.PositiveBigIntegerField( + "MCP 服务包大小", + help_text="MCP 服务包大小，单位为 Bytes" + ) + repo = models.CharField("MCP 服务代码仓库链接", max_length=2048, blank=True, null=True) + readme = models.CharField("MCP 服务 README 文件链接", max_length=2048, blank=True, null=True) + icon = models.CharField("MCP 服务图标链接", max_length=2048) diff --git a/backend/artifacts/serializers.py b/backend/artifacts/serializers.py new file mode 100644 index 0000000..eef8e1c --- /dev/null +++ b/backend/artifacts/serializers.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +import os +import re + +from rest_framework import serializers + +from artifacts.models import MCPService, Plugin +from constants.choices import ArtifactTag +from constants.paths import PLUGIN_CACHE_DIR +from tasks.models import Task +from utils.cmd_executor import CommandExecutor +from utils.common import is_process_running +from utils.logger import init_log + +logger = init_log('run.log') + + +class ArtifactSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + version = serializers.CharField() + author = serializers.CharField() + description = serializers.CharField() + icon = serializers.CharField() + updated_at = serializers.DateTimeField() + tag = serializers.SerializerMethodField() + + @staticmethod + def get_tag(obj): + if isinstance(obj, MCPService): + return ArtifactTag.MCP + elif isinstance(obj, Plugin): + return ArtifactTag.OEDP + else: + return '' + + +class MCPDetailSerializer(serializers.ModelSerializer): + tag = serializers.SerializerMethodField() + installed_status = serializers.SerializerMethodField() + + class Meta: + model = MCPService + fields = ( + 'id', + 'name', + 'version', + 'description', + 'readme', + 'tag', + 'installed_status', + ) + + @staticmethod + def get_tag(obj): + return ArtifactTag.MCP + + @staticmethod + def get_installed_status(obj): + if is_process_running(f'yum install -y {obj.package_name}'): + return Task.Status.IN_PROCESS + cmd = ['rpm', '-q', obj.package_name] + cmd_executor = CommandExecutor(cmd) + _, _, code = cmd_executor.run() + if code != 0: + return Task.Status.NOT_YET + return Task.Status.SUCCESS + + +class PluginDetailSerializer(serializers.ModelSerializer): + tag = serializers.SerializerMethodField() + download_status = serializers.SerializerMethodField() + + class Meta: + model = Plugin + fields = ( + 'name', + 'version', + 'description', + 'readme', + 'tag', + 'download_status', + ) + + @staticmethod + def get_tag(obj): + return ArtifactTag.OEDP + + @staticmethod + def get_download_status(obj): + if is_process_running(f'oedp init {obj.name}'): + return Task.Status.IN_PROCESS + if os.path.exists(os.path.join(PLUGIN_CACHE_DIR, obj.name)): + return Task.Status.SUCCESS + return Task.Status.NOT_YET + + +class PluginListSerializer(serializers.ListSerializer): + + def create(self, validated_data): + plugins = [Plugin(**item) for item in validated_data] + Plugin.objects.bulk_create(plugins) + return plugins + + +class PluginBulkCreateSerializer(serializers.ModelSerializer): + + class Meta: + model = Plugin + fields = ( + 'name', + 'version', + 'updated_at', + 'description', + 'type', + 'sha256sum', + 'size', + 'icon', + 'download_url', + ) + list_serializer_class = PluginListSerializer + + @staticmethod + def validate_sha256sum(value): + pattern = r'^[a-fA-F0-9]{64}$' + if not re.fullmatch(pattern, value.strip()): + msg = 'Invalid sha256sum checksum format.' + logger.error(msg) + raise serializers.ValidationError(msg) + return value + + +class MCPListSerializer(serializers.ListSerializer): + + def create(self, validated_data): + mcps = [MCPService(**item) for item in validated_data] + MCPService.objects.bulk_create(mcps) + return mcps + + +class MCPBulkCreateSerializer(serializers.ModelSerializer): + + class Meta: + model = MCPService + fields = ( + 'name', + 'package_name', + 'version', + 'updated_at', + 'description', + 'size', + 'repo', + ) + list_serializer_class = MCPListSerializer diff --git a/backend/artifacts/tasks/__init__.py b/backend/artifacts/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/artifacts/tasks/install_mcp_task.py b/backend/artifacts/tasks/install_mcp_task.py new file mode 100644 index 0000000..e13fadb --- /dev/null +++ b/backend/artifacts/tasks/install_mcp_task.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +from tasks.base_task import BaseTask, TaskExecuteError +from tasks.models import Task +from utils.cmd_executor import CommandExecutor + + +class InstallMCPTask(BaseTask): + + def __init__(self, pkg_name, **kwargs): + super().__init__(task_type=Task.Type.INSTALL_MCP, **kwargs) + self.pkg_name = pkg_name + + def run(self): + cmd = ['yum', 'install', '-y', self.pkg_name] + cmd_executor = CommandExecutor(cmd) + _, stderr, code = cmd_executor.run() + if code != 0: + raise TaskExecuteError(stderr) + return f"Install {self.pkg_name} successfully." diff --git a/backend/artifacts/tasks/uninstall_mcp_task.py b/backend/artifacts/tasks/uninstall_mcp_task.py new file mode 100644 index 0000000..6aae260 --- /dev/null +++ b/backend/artifacts/tasks/uninstall_mcp_task.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== diff --git a/backend/artifacts/views.py b/backend/artifacts/views.py new file mode 100644 index 0000000..ea8711d --- /dev/null +++ b/backend/artifacts/views.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +import glob +import gzip +import os.path +import shutil +from pathlib import Path +from xml.etree import ElementTree + +import yaml +from django.db import connection +from rest_framework import viewsets, status +from rest_framework.decorators import action +from rest_framework.response import Response + +from artifacts.models import MCPService, Plugin +from artifacts.serializers import ( + ArtifactSerializer, + PluginBulkCreateSerializer, + MCPBulkCreateSerializer, + MCPDetailSerializer, + PluginDetailSerializer +) +from artifacts.tasks.install_mcp_task import InstallMCPTask +from constants.choices import ArtifactTag +from constants.paths import PLUGIN_REPO_DIR, PLUGIN_CACHE_DIR +from tasks.models import Task +from tasks.scheduler import scheduler, check_scheduler_load +from utils.cmd_executor import CommandExecutor +from utils.file_handler.base_handler import FileError +from utils.file_handler.yaml_handler import YAMLHandler +from utils.logger import init_log +from utils.time import timestamp2local + +logger = init_log('run.log') + + +class ArtifactViewSet(viewsets.GenericViewSet): + + @staticmethod + def _update_plugin_info(): + # 检查 oedp 命令是否可用 + logger.info("Start to check if the oedp is installed.") + cmd_executor = CommandExecutor(['rpm', '-q', 'oedp']) + _, _, return_code = cmd_executor.run() + if return_code != 0: + msg = "The oedp is not installed." + logger.error(msg) + return False, msg + logger.info("The oedp has already been installed.") + + # 执行 oedp repo update + cmd = ['oedp', 'repo', 'update'] + logger.info(f'Start to execute command [{" ".join(cmd)}].') + cmd_executor = CommandExecutor(cmd) + _, stderr, return_code = cmd_executor.run() + if return_code != 0: + msg = f"Failed to execute command: [{' '.join(cmd)}]. Error: {stderr}" + logger.error(msg) + return False, msg + + msg = 'Update plugin repo successfully.' + logger.info(msg) + return True, msg + + @staticmethod + def _read_plugin_info(): + # 获取记录插件信息的 YAML 文件 + logger.info("Start to get YAML files.") + plugin_repo_dir = Path(PLUGIN_REPO_DIR) + if not plugin_repo_dir.exists(): + msg = f'Path {plugin_repo_dir} not exists.' + logger.error(msg) + return [], msg + if not plugin_repo_dir.is_dir(): + msg = f'Path {plugin_repo_dir} is not a directory.' + logger.error(msg) + return [], msg + plugin_yaml_list = [str(file) for file in plugin_repo_dir.iterdir() + if file.is_file() and (file.suffix == '.yaml' or file.suffix == '.yml')] + logger.info(f"The plugin meta files: {plugin_yaml_list}") + + # 读取 YAML 文件, 生成插件信息列表 + logger.info("Start to read YAML files and generate plugin data.") + plugin_data = [] + for plugin_yaml in plugin_yaml_list: + try: + yaml_handler = YAMLHandler(file_path=plugin_yaml, logger=logger) + except (FileError, yaml.YAMLError) as ex: + return [], str(ex) + for multi_version_plugins in yaml_handler.data.get('plugins'): + for plugin_info in list(multi_version_plugins.values())[0]: + plugin_info['updated_at'] = plugin_info.pop('updated') + plugin_info['download_url'] = " ".join(plugin_info.pop('urls')) + plugin_data.append(plugin_info) + + msg = 'Generate plugin data successfully.' + logger.info(msg) + return plugin_data, msg + + @staticmethod + def _update_mcp_info(): + cmd = ['yum', 'makecache', '--disablerepo=*', '--enablerepo=mcp'] + logger.info(f"Start to execute command [{' '.join(cmd)}].") + cmd_executor = CommandExecutor(cmd) + _, stderr, code = cmd_executor.run() + if code != 0: + msg = f"Failed to execute command: [{' '.join(cmd)}]. Error: {stderr}" + logger.error(msg) + return False, msg + + msg = 'Update MCP repo successfully.' + logger.info(msg) + return True, msg + + @staticmethod + def _read_mcp_info(): + # 匹配 primary.xml.gz 文件 + logger.info('Start to match MCP meta file.') + pattern = "/var/cache/dnf/mcp-*/repodata/*-primary.xml.gz" + matches = glob.glob(pattern) + if not matches: + msg = "No match for *-primary.xml.gz." + logger.error(msg) + return [], msg + primary_file = matches[0] + logger.info(f"The MCP meta file: {primary_file}") + + # 开始解压 primary.xml.gz + logger.info(f'Start to extract {primary_file}') + output_file = primary_file.rstrip('.gz') + try: + with gzip.open(primary_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + logger.info(f"extract successfully: {primary_file} -> {output_file}") + except PermissionError: + msg = "No permission." + logger.error(msg) + return [], msg + except gzip.BadGzipFile: + msg = f"{primary_file} is not a valid gzip format." + logger.error(msg) + return [], msg + + # 读取 primary.xml 文件，生成 MCP 服务信息列表 + logger.info("Start to read MCP information files and generate MCP data.") + tree = ElementTree.parse(output_file) + root = tree.getroot() + namespace = {'common': 'http://linux.duke.edu/metadata/common'} + mcp_data = [] + for package in root.findall('common:package', namespace): + mcp_info = {} + package_name = package.find('common:name', namespace).text.strip() + if package_name == 'mcp-servers': + continue + else: + mcp_info['package_name'] = package_name + mcp_info['name'] = package_name.removeprefix('mcp-servers-') + version = package.find('common:version', namespace) + mcp_info['version'] = f"{version.get('epoch')}:{version.get('ver')}-{version.get('rel')}" + timestamp = package.find('common:time', namespace).get('file') + mcp_info['updated_at'] = timestamp2local(int(timestamp)) + mcp_info['description'] = package.find('common:description', namespace).text + mcp_info['size'] = int(package.find('common:size', namespace).get('package')) + mcp_info['repo'] = package.find('common:url', namespace).text + mcp_data.append(mcp_info) + + if not mcp_data: + msg = f"Failed to read mcp information." + logger.error(msg) + return [], msg + + msg = 'Generate MCP service data successfully.' + logger.info(msg) + return mcp_data, msg + + @staticmethod + def _clear_table(table_name): + """ + 清空指定数据库表并重置自增主键 + """ + logger.info(f"Start to clear table '{table_name}'") + with connection.cursor() as cursor: + cursor.execute(f"TRUNCATE TABLE {table_name}") + + def list(self, request): + logger.info("==== API: [GET] /v1.0/artifacts/ ====") + tag = request.query_params.get('tag') + if tag == ArtifactTag.OEDP: + queryset = Plugin.objects.all() + elif tag == ArtifactTag.MCP: + queryset = MCPService.objects.all() + else: + msg = 'The query parameter [tag] is missing, or the value of the query parameter [tag] is invalid.' + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + queryset = self.paginate_queryset(queryset) + serializer = ArtifactSerializer(queryset, many=True) + logger.info("Get list information successfully.") + return self.get_paginated_response(serializer.data) + + @staticmethod + def retrieve(request, pk): + # TODO 添加任务状态判断逻辑 + logger.info(f"==== API: [GET] /v1.0/artifacts/{pk}/ ====") + tag = request.query_params.get('tag') + if tag == ArtifactTag.MCP: + try: + mcp_service = MCPService.objects.get(id=pk) + except MCPService.DoesNotExist: + msg = f"The MCP service with ID {pk} does not exist." + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + serializer = MCPDetailSerializer(mcp_service) + elif tag == ArtifactTag.OEDP: + try: + plugin = Plugin.objects.get(id=pk) + except Plugin.DoesNotExist: + msg = f"The plugin with ID {pk} does not exist." + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + serializer = PluginDetailSerializer(plugin) + else: + msg = 'The query parameter [tag] is missing, or the value of the query parameter [tag] is invalid.' + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + msg = 'Get detail successfully.' + logger.info(msg) + return Response({ + 'is_success': True, + 'message': msg, + 'data': serializer.data + }, status=status.HTTP_200_OK) + + @action(methods=['GET'], detail=False) + def get_task_info(self, request): + logger.info(f'==== API: [GET] /v1.0/artifacts/get_task_info/ ====') + task_name = request.query_params.get('task_name') + try: + task = Task.objects.get(name=task_name) + except Task.DoesNotExist: + msg = f"Task {task_name} not found." + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + return Response({ + 'is_success': True, + 'message': "Checking task status successfully.", + 'data': { + 'name': task.name, + 'type': task.type, + 'status': task.status, + 'msg': task.msg + } + }, status=status.HTTP_200_OK) + + @action(methods=['GET'], detail=True) + @check_scheduler_load + def install_mcp(self, request, pk): + # TODO 同一时间只能安装一个 mcp-servers-xxx 的包 + # TODO 无法安装正在卸载的包 + logger.info(f'==== API: [GET] /v1.0/artifacts/{pk}/install_mcp/ ====') + # 查询 MCP 服务包信息 + logger.info("Start query MCP service package information by id.") + try: + mcp_service = MCPService.objects.get(id=pk) + except MCPService.DoesNotExist: + msg = f"The MCP service with ID {pk} does not exist." + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status.HTTP_400_BAD_REQUEST) + logger.info("Query MCP service package successfully.") + + # 检查 MCP 服务包是否已经安装，后端进行二次校验 + pkg_name = mcp_service.package_name + logger.info(f"Start to check whether package {pkg_name} is installed.") + cmd = ['rpm', '-q', pkg_name] + cmd_executor = CommandExecutor(cmd) + _, _, code = cmd_executor.run() + if code == 0: + msg = f"{pkg_name} has been installed." + logger.info(msg) + return Response({ + 'is_success': True, + 'message': msg + }, status=status.HTTP_200_OK) + + # 安装 MCP 服务包 + logger.info(f"Start to install package {pkg_name}") + install_mcp_task = InstallMCPTask(pkg_name, name=f"install_{pkg_name}_task") + scheduler.add_task(install_mcp_task) + return Response({ + 'is_success': True, + 'message': f"Package {pkg_name} is being installed.", + 'task_name': install_mcp_task.name + }, status=status.HTTP_202_ACCEPTED) + + @action(methods=['GET'], detail=True) + def uninstall_mcp(self, request, pk): + # TODO 无法卸载正在安装的包 + logger.info(f"==== API: [GET] /v1.0/artifacts/{pk}/uninstall_mcp/ ====") + # 查询 MCP 服务包信息 + logger.info("Start query MCP service package information by id.") + try: + mcp_service = MCPService.objects.get(id=pk) + except MCPService.DoesNotExist: + msg = f"The MCP service with ID {pk} does not exist." + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status.HTTP_400_BAD_REQUEST) + logger.info("Query MCP service package successfully.") + + # 检查 MCP 服务包是否已经安装，后端进行二次校验 + pkg_name = mcp_service.package_name + logger.info(f"Start to check whether package {pkg_name} is installed.") + cmd = ['rpm', '-q', pkg_name] + cmd_executor = CommandExecutor(cmd) + _, _, code = cmd_executor.run() + if code != 0: + msg = f"{pkg_name} isn't installed." + logger.info(msg) + return Response({ + 'is_success': True, + 'message': msg + }, status=status.HTTP_200_OK) + + # 卸载 MCP 服务包 + logger.info(f"Start to uninstall package {pkg_name}") + cmd = ['yum', 'remove', '-y', pkg_name] + cmd_executor = CommandExecutor(cmd) + _, stderr, code = cmd_executor.run() + if code != 0: + logger.error(f"Failed to uninstall {pkg_name}, error: {stderr}") + return Response({ + 'is_success': False, + 'message': stderr + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + msg = f"Package {pkg_name} is uninstalled successfully." + logger.info(msg) + return Response({ + 'is_success': True, + 'message': msg + }, status=status.HTTP_200_OK) + + @action(methods=['GET'], detail=True) + def add_to_agent_app(self, request, pk): + pass + + @action(methods=['GET'], detail=True) + def download_plugin(self, request, pk): + logger.info(f"==== API: [GET] /v1.0/artifacts/{pk}/download_plugin/ ====") + # 查询插件信息 + logger.info("Start query plugin package information by id.") + try: + plugin = Plugin.objects.get(id=pk) + except Plugin.DoesNotExist: + msg = f"The plugin with ID {pk} does not exist." + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status.HTTP_400_BAD_REQUEST) + logger.info("Query plugin package information successfully.") + + # 检查本地插件是否已经存在 + logger.info(f"Start to check whether plugin {plugin.name} already exists.") + if os.path.exists(os.path.join(PLUGIN_CACHE_DIR, plugin.name)): + msg = f"Plugin {plugin.name} already exists." + logger.info(msg) + return Response({ + 'is_success': True, + "message": msg + }, status=status.HTTP_200_OK) + + # 下载插件 + logger.info(f"Start to download plugin {plugin.name}.") + if not os.path.exists(PLUGIN_CACHE_DIR): + os.makedirs(PLUGIN_CACHE_DIR) + logger.info(f"Create directory: {PLUGIN_CACHE_DIR}.") + cmd = ['oedp', 'init', plugin.name, '-d', PLUGIN_CACHE_DIR] + cmd_executor = CommandExecutor(cmd) + _, stderr, code = cmd_executor.run() + if code != 0: + logger.error(f"Failed to download plugin {plugin.name}, error: {stderr}") + return Response({ + 'is_success': False, + "message": stderr + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + msg = f"Download plugin {plugin.name} successfully." + logger.info(msg) + return Response({ + 'is_success': True, + 'message': msg + }, status=status.HTTP_200_OK) + + @action(methods=['GET'], detail=False) + def sync(self, request): + # TODO 同步加锁，禁用黑名单接口 + logger.info("==== API: [GET] /v1.0/artifacts/sync/ ====") + # 更新插件信息 + update_result, msg = self._update_plugin_info() + if not update_result: + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # 读取插件的信息 + plugin_data, msg = self._read_plugin_info() + if not plugin_data: + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # 将插件的信息存入数据库中 + serializer = PluginBulkCreateSerializer(data=plugin_data, many=True) + self._clear_table(Plugin._meta.db_table) + if not serializer.is_valid(): + logger.error(f"Failed to validate plugin data, errors: {serializer.errors}") + return Response({ + 'is_success': False, + 'errors': serializer.errors + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + plugins = serializer.save() + logger.info("Store plugin data to database successfully.") + + # 更新 MCP 服务的信息 + update_result, msg = self._update_mcp_info() + if not update_result: + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # 读取 MCP 服务的信息 + mcp_data, msg = self._read_mcp_info() + if not mcp_data: + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # 将 MCP 服务的信息存入数据库中 + serializer = MCPBulkCreateSerializer(data=mcp_data, many=True) + self._clear_table(MCPService._meta.db_table) + if not serializer.is_valid(): + logger.error(f"Failed to validate MCP data, errors: {serializer.errors}") + return Response({ + 'is_success': False, + 'errors': serializer.errors + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + mcps = serializer.save() + logger.info("Store MCP data to database successfully.") + + # 根据 tag 返回 OEDP 插件分页信息或 MCP 服务分页信息 + tag = request.query_params.get('tag') if request.query_params.get('tag') else ArtifactTag.OEDP + if tag == ArtifactTag.OEDP: + plugin_ids = [plugin.id for plugin in plugins] + queryset = Plugin.objects.filter(id__in=plugin_ids) + elif tag == ArtifactTag.MCP: + mcp_ids = [mcp.id for mcp in mcps] + queryset = MCPService.objects.filter(id__in=mcp_ids) + else: + msg = f'Invalid value of request parameter "tag", the values: {tag}' + logger.error(msg) + return Response({ + 'is_success': False, + 'message': msg + }, status=status.HTTP_400_BAD_REQUEST) + + plugin_queryset = self.paginate_queryset(queryset) + artifact_serializer = ArtifactSerializer(plugin_queryset, many=True) + return self.get_paginated_response(artifact_serializer.data) diff --git a/backend/constants/__init__.py b/backend/constants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/constants/choices.py b/backend/constants/choices.py new file mode 100644 index 0000000..1d4ba75 --- /dev/null +++ b/backend/constants/choices.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +class ArtifactTag: + # 表示 MCP 服务包 + MCP = 'mcp' + # 表示 oedp 插件包 + OEDP = 'oedp' diff --git a/backend/constants/configs/__init__.py b/backend/constants/configs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/constants/configs/mariadb_config.py b/backend/constants/configs/mariadb_config.py new file mode 100644 index 0000000..90444a9 --- /dev/null +++ b/backend/constants/configs/mariadb_config.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +import json +from configparser import MissingSectionHeaderError, ParsingError + +from constants.paths import MARIADB_CONFIG_FILE, MARIADB_JSON_FILE +from utils.cipher import CustomCipher +from utils.file_handler.base_handler import FileError +from utils.file_handler.conf_handler import ConfHandler +from utils.logger import init_log + +__all__ = ['MariaDBConfig', 'get_settings_mariadb_config'] +run_logger = init_log("run.log") + + +class MariaDBConfig: + NAME = '' + HOST = '' + PORT = '' + USER = '' + PASSWORD = '' + + +try: + conf_handler = ConfHandler(file_path=MARIADB_CONFIG_FILE, logger=run_logger) +except (FileError, MissingSectionHeaderError, ParsingError): + pass +else: + MariaDBConfig.NAME = conf_handler.get('mariadb', 'name', default='') + MariaDBConfig.HOST = conf_handler.get('mariadb', 'host', default='') + MariaDBConfig.PORT = conf_handler.get('mariadb', 'port', default='') + MariaDBConfig.USER = conf_handler.get('mariadb', 'user', default='') + MariaDBConfig.PASSWORD = conf_handler.get('mariadb', 'password', default='') + + +def get_settings_mariadb_config(): + with open(MARIADB_JSON_FILE, mode='r') as fr_handle: + ciphertext_data = json.load(fr_handle) + custom_cipher = CustomCipher() + plaintext = custom_cipher.decrypt_ciphertext_data(ciphertext_data) + database_config = { + 'NAME': MariaDBConfig.NAME, + 'HOST': MariaDBConfig.HOST, + 'PORT': MariaDBConfig.PORT, + 'USER': MariaDBConfig.USER, + 'PASSWORD': plaintext, + 'ENGINE': 'django.db.backends.mysql', + 'OPTIONS': { + 'init_command': 'SET sql_mode="STRICT_TRANS_TABLES"', + 'charset': 'utf8', + 'autocommit': True + }, + 'TEST': { + 'CHARSET': 'utf8', + 'COLLATION': 'utf8_bin' + } + } + del plaintext + return database_config diff --git a/backend/constants/configs/task_scheduler_config.py b/backend/constants/configs/task_scheduler_config.py new file mode 100644 index 0000000..496804f --- /dev/null +++ b/backend/constants/configs/task_scheduler_config.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +from configparser import MissingSectionHeaderError, ParsingError + +from constants.paths import TASK_SCHEDULER_CONFIG_FILE +from utils.file_handler.base_handler import FileError +from utils.file_handler.conf_handler import ConfHandler +from utils.logger import init_log + +run_logger = init_log("run.log") + +# 默认值 +MAX_CONCURRENCY = 10 +try: + conf_handler = ConfHandler(file_path=TASK_SCHEDULER_CONFIG_FILE, logger=run_logger) +except (FileError, MissingSectionHeaderError, ParsingError): + pass +else: + try: + MAX_CONCURRENCY = conf_handler.getint('scheduler', 'max_concurrency', default=10) + except ValueError as ex: + run_logger.warning(f"Failed to get value of max_concurrency, error: {ex}") diff --git a/backend/constants/paths.py b/backend/constants/paths.py new file mode 100644 index 0000000..cc9d6b2 --- /dev/null +++ b/backend/constants/paths.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Huawei Technologies Co., Ltd. +# oeDeploy is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Create: 2025-07-18 +# ====================================================================================================================== + +import os + +# 配置文件目录 +CONFIG_DIR = '/etc/dev-store' +# mariadb.conf 配置文件路径 +MARIADB_CONFIG_FILE = os.path.join(CONFIG_DIR, 'mariadb', 'mariadb.conf') +# MariaDB 密文数据 json 文件 +MARIADB_JSON_FILE = os.path.join(CONFIG_DIR, 'mariadb', 'mariadb_ciphertext_data.json') +# task_scheduler.conf 配置文件路径 +TASK_SCHEDULER_CONFIG_FILE = os.path.join(CONFIG_DIR, 'task_scheduler.conf') + +# 日志目录 +LOG_DIR = '/var/log/dev-store' + +# 插件 repo 缓存目录 +PLUGIN_REPO_DIR = '/etc/oedp/config/repo/cache' +# MCP 服务 repo 文件 +MCP_REPO_FILE = '/etc/yum.repos.d/mcp.repo' +# 家目录 +HOME_DIR = os.path.expanduser('~') +# 插件包缓存目录 +PLUGIN_CACHE_DIR = os.path.join(HOME_DIR, '.oedp') + -- Gitee