From 0af65016ad6ac02b381c8471c68148438786c40a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 13 May 2025 11:17:47 +0800 Subject: [PATCH 01/39] bisect-task: add simplified Manticore client implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor: separate Manticore HTTP and SQL ports for clarity feat: add ManticoreClient import to bisect-task.py feat: implement SHA256-based task ID generation fix: import randint for unique ID generation refactor: replace SQL with ManticoreClient for task insertion feat: add graceful shutdown and cleanup for bisect tasks refactor: 简化bisect任务处理逻辑并移除冗余代码 refactor: 重构 bisect 任务处理逻辑，使用 ManticoreClient 进行写操作 --- container/bisect/bisect-task.py | 301 +++++++++++++++++++------------- lib/bisect_database.py | 13 +- lib/manticore_simple.py | 53 ++++++ sbin/manti-table-bisect.sql | 7 +- 4 files changed, 249 insertions(+), 125 deletions(-) create mode 100644 lib/manticore_simple.py diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 4fe21017..b73c9838 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -26,6 +26,8 @@ import time import threading import traceback import mysql.connector +import hashlib +import signal from random import randint from flask import Flask, jsonify, request from flask.views import MethodView @@ -38,14 +40,100 @@ from py_bisect import GitBisect from log_config import logger sys.path.append((os.environ['CCI_SRC']) + '/lib') from bisect_database import BisectDB +from manticore_simple import ManticoreClient app = Flask(__name__) +def generate_task_id(bad_job_id: str, error_id: str) -> int: + """ + 生成符合 Manticore 要求的 63 位正整数 ID + 算法：SHA256(业务ID+时间戳)[前16位] → 十六进制转十进制 → 掩码保留63位 + + Args: + bad_job_id: 故障任务ID + error_id: 错误标识符 + + Returns: + 63位正整数 (JavaScript 安全整数范围) + """ + # 1. 构造唯一字符串 + timestamp = f"{time.time():.6f}" # 微秒级时间戳 + unique_str = f"{bad_job_id}|{error_id}|{timestamp}" + + # 2. 计算 SHA256 哈希 + hash_bytes = hashlib.sha256(unique_str.encode()).digest() + + # 3. 取前8字节（64位）并转换为整数 + hash_int = int.from_bytes(hash_bytes[:8], byteorder='big') + + # 4. 确保最高位为0，得到63位正整数 + return hash_int & 0x7FFFFFFFFFFFFFFF + class BisectTask: + def _register_signal_handlers(self): + """注册信号处理""" + signal.signal(signal.SIGINT, self._handle_exit_signal) + signal.signal(signal.SIGTERM, self._handle_exit_signal) + logger.info("已注册退出信号处理器") + + def _handle_exit_signal(self, signum, frame): + """信号处理回调""" + logger.warning(f"收到终止信号 {signum}，开始清理...") + self.running = False + self._cleanup_interrupted_tasks() + sys.exit(1) + + def _cleanup_interrupted_tasks(self): + """清理被中断的任务""" + try: + # 1. 获取所有 processing 状态的任务 + tasks = self.bisect_db.execute_query( + "SELECT id FROM bisect " + "WHERE bisect_status = 'processing'" + ) + + if tasks: + logger.info(f"发现 {len(tasks)} 个需要清理的任务") + + # 2. 更新状态为 wait + update_sql = """ + UPDATE bisect + SET bisect_status = 'wait' + WHERE bisect_status = 'processing' + """ + self.bisect_db.execute_update(update_sql) + logger.info(f"已重置 {len(tasks)} 个任务状态") + + # 3. 删除数据目录 + for task in tasks: + result_root = os.path.abspath(os.path.join('bisect_results', task['id'])) + if os.path.exists(result_root): + try: + shutil.rmtree(result_root) + logger.info(f"成功删除数据目录: {result_root}") + except Exception as e: + logger.error(f"删除目录失败 {result_root}: {str(e)}") + + except Exception as e: + logger.error(f"清理过程中发生错误: {str(e)}") + logger.error(traceback.format_exc()) + finally: + logger.info("资源清理完成") + def __init__(self): self.bisect_task = None + self.running = True # 运行状态标志 + + # Initialize ManticoreClient for HTTP API operations + self.client = ManticoreClient( + host=os.environ.get('MANTICORE_HOST', 'localhost'), + port=int(os.environ.get('MANTICORE_WRITE_PORT', '9308')) + ) + + self._register_signal_handlers() # 注册信号处理器 + # Initialize read-only jobs database connection self.jobs_db = BisectDB( host=os.environ.get('MANTICORE_HOST', 'localhost'), @@ -68,7 +156,7 @@ class BisectTask: port=os.environ.get('MANTICORE_PORT', '9306'), database="regression", pool_size=5 - ) + ) self._start_monitor() @@ -80,41 +168,15 @@ class BisectTask: raise ValueError(f"Missing required field: {field}") try: - # Generate unique identifier based on bad_job_id and error_id - task_fingerprint = int(time.time() * 1e6) + randint(0, 999) - time.sleep(5) - if self.bisect_db.execute_query(f"""SELECT id FROM bisect WHERE error_id='{task['error_id']}' AND bad_job_id='{task['bad_job_id']}'"""): - logger.info(f"Already have bisect job {task} in database") + # 生成基于业务数据的强唯一ID + task_fingerprint = generate_task_id( + task["bad_job_id"], + task["error_id"] + ) + # 原子插入操作 insert(self, index: str, id: int, document: dict) + if self.client.insert(index="bisect", id=task_fingerprint, document=task): return True - # Set priority (ensure integer type) - #priority = self.set_priority_level(job_info) - - # Atomic insert operation - insert_sql = f""" - INSERT INTO bisect - (id, bad_job_id, error_id, bisect_status) - VALUES ('{task_fingerprint}', '{task.get("bad_job_id", "")}', '{task.get("error_id", "")}', 'wait') - """ - # Execute write and get affected rows - affected_rows = self.bisect_db.execute_write(insert_sql) - - # Determine result based on affected rows - if affected_rows == 1: - logger.info(f"Successfully inserted new task | ID: {task_fingerprint}") - return True - elif affected_rows == 0: - logger.debug(f"Task already exists | ID: {task_fingerprint}") - return False - else: - logger.warning(f"Unexpected affected rows: {affected_rows}") - return False - - except mysql.connector.Error as e: - if e.errno == 1062: # Duplicate entry error code - logger.debug(f"Task duplicate (caught at database level) | ID: {task_fingerprint}") - return False else: - logger.error(f"Database error | Code: {e.errno} | Message: {e.msg}") return False except Exception as e: logger.error(f"Unknown error: {str(e)}") @@ -162,7 +224,10 @@ class BisectTask: """Producer function optimized for batch processing and rate limiting""" error_count = 0 - while True: + while self.running: + if not self.running: + logger.info("生产者线程收到停止信号") + break start_time = time.time() try: # Fetch new tasks with time filtering @@ -202,7 +267,10 @@ class BisectTask: This function runs in an infinite loop, checking for tasks every 30 seconds. Tasks are either submitted to a scheduler or run locally, depending on the environment variable 'bisect_mode'. """ - while True: + while self.running: + if not self.running: + logger.info("消费者线程收到停止信号") + break cycle_start = time.time() processed = 0 try: @@ -221,6 +289,8 @@ class BisectTask: else: # If mode is not 'submit', run tasks locally logger.debug("Running bisect tasks locally") + # 多线程在这里处理? + # 一次处理一个 error_id 的系列 self.run_bisect_tasks(bisect_tasks) processed = len(bisect_tasks) @@ -239,97 +309,99 @@ class BisectTask: time.sleep(max(10, sleep_time - cycle_time)) # 保证最小间隔 def run_bisect_tasks(self, bisect_tasks): - """ - Process a list of bisect tasks locally by running Git bisect to find the first bad commit. - Updates the task status in Elasticsearch to 'processing' before starting the bisect process, - and updates the result after the bisect process completes. - - :param bisect_tasks: List of bisect tasks to process. - """ + """处理 bisect 任务列表（使用 ManticoreClient 进行写操作）""" for bisect_task in bisect_tasks: - task_id = bisect_task.get('id', 'unknown') - task_id = int(task_id) + task_id = int(bisect_task.get('id', 0)) + if not task_id: + logger.error("无效的任务ID") + continue + bad_job_id = bisect_task.get('bad_job_id') - task_result_root = bisect_task.get('bisect_result_root', None) - if task_result_root is None: - task_result_root = os.path.abspath( - os.path.join('bisect_results', bad_job_id) - ) + error_id = bisect_task.get('error_id') + task_result_root = os.path.abspath(os.path.join('bisect_results', str(task_id))) - task_metric = None - task_good_commit = None try: - time.sleep(5) - #检查bisect_task的状态，乐观锁 - if bisect_task.get('bisect_status') != 'wait': - logger.warning(f"Skipping task {task_id} with invalid status: {bisect_task['bisect_status']}") + # 检查任务状态 + status_check = self.bisect_db.execute_query( + f"SELECT bisect_status FROM bisect WHERE id = {task_id} AND bisect_status = 'wait'" + ) + if not status_check: + logger.warning(f"跳过无效任务 | ID: {task_id}") continue - # Update task status to 'processing' in Elasticsearch + + current_time = int(time.time()) + # 更新任务状态为处理中 bisect_task["bisect_status"] = "processing" # Convert to Manticore SQL update update_sql = f""" UPDATE bisect - SET bisect_status = 'processing' + SET bisect_status = 'processing', + start_time = {current_time}, + updated_at = {current_time} WHERE id = {task_id} AND bisect_status = 'wait' """ - #TODO UPDATE affected_rows = self.bisect_db.execute_update(update_sql) if affected_rows == 0: - logger.warning(f"跳过已被处理的任务 ID: {task_id}") + logger.warning(f"<87><84><90><86><9A><84><8A> ID: {task_id}") continue - logger.debug(f"Started processing task: {task_id}") - # Prepare task data for Git bisect with result root - # Create unique temporary directory with cleanup - # Create unique clone path using task ID + logger.info(f"开始处理任务 | ID: {task_id}") + + # 准备任务数据 task = { - 'bad_job_id': bisect_task['bad_job_id'], - 'error_id': bisect_task['error_id'], - 'good_commit': task_good_commit, - 'bisect_result_root': task_result_root, - 'metric': task_metric + 'bad_job_id': bad_job_id, + 'error_id': error_id, + 'bisect_result_root': task_result_root } - # Handle bad_job_id conversion with validation - try: - gb = GitBisect() - result = gb.find_first_bad_commit(task) - except (ValueError, KeyError) as e: - raise ValueError(f"Invalid bad_job_id: {task.get('bad_job_id')}") from e - # TODO: save error_id to regression - # Update task status and result in Elasticsearch + # 执行 bisect + gb = GitBisect() + result = gb.find_first_bad_commit(task) + if result: - # Convert to Manticore SQL update - update_sql = f""" - UPDATE bisect - SET bisect_status = 'completed', - project = {result['repo']}, - git_url = {result['git_url']}, - bad_commit = {result['first_bad_commit']}, - first_bad_id = {result['first_bad_id']}, - bad_result_root = {result['bad_result_root']}, - work_dir = {result['work_dir']}, - start_time = {result['start_time']}, - end_time = {result['end_time']}, - WHERE id = {task_id} - """ - # TODO update - self.bisect_db.execute_update(update_sql) - self.update_regression(task, result) - logger.debug(f"Completed processing task: {bisect_task['id']}") + # 构造完整结果文档 + complete_doc = { + "bisect_status": "completed", + "project": result.get('repo', ''), + "git_url": result.get('git_url', ''), + "bad_commit": result.get('first_bad_commit', ''), + "first_bad_id": result.get('first_bad_id', ''), + "first_result_root": result.get('bad_result_root', ''), + "work_dir": result.get('work_dir', ''), + "start_time": result.get('start_time', 0), + "end_time": int(time.time()), + "updated_at": int(time.time()) + } + + # 使用重试机制更新结果 + if not self.client.replace_with_retry("bisect", task_id, complete_doc): + logger.error(f"结果更新失败 | ID: {task_id}") + else: + # 更新回归数据 + self.update_regression(task, result) + logger.info(f"任务完成 | ID: {task_id}") + + else: + # 处理失败情况 + fail_doc = { + "bisect_status": "failed", + "last_error": "Bisect execution failed", + "updated_at": int(time.time()) + } + self.client.replace_with_retry("bisect", task_id, fail_doc) + logger.error(f"任务执行失败 | ID: {task_id}") except Exception as e: - # Update task status to 'failed' in case of an error - # Convert to Manticore SQL update - # Remove bisect_result column from update - update_sql = f""" - UPDATE bisect - SET bisect_status = 'failed - WHERE id = {task_id} - """ - self.bisect_db.execute_update(update_sql) - logger.error(f"Marked task {bisect_task['id']} as failed due to error: {e}") + # 异常处理 + error_doc = { + "bisect_status": "failed", + "last_error": str(e)[:200], # 限制错误信息长度 + "updated_at": int(time.time()) + } + self.client.replace_with_retry("bisect", task_id, error_doc) + logger.error(f"任务异常 | ID: {task_id} | 错误: {str(e)}") + logger.error(traceback.format_exc()) def update_regression(self, task, result): """Update regression database with bisect results""" @@ -401,6 +473,7 @@ class BisectTask: except Exception as e: logger.error(f"Failed to update regression: {str(e)}") logger.error(f"Task: {task} | Result: {result}") + def submit_bisect_tasks(self, bisect_tasks): """ Submit a list of bisect tasks to the scheduler if they are not already in the database. @@ -572,23 +645,7 @@ class BisectTask: logger.info(f"生成任务数：{len(result)} | 白名单模式：{bool(white_list)}") return result - def check_existing_bisect_task(self, bad_job_id, error_id): - """ - Check if a bisect task with the given bad_job_id and error_id already exists. - :param bad_job_id: The ID of the bad job to check. - :param error_id: The error ID associated with the bad job. - :return: Boolean indicating if task exists. - """ - try: - result = self.bisect_db.execute_query( - "SELECT 1 FROM bisect WHERE bad_job_id = %s AND error_id = %s LIMIT 1", - (bad_job_id, error_id) - ) - return bool(result) - except Exception as e: - logger.error(f"Error checking existing task: {str(e)}") - return True def _start_monitor(self): def monitor(): while True: @@ -597,8 +654,10 @@ class BisectTask: try: jobs_active = self.jobs_db.pool._cnx_queue.qsize() bisect_active = self.bisect_db.pool._cnx_queue.qsize() + bisect_active = self.regression_db.pool._cnx_queue.qsize() logger.info(f"Jobs DB 活跃连接: {jobs_active}") logger.info(f"Bisect DB 活跃连接: {bisect_active}") + logger.info(f"Regression DB 活跃连接: {bisect_active}") except AttributeError as e: logger.warning(f"连接池状态获取失败: {str(e)}") diff --git a/lib/bisect_database.py b/lib/bisect_database.py index 6c98479d..6944285b 100644 --- a/lib/bisect_database.py +++ b/lib/bisect_database.py @@ -10,6 +10,7 @@ from typing import Optional, Dict, Any, Tuple, List from functools import wraps import time import threading +from manticore_simple import ManticoreClient class BisectDB(GenericSQLClient): """Database operations for bisect process with connection pool""" @@ -184,9 +185,15 @@ class BisectDB(GenericSQLClient): return f"'{json_str}'" def close(self): - """Close database connection""" - if self._connection and self._connection.is_connected(): - self._connection.close() + """安全关闭连接池""" + try: + if hasattr(self, 'pool'): + self.pool.close() + logger.info(f"已关闭 {self.database} 数据库连接池") + # 防止重复关闭 + del self.pool + except Exception as e: + logger.error(f"关闭数据库连接池失败: {str(e)}") def check_connection_leaks(self): """增强泄漏检查""" diff --git a/lib/manticore_simple.py b/lib/manticore_simple.py new file mode 100644 index 00000000..7d6177ab --- /dev/null +++ b/lib/manticore_simple.py @@ -0,0 +1,53 @@ +import requests +import json +import time +from typing import Dict, List, Union, Optional + +class ManticoreClient: + def __init__(self, host="localhost", port=9308): + self.base_url = f"http://{host}:{port}" + + def insert(self, index: str, id: int, document: dict) -> bool: + """插入文档""" + return self._request("insert", index, id, document) + + def replace(self, index: str, id: int, document: dict) -> bool: + """替换文档""" + return self._request("replace", index, id, document) + + def replace_with_retry(self, index: str, id: int, document: dict, retries: int = 3) -> bool: + """带重试机制的替换操作""" + for i in range(retries): + if self.replace(index, id, document): + return True + time.sleep(2 ** i) + return False + + def bulk_replace(self, index: str, documents: Dict[int, dict]) -> bool: + """批量替换文档""" + try: + bulk_body = [] + for doc_id, doc in documents.items(): + bulk_body.append(json.dumps({"replace": {"_index": index, "_id": doc_id}})) + bulk_body.append(json.dumps(doc)) + + resp = requests.post( + f"{self.base_url}/bulk", + data="\n".join(bulk_body), + headers={"Content-Type": "application/x-ndjson"}, + timeout=10 + ) + return resp.status_code == 200 + except requests.exceptions.RequestException: + return False + + def _request(self, endpoint: str, index: str, id: int, doc: dict) -> bool: + try: + resp = requests.post( + f"{self.base_url}/{endpoint}", + json={"index": index, "id": id, "doc": doc}, + timeout=3 + ) + return resp.status_code == 200 + except requests.exceptions.RequestException: + return False diff --git a/sbin/manti-table-bisect.sql b/sbin/manti-table-bisect.sql index 12a5f68d..c343ead4 100755 --- a/sbin/manti-table-bisect.sql +++ b/sbin/manti-table-bisect.sql @@ -13,5 +13,10 @@ CREATE TABLE bisect( start_time BIGINT, end_time BIGINT, priority_level INT, - timeout INT + timeout INT, + retry_count INT, + last_error string, + created_at BIGINT, + updated_at BIGINT, + j json ) charset_table='U+0021..U+007E'; -- Gitee From abb5abeb3cc527e5037339d585f070c5ae99c88a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 15 May 2025 16:54:48 +0800 Subject: [PATCH 02/39] bisect-tasks: fix requirement Signed-off-by: jacknichao --- container/bisect/Dockerfile | 8 ++++---- container/bisect/build | 1 + container/bisect/start | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/container/bisect/Dockerfile b/container/bisect/Dockerfile index 9b9bb6fe..a651141c 100644 --- a/container/bisect/Dockerfile +++ b/container/bisect/Dockerfile @@ -8,12 +8,12 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories RUN apk update --no-cache && \ apk upgrade && \ apk add --no-cache git coreutils util-linux ruby bash python3 python3-dev py3-pip \ - mariadb-dev build-base mariadb-connector-c \ - py3-requests py3-yaml py3-httpx py3-flask py3-numpy py3-gitpython - + py3-requests py3-yaml py3-httpx py3-flask py3-numpy \ + cpio +# mariadb-dev build-base mariadb-connector-c \ RUN gem install rest-client -RUN pip3 install mysql-connector-python --break-system-packages +RUN pip3 install mysql-connector-python --break-system-packages -i https://mirrors.aliyun.com/pypi/simple/ RUN addgroup -S bisect && adduser -S bisect -G bisect ENV WORK_DIR /home/bisect diff --git a/container/bisect/build b/container/bisect/build index 37314921..7dd85116 100755 --- a/container/bisect/build +++ b/container/bisect/build @@ -11,6 +11,7 @@ load_cci_defaults if [ ! -d compass-ci ]; then git clone https://gitee.com/openeuler/compass-ci.git fi + if [ ! -d lkp-tests ]; then git clone https://gitee.com/compass-ci/lkp-tests.git fi diff --git a/container/bisect/start b/container/bisect/start index 6d9d5bf3..349ff395 100755 --- a/container/bisect/start +++ b/container/bisect/start @@ -24,7 +24,7 @@ DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' DEFAULT_CONFIG_DIR = '/etc/compass-ci/defaults' DEFAULT_USER_CONFIG_DIR = File.expand_path("~/.config/compass-ci/") -DEFAULT_BISECT_CONFIG_DIR = '/root/.config/compass-ci/' +DEFAULT_BISECT_CONFIG_DIR = '/home/bisect/.config/compass-ci/' MANTICORE_HOST='172.17.0.1' docker_rm 'bisect' @@ -41,7 +41,7 @@ cmd = %w[ -v #{DEFAULT_USER_CONFIG_DIR}:#{DEFAULT_BISECT_CONFIG_DIR}:ro -v /etc/localtime:/etc/localtime:ro -v /etc/compass-ci/register:/etc/compass-ci/register:ro - -p 10000:9999 + -p 9999:9999 bisect ] -- Gitee From ba7ac91928611fcd544988cd329ec16422670128 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Mon, 19 May 2025 16:50:52 +0800 Subject: [PATCH 03/39] bisect-task: fix error_id white list File "/home/bisect/bisect-task.py", line 234, in bisect_producer new_bisect_tasks = self.get_new_bisect_task_from_jobs() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/bisect/bisect-task.py", line 613, in get_new_bisect_task_from_jobs errid_tasks = self.process_data(result, errid_white_list) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/bisect/bisect-task.py", line 629, in process_data candidates = set(errids) & set(white_list) ^^^^^^^^^^^^^^^ TypeError: unhashable type: 'dict' Default white list should be a set or list, not dict --- container/bisect/bisect-task.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index b73c9838..b03b4525 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -238,7 +238,7 @@ class BisectTask: if not self.bisect_db.execute_query(f"""SELECT id FROM bisect WHERE error_id='{task['error_id']}' AND bad_job_id='{task['bad_job_id']}'"""): if self.add_bisect_task(task): success_count += 1 - logger.info(f"Add {task['error_id']} and {task['bad_job_id']} OK") + logger.info(f"Add {task['error_id']} and {task['bad_job_id']} OK") error_count = max(0, error_count - 1) # Reduce error count on success @@ -603,7 +603,8 @@ class BisectTask: AND valid = 'true' ORDER BY id DESC """ - errid_white_list = self.regression_db.execute_query(sql_error_id) + errid_white_list_raw = self.regression_db.execute_query(sql_error_id) + errid_white_list = {item['errid'] for item in errid_white_list_raw} if errid_white_list_raw else set() # Execute Manticore SQL queries result = self.bisect_db.execute_query(sql_failure) -- Gitee From 4c8e1d0839fa76ae9d9a21a28b2511e600a79eda Mon Sep 17 00:00:00 2001 From: jacknichao Date: Mon, 19 May 2025 17:14:15 +0800 Subject: [PATCH 04/39] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96bisect=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E7=9B=AE=E5=BD=95=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 47 ++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index b03b4525..550d30b1 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -18,6 +18,8 @@ import json import re import subprocess +import hashlib +from datetime import datetime import yaml import os import sys @@ -85,12 +87,39 @@ class BisectTask: self._cleanup_interrupted_tasks() sys.exit(1) + def get_task_result_root(self, task): + """生成带日期的扁平化存储路径""" + # 安全处理仓库名（替换特殊字符） + repo_name = re.sub(r'[^\w\-]', '_', task.get('repo', 'unknown_repo'))[:32] + + # 获取任务时间（使用当前时间作为默认） + create_time = task.get('create_time', time.time()) + date_str = datetime.fromtimestamp(create_time).strftime("%Y-%m-%d") + + # 压缩错误ID为8位哈希 + error_id_hash = hashlib.md5(task['error_id'].encode()).hexdigest()[:8] + + # 构建路径 + path = os.path.join( + 'bisect_results', + repo_name, + date_str, + str(task['bad_job_id']), + error_id_hash, + str(task['id']) + ) + + # 创建目录并返回绝对路径 + abs_path = os.path.abspath(path) + os.makedirs(abs_path, exist_ok=True, mode=0o755) + return abs_path + def _cleanup_interrupted_tasks(self): """清理被中断的任务""" try: # 1. 获取所有 processing 状态的任务 tasks = self.bisect_db.execute_query( - "SELECT id FROM bisect " + "SELECT id, work_dir as result_root FROM bisect " "WHERE bisect_status = 'processing'" ) @@ -108,7 +137,6 @@ class BisectTask: # 3. 删除数据目录 for task in tasks: - result_root = os.path.abspath(os.path.join('bisect_results', task['id'])) if os.path.exists(result_root): try: shutil.rmtree(result_root) @@ -318,7 +346,13 @@ class BisectTask: bad_job_id = bisect_task.get('bad_job_id') error_id = bisect_task.get('error_id') - task_result_root = os.path.abspath(os.path.join('bisect_results', str(task_id))) + task_result_root = self.get_task_result_root({ + 'suite': bisect_task.get('suite'), + 'bad_job_id': bisect_task['bad_job_id'], + 'error_id': bisect_task['error_id'], + 'id': task_id, + 'create_time': bisect_task.get('start_time', time.time()) + }) try: # 检查任务状态 @@ -337,7 +371,8 @@ class BisectTask: UPDATE bisect SET bisect_status = 'processing', start_time = {current_time}, - updated_at = {current_time} + updated_at = {current_time}, + work_dir = {task_result_root} WHERE id = {task_id} AND bisect_status = 'wait' """ @@ -585,10 +620,10 @@ class BisectTask: :return: A list of processed tasks that match the white list criteria. """ # Define SQL for PKGBUILD tasks - # TODO: AND submit_time > NOW() - INTERVAL 7 DAY + # TODO: 增加判断 AND submit_time > NOW() - INTERVAL 7 DAY # Perf monitor sql_failure = """ - SELECT id, errid as errid + SELECT id, errid as errid, suite, category FROM jobs WHERE j.job_health = 'abort' AND j.stats IS NOT NULL -- Gitee From cf22f517c5529e14f54b1a034e542500c780e3f4 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Mon, 19 May 2025 20:05:56 +0800 Subject: [PATCH 05/39] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E5=A4=9A?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E5=A4=84=E7=90=86=E4=BA=8C=E5=88=86=E6=9F=A5?= =?UTF-8?q?=E6=89=BE=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 94 ++++++++++++++++++++++++++++++--- lib/bisect_database.py | 12 +++++ lib/generic_sql_client.py | 6 ++- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 550d30b1..7a8d94d2 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -114,6 +114,49 @@ class BisectTask: os.makedirs(abs_path, exist_ok=True, mode=0o755) return abs_path + def _process_single_task(self, task_dict: dict): + """子进程内执行的原子操作""" + # 重新初始化必要组件 + from lib.bisect_database import BisectDB + from lib.log_config import logger + + # 每个进程独立数据库连接 + local_db = BisectDB( + host=os.getenv('MANTICORE_HOST'), + port=os.getenv('MANTICORE_PORT'), + database="bisect", + pool_size=1 # 单连接避免冲突 + ) + + try: + # 任务处理逻辑 + task_id = task_dict['id'] + logger.info(f"Process {os.getpid()} handling task {task_id}") + + # 获取完整任务数据 + task = local_db.execute_query( + "SELECT * FROM bisect WHERE id = %s", + (task_id,) + )[0] + + # 执行核心逻辑 + gb = GitBisect() + result = gb.find_first_bad_commit(task) + + # 更新状态 + local_db.execute_update( + "UPDATE bisect SET status='completed' WHERE id=%s", + (task_id,) + ) + + return {'id': task_id, 'status': 'success'} + except Exception as e: + error_msg = f"Task {task_id} failed: {str(e)}" + logger.error(error_msg) + return {'id': task_id, 'status': 'failed', 'error': error_msg} + finally: + local_db.close() + def _cleanup_interrupted_tasks(self): """清理被中断的任务""" try: @@ -160,8 +203,25 @@ class BisectTask: port=int(os.environ.get('MANTICORE_WRITE_PORT', '9308')) ) + # 初始化进程池 + self.process_pool = ProcessPoolExecutor( + max_workers=min(4, multiprocessing.cpu_count()), + mp_context=multiprocessing.get_context('spawn'), + initializer=self._child_process_init + ) + self.task_futures = [] + self._register_signal_handlers() # 注册信号处理器 + @staticmethod + def _child_process_init(): + """子进程初始化函数""" + # 关闭父进程的连接池 + if hasattr(BisectDB, 'pool'): + BisectDB.pool.disconnect() + # 重置日志配置 + logger.reinit_for_process() + # Initialize read-only jobs database connection self.jobs_db = BisectDB( host=os.environ.get('MANTICORE_HOST', 'localhost'), @@ -337,12 +397,34 @@ class BisectTask: time.sleep(max(10, sleep_time - cycle_time)) # 保证最小间隔 def run_bisect_tasks(self, bisect_tasks): - """处理 bisect 任务列表（使用 ManticoreClient 进行写操作）""" - for bisect_task in bisect_tasks: - task_id = int(bisect_task.get('id', 0)) - if not task_id: - logger.error("无效的任务ID") - continue + """新版多进程任务处理""" + if not bisect_tasks: + return + + # 提交任务到进程池 + futures = [] + for task in bisect_tasks: + # 只传递必要ID，减少序列化开销 + future = self.process_pool.submit( + self._process_single_task, + {'id': task['id']} # 可序列化的最小数据 + ) + futures.append(future) + self.task_futures.append(future) + + # 处理结果 + for future in as_completed(futures, timeout=7200): # 2小时超时 + try: + result = future.result() + if result['status'] == 'success': + logger.info(f"任务完成: {result['id']}") + else: + logger.error(f"任务失败: {result['id']} - {result['error']}") + except TimeoutError: + logger.error("任务处理超时，可能发生死锁") + future.cancel() + except Exception as e: + logger.error(f"结果处理异常: {str(e)}") bad_job_id = bisect_task.get('bad_job_id') error_id = bisect_task.get('error_id') diff --git a/lib/bisect_database.py b/lib/bisect_database.py index 6944285b..22bd30f5 100644 --- a/lib/bisect_database.py +++ b/lib/bisect_database.py @@ -25,6 +25,7 @@ class BisectDB(GenericSQLClient): ): """Initialize database connection pool""" self._active_connections = threading.local() # Thread local storage + self.pid = os.getpid() # 记录进程ID super().__init__( host=host, port=port, @@ -184,6 +185,17 @@ class BisectDB(GenericSQLClient): json_str = json.dumps(value).replace("'", "''") return f"'{json_str}'" + def _reinit_pool(self, pid): + """进程变化时重新初始化连接池""" + self.pid = pid + if hasattr(self, 'pool'): + self.pool.close() + self.pool = mysql.connector.pooling.MySQLConnectionPool( + pool_name=f"bisect_pool_{pid}", + pool_size=self.pool_size, + **self.connect_args + ) + def close(self): """安全关闭连接池""" try: diff --git a/lib/generic_sql_client.py b/lib/generic_sql_client.py index 5e18154a..35e8c589 100644 --- a/lib/generic_sql_client.py +++ b/lib/generic_sql_client.py @@ -64,7 +64,11 @@ class GenericSQLClient: raise RuntimeError(f"无法连接数据库 {database}@{host}:{port}") def get_connection(self): - """从连接池获取数据库连接（基类实现）""" + """进程安全的连接获取""" + current_pid = os.getpid() + if not hasattr(self, '_pool') or self.pid != current_pid: + self._reinit_pool(current_pid) + try: conn = self.pool.get_connection() if not hasattr(self._active_connections, 'count'): -- Gitee From 242f90a99a6934abdc55058130a039ecf3ff0d61 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Mon, 19 May 2025 20:07:31 +0800 Subject: [PATCH 06/39] =?UTF-8?q?bisect-task:=20=20=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E6=B1=A0=E7=AE=A1=E7=90=86=20bisect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix: 修复未定义变量和缺少导入的问题 fix: 修复未定义变量引用问题 feat: 添加bisect_db和regression_db数据库连接配置 fix: 移除线程本地存储并重构多进程任务处理 fix: 修复未定义的GitBisect和_process_task_worker引用 feat: 添加多进程序列化问题演示脚本 refactor: 重构二分查找任务处理逻辑并优化线程池管理 fix: 重构进程池实现以解决资源隔离和序列化问题 fix: 修复静态方法中错误使用self的问题 refactor: 将regression_db重命名为process_regression_db feat: 为主进程添加独立数据库连接并优化查询逻辑 fix: 修复SQL注入风险并验证数据库连接 --- container/bisect/bisect-task.py | 360 +++++++++++++++----------------- lib/bisect_database.py | 42 +++- lib/generic_sql_client.py | 8 +- lib/log_config.py | 9 - pickle_error_demo.py | 92 ++++++++ 5 files changed, 308 insertions(+), 203 deletions(-) create mode 100644 pickle_error_demo.py diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 7a8d94d2..8175ec43 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -25,11 +25,13 @@ import os import sys import shutil import time -import threading import traceback import mysql.connector import hashlib import signal +import multiprocessing +import threading +from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed from random import randint from flask import Flask, jsonify, request from flask.views import MethodView @@ -74,6 +76,55 @@ def generate_task_id(bad_job_id: str, error_id: str) -> int: return hash_int & 0x7FFFFFFFFFFFFFFF class BisectTask: + @staticmethod + def _init_process_resources(config): + """子进程资源初始化""" + global process_bisect_db, process_jobs_db, process_regression_db, process_client + + process_bisect_db = BisectDB( + host=config['manticore_host'], + port=config['manticore_port'], + database=config['bisect_db'], + pool_size=5 + ) + + process_jobs_db = BisectDB( + host=config['manticore_host'], + port=config['manticore_port'], + database=config['jobs_db'], + pool_size=3 + ) + + process_regression_db = BisectDB( + host=config['manticore_host'], + port=config['manticore_port'], + database=config['regression_db'], + pool_size=3 + ) + + process_client = ManticoreClient( + host=config['manticore_host'], + port=9308 + ) + + @staticmethod + def _generate_task_path(config: dict, task: dict) -> str: + """进程安全的路径生成""" + pid = os.getpid() + tid = threading.get_ident() + + path = os.path.join( + 'bisect_results', + f"proc-{pid}-thread-{tid}", + re.sub(r'[^\w\-]', '_', task.get('repo', 'unknown_repo'))[:32], + datetime.now().strftime("%Y-%m-%d"), + str(task['bad_job_id']), + hashlib.md5(task['error_id'].encode()).hexdigest()[:8], + str(task['id']) + ) + os.makedirs(path, exist_ok=True, mode=0o755) + return os.path.abspath(path) + def _register_signal_handlers(self): """注册信号处理""" signal.signal(signal.SIGINT, self._handle_exit_signal) @@ -97,6 +148,7 @@ class BisectTask: date_str = datetime.fromtimestamp(create_time).strftime("%Y-%m-%d") # 压缩错误ID为8位哈希 + # TODO remove error_id_hash = hashlib.md5(task['error_id'].encode()).hexdigest()[:8] # 构建路径 @@ -114,48 +166,87 @@ class BisectTask: os.makedirs(abs_path, exist_ok=True, mode=0o755) return abs_path - def _process_single_task(self, task_dict: dict): - """子进程内执行的原子操作""" - # 重新初始化必要组件 - from lib.bisect_database import BisectDB - from lib.log_config import logger - - # 每个进程独立数据库连接 - local_db = BisectDB( - host=os.getenv('MANTICORE_HOST'), - port=os.getenv('MANTICORE_PORT'), - database="bisect", - pool_size=1 # 单连接避免冲突 - ) - + @staticmethod + def _process_single_task(config: dict, task: dict): + """静态任务处理方法""" try: - # 任务处理逻辑 - task_id = task_dict['id'] - logger.info(f"Process {os.getpid()} handling task {task_id}") + global process_bisect_db, process_client - # 获取完整任务数据 - task = local_db.execute_query( - "SELECT * FROM bisect WHERE id = %s", - (task_id,) - )[0] - - # 执行核心逻辑 - gb = GitBisect() - result = gb.find_first_bad_commit(task) - - # 更新状态 - local_db.execute_update( - "UPDATE bisect SET status='completed' WHERE id=%s", - (task_id,) - ) + task_id = task['id'] + task_result_root = BisectTask._generate_task_path(config, task) + + try: + # 检查任务状态 + status_check = process_bisect_db.execute_query( + f"SELECT bisect_status FROM bisect WHERE id = {task_id} AND bisect_status = 'wait'" + ) + if not status_check: + logger.warning(f"跳过无效任务 | ID: {task_id}") + return + + # 更新任务状态为处理中 + task["bisect_status"] = "processing" + # Convert to Manticore SQL update + + if process_client.replace("bisect", task_id, task): + logger.info(f"开始处理任务 | ID: {task_id}") + + # 准备任务数据 + task['bisect_result_root'] = task_result_root + # 执行 bisect + logger.info(f"开始处理任务 | {task}") + gb = GitBisect() + result = gb.find_first_bad_commit(task) + + if result: + # 构造完整结果文档 + complete_doc = { + "bisect_status": "completed", + "project": result.get('repo', ''), + "git_url": result.get('git_url', ''), + "bad_commit": result.get('first_bad_commit', ''), + "first_bad_id": result.get('first_bad_id', ''), + "first_result_root": result.get('bad_result_root', ''), + "work_dir": result.get('work_dir', ''), + "start_time": result.get('start_time', 0), + "end_time": int(time.time()), + "updated_at": int(time.time()) + } + + # 使用重试机制更新结果 + if not process_client.replace_with_retry("bisect", task_id, complete_doc): + logger.error(f"结果更新失败 | ID: {task_id}") + else: + # 更新回归数据 + BisectTask.update_regression(task, result) + logger.info(f"任务完成 | ID: {task_id}") + + else: + # 处理失败情况 + fail_doc = { + "bisect_status": "failed", + "last_error": "Bisect execution failed", + "updated_at": int(time.time()) + } + process_client.replace_with_retry("bisect", task_id, fail_doc) + logger.error(f"任务执行失败 | ID: {task_id}") + + except Exception as e: + # 异常处理 + error_doc = { + "bisect_status": "failed", + "last_error": str(e)[:200], # 限制错误信息长度 + "updated_at": int(time.time()) + } + process_client.replace_with_retry("bisect", task_id, error_doc) + logger.error(f"任务异常 | ID: {task_id} | 错误: {str(e)}") + logger.error(traceback.format_exc()) return {'id': task_id, 'status': 'success'} except Exception as e: error_msg = f"Task {task_id} failed: {str(e)}" logger.error(error_msg) return {'id': task_id, 'status': 'failed', 'error': error_msg} - finally: - local_db.close() def _cleanup_interrupted_tasks(self): """清理被中断的任务""" @@ -180,7 +271,8 @@ class BisectTask: # 3. 删除数据目录 for task in tasks: - if os.path.exists(result_root): + result_root = task.get('result_root') + if result_root and os.path.exists(result_root): try: shutil.rmtree(result_root) logger.info(f"成功删除数据目录: {result_root}") @@ -196,55 +288,53 @@ class BisectTask: def __init__(self): self.bisect_task = None self.running = True # 运行状态标志 - - # Initialize ManticoreClient for HTTP API operations self.client = ManticoreClient( host=os.environ.get('MANTICORE_HOST', 'localhost'), port=int(os.environ.get('MANTICORE_WRITE_PORT', '9308')) ) - - # 初始化进程池 - self.process_pool = ProcessPoolExecutor( - max_workers=min(4, multiprocessing.cpu_count()), - mp_context=multiprocessing.get_context('spawn'), - initializer=self._child_process_init - ) - self.task_futures = [] - - self._register_signal_handlers() # 注册信号处理器 - - @staticmethod - def _child_process_init(): - """子进程初始化函数""" - # 关闭父进程的连接池 - if hasattr(BisectDB, 'pool'): - BisectDB.pool.disconnect() - # 重置日志配置 - logger.reinit_for_process() - - # Initialize read-only jobs database connection self.jobs_db = BisectDB( host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="jobs", - readonly=True, - pool_size=10 + pool_size=3 ) - - # Initialize read-write bisect database connection + logger.info(f"主进程 jobs DB 连接 ID: {id(self.jobs_db)}") + self.bisect_db = BisectDB( host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="bisect", - pool_size=15 + pool_size=3 ) + logger.info(f"主进程 bisect DB 连接 ID: {id(self.bisect_db)}") + # 主进程数据库连接初始化 self.regression_db = BisectDB( host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="regression", - pool_size=5 + pool_size=3 ) + logger.info(f"主进程 RegressionDB 连接 ID: {id(self.regression_db)}") + + # 子进程配置 + self._config = { + "manticore_host": os.environ.get('MANTICORE_HOST', 'localhost'), + "manticore_port": os.environ.get('MANTICORE_PORT', '9306'), + "bisect_db": "bisect", + "jobs_db": "jobs", + "regression_db": "regression" + } + + self.process_pool = ProcessPoolExecutor( + max_workers=os.cpu_count(), + initializer=self._init_process_resources, + initargs=(self._config,) + ) + self.task_futures = [] + + self._register_signal_handlers() # 注册信号处理器 + self._start_monitor() @@ -395,25 +485,21 @@ class BisectTask: # 动态休眠控制（无任务时延长休眠） sleep_time = 30 if processed > 0 else 60 time.sleep(max(10, sleep_time - cycle_time)) # 保证最小间隔 - def run_bisect_tasks(self, bisect_tasks): - """新版多进程任务处理""" + """改造后的任务提交方法""" if not bisect_tasks: return - - # 提交任务到进程池 futures = [] for task in bisect_tasks: - # 只传递必要ID，减少序列化开销 future = self.process_pool.submit( self._process_single_task, - {'id': task['id']} # 可序列化的最小数据 + self._config, + task ) futures.append(future) self.task_futures.append(future) - - # 处理结果 - for future in as_completed(futures, timeout=7200): # 2小时超时 + + for future in as_completed(futures, timeout=72000): # 2小时超时 try: result = future.result() if result['status'] == 'success': @@ -426,100 +512,6 @@ class BisectTask: except Exception as e: logger.error(f"结果处理异常: {str(e)}") - bad_job_id = bisect_task.get('bad_job_id') - error_id = bisect_task.get('error_id') - task_result_root = self.get_task_result_root({ - 'suite': bisect_task.get('suite'), - 'bad_job_id': bisect_task['bad_job_id'], - 'error_id': bisect_task['error_id'], - 'id': task_id, - 'create_time': bisect_task.get('start_time', time.time()) - }) - - try: - # 检查任务状态 - status_check = self.bisect_db.execute_query( - f"SELECT bisect_status FROM bisect WHERE id = {task_id} AND bisect_status = 'wait'" - ) - if not status_check: - logger.warning(f"跳过无效任务 | ID: {task_id}") - continue - - current_time = int(time.time()) - # 更新任务状态为处理中 - bisect_task["bisect_status"] = "processing" - # Convert to Manticore SQL update - update_sql = f""" - UPDATE bisect - SET bisect_status = 'processing', - start_time = {current_time}, - updated_at = {current_time}, - work_dir = {task_result_root} - WHERE id = {task_id} - AND bisect_status = 'wait' - """ - affected_rows = self.bisect_db.execute_update(update_sql) - if affected_rows == 0: - logger.warning(f"<87><84><90><86><9A><84><8A> ID: {task_id}") - continue - - logger.info(f"开始处理任务 | ID: {task_id}") - - # 准备任务数据 - task = { - 'bad_job_id': bad_job_id, - 'error_id': error_id, - 'bisect_result_root': task_result_root - } - - # 执行 bisect - gb = GitBisect() - result = gb.find_first_bad_commit(task) - - if result: - # 构造完整结果文档 - complete_doc = { - "bisect_status": "completed", - "project": result.get('repo', ''), - "git_url": result.get('git_url', ''), - "bad_commit": result.get('first_bad_commit', ''), - "first_bad_id": result.get('first_bad_id', ''), - "first_result_root": result.get('bad_result_root', ''), - "work_dir": result.get('work_dir', ''), - "start_time": result.get('start_time', 0), - "end_time": int(time.time()), - "updated_at": int(time.time()) - } - - # 使用重试机制更新结果 - if not self.client.replace_with_retry("bisect", task_id, complete_doc): - logger.error(f"结果更新失败 | ID: {task_id}") - else: - # 更新回归数据 - self.update_regression(task, result) - logger.info(f"任务完成 | ID: {task_id}") - - else: - # 处理失败情况 - fail_doc = { - "bisect_status": "failed", - "last_error": "Bisect execution failed", - "updated_at": int(time.time()) - } - self.client.replace_with_retry("bisect", task_id, fail_doc) - logger.error(f"任务执行失败 | ID: {task_id}") - - except Exception as e: - # 异常处理 - error_doc = { - "bisect_status": "failed", - "last_error": str(e)[:200], # 限制错误信息长度 - "updated_at": int(time.time()) - } - self.client.replace_with_retry("bisect", task_id, error_doc) - logger.error(f"任务异常 | ID: {task_id} | 错误: {str(e)}") - logger.error(traceback.format_exc()) - def update_regression(self, task, result): """Update regression database with bisect results""" try: @@ -534,7 +526,7 @@ class BisectTask: error_id = task['error_id'].replace("'", "''") # 转义单引号 # 检查是否已存在有效记录 - existing = self.regression_db.execute_query( + existing = process_regression_db.execute_query( f"SELECT id, bisect_count, related_jobs " f"FROM regression " f"WHERE record_type = 'errid' " @@ -547,7 +539,7 @@ class BisectTask: new_id = int(f"{current_time}{randint(1000,9999)}") # 生成唯一ID category = result.get('category', 'unknown').replace("'", "''") related_jobs_json = json.dumps([bad_job_id]) # 初始化为数组 - + # TODO replace by manticore insert_sql = f""" INSERT INTO regression (id, record_type, errid, category, @@ -581,9 +573,9 @@ class BisectTask: '$', '{bad_job_id}' ) - WHERE id = {record_id} + WHERE id = %s """ - self.regression_db.execute_update(update_sql) + process_regression_db.execute_update(update_sql, (record_id,)) logger.info(f"Regression updated | ErrorID: {error_id}") @@ -639,7 +631,7 @@ class BisectTask: SELECT * FROM bisect WHERE bisect_status = 'wait' - LIMIT 20 + LIMIT 10 """ result = self.bisect_db.execute_query(sql) @@ -705,7 +697,7 @@ class BisectTask: # TODO: 增加判断 AND submit_time > NOW() - INTERVAL 7 DAY # Perf monitor sql_failure = """ - SELECT id, errid as errid, suite, category + SELECT id, errid as errid, j.suite as suite, j.category as category FROM jobs WHERE j.job_health = 'abort' AND j.stats IS NOT NULL @@ -724,7 +716,7 @@ class BisectTask: errid_white_list = {item['errid'] for item in errid_white_list_raw} if errid_white_list_raw else set() # Execute Manticore SQL queries - result = self.bisect_db.execute_query(sql_failure) + result = self.jobs_db.execute_query(sql_failure) # Convert the list of tasks into a dictionary with task IDs as keys # 添加详细日志记录原始数据格式 # Process the tasks to filter and transform them based on the white list @@ -795,7 +787,6 @@ class BisectAPI(MethodView): def post(self): task = request.json - print(task) if not task: return jsonify({"error": "No data provided"}), 400 self.bisect_api.add_bisect_task(task) @@ -887,20 +878,11 @@ def run_flask(): def main(): try: - # 先启动后台任务 - #set_log() + executor = ThreadPoolExecutor(max_workers=3) run = BisectTask() - bisect_producer_thread = threading.Thread(target=run.bisect_producer, daemon=True) - bisect_producer_thread.start() - # 在独立线程运行Flask - flask_thread = threading.Thread(target=run_flask, daemon=True) - flask_thread.start() - time.sleep(5) - - num_consumer_threads = 2 - for i in range(num_consumer_threads): - bisect_consumer_thread = threading.Thread(target=run.bisect_consumer, daemon=True) - bisect_consumer_thread.start() + executor.submit(run.bisect_producer) + executor.submit(run_flask) + executor.submit(run.bisect_consumer) # 主线程保持活跃 while True: diff --git a/lib/bisect_database.py b/lib/bisect_database.py index 22bd30f5..ab12832b 100644 --- a/lib/bisect_database.py +++ b/lib/bisect_database.py @@ -1,8 +1,48 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MulanPSL-2.0+ -from log_config import logger +import os +from log_config import logger, StructuredLogger import mysql.connector +from py_bisect import GitBisect + +def _process_task_worker(task_id: int): + """独立进程任务处理函数""" + logger = StructuredLogger() # 每个进程独立实例 + + # 每个进程独立数据库连接 + local_db = BisectDB( + host=os.getenv('MANTICORE_HOST'), + port=os.getenv('MANTICORE_PORT'), + database="bisect", + pool_size=1 + ) + + try: + logger.info(f"Process {os.getpid()} handling task {task_id}") + + # 获取任务数据 + task = local_db.execute_query( + "SELECT * FROM bisect WHERE id = %s", + (task_id,) + )[0] + + # 执行核心逻辑 + gb = GitBisect() + result = gb.find_first_bad_commit(task) + + # 更新状态 + local_db.execute_update( + "UPDATE bisect SET status='completed' WHERE id=%s", + (task_id,) + ) + return {'id': task_id, 'status': 'success'} + except Exception as e: + error_msg = f"Task {task_id} failed: {str(e)}" + logger.error(error_msg) + return {'id': task_id, 'status': 'failed', 'error': error_msg} + finally: + local_db.close() from generic_sql_client import GenericSQLClient from mysql.connector import Error import json diff --git a/lib/generic_sql_client.py b/lib/generic_sql_client.py index 35e8c589..388b8b34 100644 --- a/lib/generic_sql_client.py +++ b/lib/generic_sql_client.py @@ -25,6 +25,8 @@ class GenericSQLClient: self.port = port self.database = database self.readonly = readonly + self._connection_counter = 0 # 简单计数 + self.pool = None # 显式初始化 connect_args = { 'host': host, @@ -71,10 +73,8 @@ class GenericSQLClient: try: conn = self.pool.get_connection() - if not hasattr(self._active_connections, 'count'): - self._active_connections.count = 0 - self._active_connections.count += 1 - logger.debug(f"获取连接 #{self._active_connections.count}") + self._connection_counter += 1 + logger.debug(f"获取连接 #{self._connection_counter}") return conn except AttributeError as e: logger.error("连接池未初始化，请检查以下配置：") diff --git a/lib/log_config.py b/lib/log_config.py index 0f59ace6..c42b7989 100644 --- a/lib/log_config.py +++ b/lib/log_config.py @@ -13,16 +13,7 @@ from datetime import datetime class StructuredLogger: """统一的结构化日志记录器""" - _instance = None - - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super().__new__(cls) - return cls._instance - def __init__(self, log_dir: str = None, name: str = 'bisect-py'): - if hasattr(self, '_initialized'): - return self.logger = logging.getLogger(name) self.logger.propagate = False # Prevent log propagation diff --git a/pickle_error_demo.py b/pickle_error_demo.py new file mode 100644 index 00000000..9b34bbf7 --- /dev/null +++ b/pickle_error_demo.py @@ -0,0 +1,92 @@ +import pickle +from concurrent.futures import ProcessPoolExecutor +import threading + +# -------------------------- +# 错误案例部分 +# -------------------------- +class ProblematicLogger: + """模拟含线程本地存储的日志记录器""" + def __init__(self): + self.local_data = threading.local() # 含_thread._local对象 + +class ProblematicTask: + def __init__(self): + self.lock = threading.Lock() # 不可序列化对象 + self.logger = ProblematicLogger() # 含不可序列化组件 + + def process(self, task_id): + with self.lock: + print(f"[ERROR DEMO] Processing task {task_id}") + +# -------------------------- +# 修正案例部分 +# -------------------------- +class FixedTask: + @staticmethod + def process(task_id): + # 子进程内部创建资源 + lock = threading.Lock() # ✅ 每个进程独立创建 + logger = ProblematicLogger() # ✅ 进程内初始化 + + with lock: + print(f"[FIXED DEMO] Processing task {task_id}") + +# -------------------------- +# 验证函数 +# -------------------------- +def verify_serialization(): + """验证对象序列化能力""" + print("\n=== 序列化验证 ===") + + # 测试错误案例 + try: + bad_obj = ProblematicTask() + pickle.dumps(bad_obj.process) # 尝试序列化绑定方法 + print("❌ 错误案例意外序列化成功") + except Exception as e: + print(f"✅ 错误案例验证成功（预期失败）") + print(f" → 错误类型: {type(e).__name__}") + print(f" → 错误信息: {e}") + + # 测试修正案例 + try: + pickle.dumps(FixedTask.process) # 序列化静态方法 + print("✅ 修正案例序列化验证通过") + except Exception as e: + print(f"❌ 修正案例验证失败") + print(f" → 错误信息: {str(e)}") + +# -------------------------- +# 多进程执行测试 +# -------------------------- +def run_process_demo(): + print("\n=== 多进程执行测试 ===") + + # 错误案例测试 + print("\n[测试错误案例]") + try: + with ProcessPoolExecutor() as executor: + tasks = [executor.submit(ProblematicTask().process, i) for i in range(3)] + for f in tasks: + f.result() + except Exception as e: + print(f"✅ 错误案例多进程测试成功（预期失败）") + print(f" → 错误类型: {type(e).__name__}") + print(f" → 错误信息: {e}") + + # 修正案例测试 + print("\n[测试修正案例]") + try: + with ProcessPoolExecutor() as executor: + tasks = [executor.submit(FixedTask.process, i) for i in range(3)] + for f in tasks: + f.result() + print("✅ 修正案例多进程测试通过") + except Exception as e: + print(f"❌ 修正案例测试失败: {str(e)}") + +if __name__ == '__main__': + verify_serialization() + run_process_demo() + print("\n测试完成") -- Gitee From b9ea84c00733ed1abe93151da5a27c2cc3274d19 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 11:19:02 +0800 Subject: [PATCH 07/39] bisect-task: remove unnecessary statement method --- container/bisect/bisect-task.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 8175ec43..5f4d5124 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -79,29 +79,13 @@ class BisectTask: @staticmethod def _init_process_resources(config): """子进程资源初始化""" - global process_bisect_db, process_jobs_db, process_regression_db, process_client - + global process_bisect_db, process_client process_bisect_db = BisectDB( host=config['manticore_host'], port=config['manticore_port'], database=config['bisect_db'], pool_size=5 - ) - - process_jobs_db = BisectDB( - host=config['manticore_host'], - port=config['manticore_port'], - database=config['jobs_db'], - pool_size=3 - ) - - process_regression_db = BisectDB( - host=config['manticore_host'], - port=config['manticore_port'], - database=config['regression_db'], - pool_size=3 - ) - + ) process_client = ManticoreClient( host=config['manticore_host'], port=9308 @@ -109,14 +93,10 @@ class BisectTask: @staticmethod def _generate_task_path(config: dict, task: dict) -> str: - """进程安全的路径生成""" - pid = os.getpid() - tid = threading.get_ident() - + """进程安全的路径生成""" path = os.path.join( 'bisect_results', - f"proc-{pid}-thread-{tid}", - re.sub(r'[^\w\-]', '_', task.get('repo', 'unknown_repo'))[:32], + re.sub(r'[^\w\-]', '_', task.get('suite', 'unknown_repo'))[:32], datetime.now().strftime("%Y-%m-%d"), str(task['bad_job_id']), hashlib.md5(task['error_id'].encode()).hexdigest()[:8], -- Gitee From ad28be7d85b371a319a3b2f149bb053a7a025f0f Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 16:30:34 +0800 Subject: [PATCH 08/39] =?UTF-8?q?test:=20=E6=B7=BB=E5=8A=A0ManticoreClient?= =?UTF-8?q?=E7=9A=84=E5=AE=8C=E6=95=B4=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_manticore_simple.py | 142 +++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tests/test_manticore_simple.py diff --git a/tests/test_manticore_simple.py b/tests/test_manticore_simple.py new file mode 100644 index 00000000..832738a3 --- /dev/null +++ b/tests/test_manticore_simple.py @@ -0,0 +1,142 @@ +import pytest +from unittest.mock import patch, Mock +import json +import time +from manticore_simple import ManticoreClient + +@pytest.fixture +def default_client(): + """默认配置客户端 (localhost:9308)""" + return ManticoreClient() + +@pytest.fixture +def bisect_test_data(): + """bisect索引测试数据""" + return { + "bad_job_id": 123123124123, + "error_id": "stderr.eid../include/linux/thread_info.h:#:#:error", + "bisect_status": "wait" + } + +# 基础功能测试 +@patch('requests.post') +def test_insert_success(mock_post, default_client): + """测试插入文档成功""" + mock_post.return_value.status_code = 200 + doc = {"key": "value"} + + assert default_client.insert("bisect", 123123124123, doc) is True + + mock_post.assert_called_once_with( + "http://localhost:9308/insert", + json={"index": "bisect", "id": 123123124123, "doc": doc}, + timeout=3 + ) + +@patch('requests.post') +def test_replace_retry_success(mock_post, default_client, bisect_test_data): + """测试带重试的替换操作""" + # 第一次失败，第二次成功 + mock_post.side_effect = [ + Mock(status_code=500), + Mock(status_code=200) + ] + + assert default_client.replace_with_retry( + index="bisect", + id=123123124123, + document=bisect_test_data, + retries=2 + ) is True + + assert mock_post.call_count == 2 + expected_call = { + "index": "bisect", + "id": 123123124123, + "doc": bisect_test_data + } + for call in mock_post.call_args_list: + assert call[1]['json'] == expected_call + +# 端口配置验证 +@patch('requests.post') +def test_default_port_usage(mock_post, default_client): + """验证默认端口9308""" + mock_post.return_value.status_code = 200 + + default_client.replace("bisect", 1, {"test": "data"}) + + assert mock_post.call_args[0][0].startswith("http://localhost:9308/") + +@patch('requests.post') +def test_custom_port_config(): + """测试自定义端口配置""" + client = ManticoreClient(port=9309) + with patch('requests.post') as mock_post: + mock_post.return_value.status_code = 200 + client.insert("bisect", 1, {}) + assert ":9309" in mock_post.call_args[0][0] + +# 批量操作测试 +@patch('requests.post') +def test_bulk_replace_format(mock_post, default_client, bisect_test_data): + """验证批量操作数据格式""" + mock_post.return_value.status_code = 200 + docs = {123123124123: bisect_test_data} + + assert default_client.bulk_replace("bisect", docs) is True + + # 构建预期请求体 + expected_lines = [ + json.dumps({"replace": {"_index": "bisect", "_id": 123123124123}}), + json.dumps(bisect_test_data) + ] + mock_post.assert_called_once_with( + "http://localhost:9308/bulk", + data="\n".join(expected_lines), + headers={"Content-Type": "application/x-ndjson"}, + timeout=10 + ) + +# 异常处理测试 +@patch('requests.post') +def test_request_timeout(default_client): + """测试请求超时处理""" + with patch('requests.post') as mock_post: + mock_post.side_effect = TimeoutError + assert default_client.insert("bisect", 1, {}) is False + +@patch('requests.post') +def test_server_error_handling(mock_post, default_client): + """测试服务端错误处理""" + mock_post.return_value.status_code = 500 + assert default_client.replace("bisect", 1, {}) is False + +# 边界条件测试 +@patch('requests.post') +def test_large_document_handling(mock_post, default_client): + """测试大文档处理""" + mock_post.return_value.status_code = 200 + large_doc = {"data": "a" * 1000000} # 1MB数据 + + assert default_client.insert("bisect", 1, large_doc) is True + assert len(mock_post.call_args[1]['json']['doc']['data']) == 1000000 + +@patch('requests.post') +def test_empty_document(mock_post, default_client): + """测试空文档处理""" + mock_post.return_value.status_code = 200 + assert default_client.insert("bisect", 1, {}) is True + assert mock_post.call_args[1]['json']['doc'] == {} + +# ID生成测试 +def test_task_id_generation(): + """测试ID生成唯一性""" + from manticore_simple import generate_task_id + + id_set = set() + for _ in range(1000): + new_id = generate_task_id("123", "error1") + assert new_id not in id_set + assert 0 <= new_id <= 0x7FFFFFFFFFFFFFFF # 63位正整数验证 + id_set.add(new_id) -- Gitee From ffedfe4784ee47ca6a10d17a5841b33c17da2f4a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 16:38:45 +0800 Subject: [PATCH 09/39] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E9=83=A8=E5=88=86=E6=9B=B4=E6=96=B0=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/manticore_simple.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/manticore_simple.py b/lib/manticore_simple.py index 7d6177ab..5f399533 100644 --- a/lib/manticore_simple.py +++ b/lib/manticore_simple.py @@ -15,6 +15,10 @@ class ManticoreClient: """替换文档""" return self._request("replace", index, id, document) + def update(self, index: str, id: int, document: dict) -> bool: + """部分更新文档（仅修改指定字段）""" + return self._request("update", index, id, document) + def replace_with_retry(self, index: str, id: int, document: dict, retries: int = 3) -> bool: """带重试机制的替换操作""" for i in range(retries): -- Gitee From 8e4da2f980d730aa169014813d4b84d7d2f63fb0 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 16:48:51 +0800 Subject: [PATCH 10/39] =?UTF-8?q?test:=20=E6=B7=BB=E5=8A=A0ManticoreClient?= =?UTF-8?q?=E5=9F=BA=E7=A1=80=E5=8A=9F=E8=83=BD=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/bisect-tasks/test_manticore_simple.py | 113 ++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tests/bisect-tasks/test_manticore_simple.py diff --git a/tests/bisect-tasks/test_manticore_simple.py b/tests/bisect-tasks/test_manticore_simple.py new file mode 100644 index 00000000..be6c9495 --- /dev/null +++ b/tests/bisect-tasks/test_manticore_simple.py @@ -0,0 +1,113 @@ +import pytest +from unittest.mock import patch, Mock +import json +import os +import sys +sys.path.append((os.environ['CCI_SRC']) + '/lib') +from manticore_simple import ManticoreClient + + +@pytest.fixture +def default_client(): + """默认配置客户端 (localhost:9308)""" + return ManticoreClient() + +@pytest.fixture +def bisect_test_data(): + """bisect索引测试数据""" + return { + "bad_job_id": 123123124123, + "error_id": "stderr.eid../include/linux/thread_info.h:#:#:error", + "bisect_status": "wait" + } + +# 基础功能测试 +@patch('requests.post') +def test_insert_success(mock_post, default_client): + """测试插入文档成功""" + mock_post.return_value.status_code = 200 + doc = {"key": "value"} + + assert default_client.insert("bisect", 123123124123, doc) is True + + mock_post.assert_called_once_with( + "http://localhost:9308/insert", + json={"index": "bisect", "id": 123123124123, "doc": doc}, + timeout=3 + ) + +@patch('requests.post') +def test_replace_retry_success(mock_post, default_client, bisect_test_data): + """测试带重试的替换操作""" + # 第一次失败，第二次成功 + mock_post.side_effect = [ + Mock(status_code=500), + Mock(status_code=200) + ] + + assert default_client.replace_with_retry( + index="bisect", + id=123123124123, + document=bisect_test_data, + retries=2 + ) is True + + assert mock_post.call_count == 2 + expected_call = { + "index": "bisect", + "id": 123123124123, + "doc": bisect_test_data + } + for call in mock_post.call_args_list: + assert call[1]['json'] == expected_call + +# 端口配置验证 +@patch('requests.post') +def test_default_port_usage(mock_post, default_client): + """验证默认端口9308""" + mock_post.return_value.status_code = 200 + + default_client.replace("bisect", 1, {"test": "data"}) + + assert mock_post.call_args[0][0].startswith("http://localhost:9308/") + + +# 批量操作测试 +@patch('requests.post') +def test_bulk_replace_format(mock_post, default_client, bisect_test_data): + """验证批量操作数据格式""" + mock_post.return_value.status_code = 200 + docs = {123123124123: bisect_test_data} + + assert default_client.bulk_replace("bisect", docs) is True + + # 构建预期请求体 + expected_lines = [ + json.dumps({"replace": {"_index": "bisect", "_id": 123123124123}}), + json.dumps(bisect_test_data) + ] + mock_post.assert_called_once_with( + "http://localhost:9308/bulk", + data="\n".join(expected_lines), + headers={"Content-Type": "application/x-ndjson"}, + timeout=10 + ) + + +# 边界条件测试 +@patch('requests.post') +def test_large_document_handling(mock_post, default_client): + """测试大文档处理""" + mock_post.return_value.status_code = 200 + large_doc = {"data": "a" * 1000000} # 1MB数据 + + assert default_client.insert("bisect", 1, large_doc) is True + assert len(mock_post.call_args[1]['json']['doc']['data']) == 1000000 + +@patch('requests.post') +def test_empty_document(mock_post, default_client): + """测试空文档处理""" + mock_post.return_value.status_code = 200 + assert default_client.insert("bisect", 1, {}) is True + assert mock_post.call_args[1]['json']['doc'] == {} + -- Gitee From bf5fc9b34651df374240161ccd12b4d2256c6708 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 16:48:56 +0800 Subject: [PATCH 11/39] =?UTF-8?q?test:=20=E6=B7=BB=E5=8A=A0ManticoreClient?= =?UTF-8?q?=E7=9A=84update=E6=96=B9=E6=B3=95=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/bisect-tasks/test_manticore_simple.py | 29 ++++ tests/test_manticore_simple.py | 142 -------------------- 2 files changed, 29 insertions(+), 142 deletions(-) delete mode 100644 tests/test_manticore_simple.py diff --git a/tests/bisect-tasks/test_manticore_simple.py b/tests/bisect-tasks/test_manticore_simple.py index be6c9495..b6a71d9b 100644 --- a/tests/bisect-tasks/test_manticore_simple.py +++ b/tests/bisect-tasks/test_manticore_simple.py @@ -36,6 +36,35 @@ def test_insert_success(mock_post, default_client): timeout=3 ) +@patch('requests.post') +def test_update_success(mock_post, default_client): + """测试部分更新文档成功""" + mock_post.return_value.status_code = 200 + update_fields = {"bisect_status": "completed"} + + assert default_client.update("bisect", 123456, update_fields) is True + + mock_post.assert_called_once_with( + "http://localhost:9308/update", + json={"index": "bisect", "id": 123456, "doc": update_fields}, + timeout=3 + ) + +@patch('requests.post') +def test_update_partial_fields(mock_post, default_client): + """测试仅更新部分字段""" + mock_post.return_value.status_code = 200 + partial_update = {"status": "done", "progress": 100} + + assert default_client.update("bisect", 789, partial_update) is True + assert mock_post.call_args[1]['json']['doc'] == partial_update + +@patch('requests.post') +def test_update_failure(mock_post, default_client): + """测试更新失败场景""" + mock_post.return_value.status_code = 500 + assert default_client.update("bisect", 456, {"key": "value"}) is False + @patch('requests.post') def test_replace_retry_success(mock_post, default_client, bisect_test_data): """测试带重试的替换操作""" diff --git a/tests/test_manticore_simple.py b/tests/test_manticore_simple.py deleted file mode 100644 index 832738a3..00000000 --- a/tests/test_manticore_simple.py +++ /dev/null @@ -1,142 +0,0 @@ -import pytest -from unittest.mock import patch, Mock -import json -import time -from manticore_simple import ManticoreClient - -@pytest.fixture -def default_client(): - """默认配置客户端 (localhost:9308)""" - return ManticoreClient() - -@pytest.fixture -def bisect_test_data(): - """bisect索引测试数据""" - return { - "bad_job_id": 123123124123, - "error_id": "stderr.eid../include/linux/thread_info.h:#:#:error", - "bisect_status": "wait" - } - -# 基础功能测试 -@patch('requests.post') -def test_insert_success(mock_post, default_client): - """测试插入文档成功""" - mock_post.return_value.status_code = 200 - doc = {"key": "value"} - - assert default_client.insert("bisect", 123123124123, doc) is True - - mock_post.assert_called_once_with( - "http://localhost:9308/insert", - json={"index": "bisect", "id": 123123124123, "doc": doc}, - timeout=3 - ) - -@patch('requests.post') -def test_replace_retry_success(mock_post, default_client, bisect_test_data): - """测试带重试的替换操作""" - # 第一次失败，第二次成功 - mock_post.side_effect = [ - Mock(status_code=500), - Mock(status_code=200) - ] - - assert default_client.replace_with_retry( - index="bisect", - id=123123124123, - document=bisect_test_data, - retries=2 - ) is True - - assert mock_post.call_count == 2 - expected_call = { - "index": "bisect", - "id": 123123124123, - "doc": bisect_test_data - } - for call in mock_post.call_args_list: - assert call[1]['json'] == expected_call - -# 端口配置验证 -@patch('requests.post') -def test_default_port_usage(mock_post, default_client): - """验证默认端口9308""" - mock_post.return_value.status_code = 200 - - default_client.replace("bisect", 1, {"test": "data"}) - - assert mock_post.call_args[0][0].startswith("http://localhost:9308/") - -@patch('requests.post') -def test_custom_port_config(): - """测试自定义端口配置""" - client = ManticoreClient(port=9309) - with patch('requests.post') as mock_post: - mock_post.return_value.status_code = 200 - client.insert("bisect", 1, {}) - assert ":9309" in mock_post.call_args[0][0] - -# 批量操作测试 -@patch('requests.post') -def test_bulk_replace_format(mock_post, default_client, bisect_test_data): - """验证批量操作数据格式""" - mock_post.return_value.status_code = 200 - docs = {123123124123: bisect_test_data} - - assert default_client.bulk_replace("bisect", docs) is True - - # 构建预期请求体 - expected_lines = [ - json.dumps({"replace": {"_index": "bisect", "_id": 123123124123}}), - json.dumps(bisect_test_data) - ] - mock_post.assert_called_once_with( - "http://localhost:9308/bulk", - data="\n".join(expected_lines), - headers={"Content-Type": "application/x-ndjson"}, - timeout=10 - ) - -# 异常处理测试 -@patch('requests.post') -def test_request_timeout(default_client): - """测试请求超时处理""" - with patch('requests.post') as mock_post: - mock_post.side_effect = TimeoutError - assert default_client.insert("bisect", 1, {}) is False - -@patch('requests.post') -def test_server_error_handling(mock_post, default_client): - """测试服务端错误处理""" - mock_post.return_value.status_code = 500 - assert default_client.replace("bisect", 1, {}) is False - -# 边界条件测试 -@patch('requests.post') -def test_large_document_handling(mock_post, default_client): - """测试大文档处理""" - mock_post.return_value.status_code = 200 - large_doc = {"data": "a" * 1000000} # 1MB数据 - - assert default_client.insert("bisect", 1, large_doc) is True - assert len(mock_post.call_args[1]['json']['doc']['data']) == 1000000 - -@patch('requests.post') -def test_empty_document(mock_post, default_client): - """测试空文档处理""" - mock_post.return_value.status_code = 200 - assert default_client.insert("bisect", 1, {}) is True - assert mock_post.call_args[1]['json']['doc'] == {} - -# ID生成测试 -def test_task_id_generation(): - """测试ID生成唯一性""" - from manticore_simple import generate_task_id - - id_set = set() - for _ in range(1000): - new_id = generate_task_id("123", "error1") - assert new_id not in id_set - assert 0 <= new_id <= 0x7FFFFFFFFFFFFFFF # 63位正整数验证 - id_set.add(new_id) -- Gitee From 0cdee3691fc01acb2a3a1f75c9a5b462e3332ca8 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 20 May 2025 17:41:25 +0800 Subject: [PATCH 12/39] remove unnecessary code Sign-off-by: jacknichao --- container/bisect/bisect-task.py | 49 ++++----------------------------- 1 file changed, 6 insertions(+), 43 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 5f4d5124..c92bfec6 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -118,34 +118,6 @@ class BisectTask: self._cleanup_interrupted_tasks() sys.exit(1) - def get_task_result_root(self, task): - """生成带日期的扁平化存储路径""" - # 安全处理仓库名（替换特殊字符） - repo_name = re.sub(r'[^\w\-]', '_', task.get('repo', 'unknown_repo'))[:32] - - # 获取任务时间（使用当前时间作为默认） - create_time = task.get('create_time', time.time()) - date_str = datetime.fromtimestamp(create_time).strftime("%Y-%m-%d") - - # 压缩错误ID为8位哈希 - # TODO remove - error_id_hash = hashlib.md5(task['error_id'].encode()).hexdigest()[:8] - - # 构建路径 - path = os.path.join( - 'bisect_results', - repo_name, - date_str, - str(task['bad_job_id']), - error_id_hash, - str(task['id']) - ) - - # 创建目录并返回绝对路径 - abs_path = os.path.abspath(path) - os.makedirs(abs_path, exist_ok=True, mode=0o755) - return abs_path - @staticmethod def _process_single_task(config: dict, task: dict): """静态任务处理方法""" @@ -168,7 +140,7 @@ class BisectTask: task["bisect_status"] = "processing" # Convert to Manticore SQL update - if process_client.replace("bisect", task_id, task): + if process_client.update("bisect", task_id, task): logger.info(f"开始处理任务 | ID: {task_id}") # 准备任务数据 @@ -194,7 +166,7 @@ class BisectTask: } # 使用重试机制更新结果 - if not process_client.replace_with_retry("bisect", task_id, complete_doc): + if not process_client.update("bisect", task_id, complete_doc): logger.error(f"结果更新失败 | ID: {task_id}") else: # 更新回归数据 @@ -208,7 +180,7 @@ class BisectTask: "last_error": "Bisect execution failed", "updated_at": int(time.time()) } - process_client.replace_with_retry("bisect", task_id, fail_doc) + process_client.update("bisect", task_id, fail_doc) logger.error(f"任务执行失败 | ID: {task_id}") except Exception as e: @@ -218,7 +190,7 @@ class BisectTask: "last_error": str(e)[:200], # 限制错误信息长度 "updated_at": int(time.time()) } - process_client.replace_with_retry("bisect", task_id, error_doc) + process_client.update("bisect", task_id, error_doc) logger.error(f"任务异常 | ID: {task_id} | 错误: {str(e)}") logger.error(traceback.format_exc()) @@ -340,14 +312,6 @@ class BisectTask: logger.error(f"Unknown error: {str(e)}") return False - def get_job_info_from_jobs(self, job_id): - job_id = int(job_id) - job_json = self.jobs_db.execute_query("SELECT j FROM jobs WHERE id = %s", (job_id,)) - if not job_json: - return {} - first_row = job_json[0] - return json.loads(first_row.get('j', {})) - def set_priority_level(self, job_info: dict) -> int: """ """ @@ -677,10 +641,9 @@ class BisectTask: # TODO: 增加判断 AND submit_time > NOW() - INTERVAL 7 DAY # Perf monitor sql_failure = """ - SELECT id, errid as errid, j.suite as suite, j.category as category + SELECT id, errid as errid, j.suite as suite FROM jobs - WHERE j.job_health = 'abort' - AND j.stats IS NOT NULL + WHERE j.errid IS NOT NULL ORDER BY id DESC """ -- Gitee From c577e45877847daabddf61816c6f5f6e80ce649c Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 22 May 2025 11:23:04 +0800 Subject: [PATCH 13/39] bisect-task: fix manticore query and SQL Make sure id is int Only select job which job stage is finish from db Sign-off-by: jacknichao --- container/bisect/bisect-task.py | 15 ++++++++----- lib/bisect_database.py | 39 --------------------------------- 2 files changed, 10 insertions(+), 44 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index c92bfec6..37606506 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -124,7 +124,7 @@ class BisectTask: try: global process_bisect_db, process_client - task_id = task['id'] + task_id = int(task['id']) task_result_root = BisectTask._generate_task_path(config, task) try: @@ -638,15 +638,19 @@ class BisectTask: :return: A list of processed tasks that match the white list criteria. """ # Define SQL for PKGBUILD tasks - # TODO: 增加判断 AND submit_time > NOW() - INTERVAL 7 DAY + # TODO: 不应该添加重复的 bad_job_id, 避免找不到 _url 的内容被查找 + # 增加判断 AND submit_time > NOW() - INTERVAL 7 DAY # Perf monitor sql_failure = """ - SELECT id, errid as errid, j.suite as suite - FROM jobs + SELECT id, errid as errid, j.suite as suite, full_text_kv as text + FROM jobs WHERE j.errid IS NOT NULL + AND j.program IS NOT NULL + AND MATCH('job_health=abort job_stage=finish') ORDER BY id DESC + LIMIT 1000 """ - + # select id, errid, full_text_kv from jobs where match('job_health=abort job_stage=finish') and j.errid is not null and j.program is not null order by id desc limit 1000 # Define the white list of error IDs sql_error_id = """ SELECT errid @@ -655,6 +659,7 @@ class BisectTask: AND valid = 'true' ORDER BY id DESC """ + # TODO: 一个 bad_job_id 应该和白名单判断一次 errid_white_list_raw = self.regression_db.execute_query(sql_error_id) errid_white_list = {item['errid'] for item in errid_white_list_raw} if errid_white_list_raw else set() diff --git a/lib/bisect_database.py b/lib/bisect_database.py index ab12832b..5bf30b56 100644 --- a/lib/bisect_database.py +++ b/lib/bisect_database.py @@ -4,45 +4,6 @@ import os from log_config import logger, StructuredLogger import mysql.connector -from py_bisect import GitBisect - -def _process_task_worker(task_id: int): - """独立进程任务处理函数""" - logger = StructuredLogger() # 每个进程独立实例 - - # 每个进程独立数据库连接 - local_db = BisectDB( - host=os.getenv('MANTICORE_HOST'), - port=os.getenv('MANTICORE_PORT'), - database="bisect", - pool_size=1 - ) - - try: - logger.info(f"Process {os.getpid()} handling task {task_id}") - - # 获取任务数据 - task = local_db.execute_query( - "SELECT * FROM bisect WHERE id = %s", - (task_id,) - )[0] - - # 执行核心逻辑 - gb = GitBisect() - result = gb.find_first_bad_commit(task) - - # 更新状态 - local_db.execute_update( - "UPDATE bisect SET status='completed' WHERE id=%s", - (task_id,) - ) - return {'id': task_id, 'status': 'success'} - except Exception as e: - error_msg = f"Task {task_id} failed: {str(e)}" - logger.error(error_msg) - return {'id': task_id, 'status': 'failed', 'error': error_msg} - finally: - local_db.close() from generic_sql_client import GenericSQLClient from mysql.connector import Error import json -- Gitee From c0628da191e0cd838af497f5265d983a673b9e36 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 22 May 2025 11:27:00 +0800 Subject: [PATCH 14/39] bisect-task: smaller connection pool Sign-off-by: jacknichao --- container/bisect/bisect-task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 37606506..feecca28 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -84,7 +84,7 @@ class BisectTask: host=config['manticore_host'], port=config['manticore_port'], database=config['bisect_db'], - pool_size=5 + pool_size=1 ) process_client = ManticoreClient( host=config['manticore_host'], @@ -248,7 +248,7 @@ class BisectTask: host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="jobs", - pool_size=3 + pool_size=2 ) logger.info(f"主进程 jobs DB 连接 ID: {id(self.jobs_db)}") @@ -256,7 +256,7 @@ class BisectTask: host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="bisect", - pool_size=3 + pool_size=2 ) logger.info(f"主进程 bisect DB 连接 ID: {id(self.bisect_db)}") @@ -265,7 +265,7 @@ class BisectTask: host=os.environ.get('MANTICORE_HOST', 'localhost'), port=os.environ.get('MANTICORE_PORT', '9306'), database="regression", - pool_size=3 + pool_size=2 ) logger.info(f"主进程 RegressionDB 连接 ID: {id(self.regression_db)}") -- Gitee From 88d7a5ead658f8a543608bd3e66a1f9766326f71 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 27 May 2025 11:26:53 +0800 Subject: [PATCH 15/39] bisect-task: ignore makepkg without _url --- container/bisect/bisect-task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index feecca28..2960f0de 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -645,7 +645,7 @@ class BisectTask: SELECT id, errid as errid, j.suite as suite, full_text_kv as text FROM jobs WHERE j.errid IS NOT NULL - AND j.program IS NOT NULL + AND (j.program.makepkg._url IS NOT NULL OR j.ss IS NOT NULL) AND MATCH('job_health=abort job_stage=finish') ORDER BY id DESC LIMIT 1000 -- Gitee From 742a24a4ee36b245f37f00384be5997bf8efdb61 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 10:53:51 +0800 Subject: [PATCH 16/39] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=A0=B9?= =?UTF-8?q?=E6=8D=AE=E7=8A=B6=E6=80=81=E6=9F=A5=E8=AF=A2=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=9A=84=E9=80=9A=E7=94=A8=E6=8E=A5=E5=8F=A3=E5=92=8C=20URL=20?= =?UTF-8?q?=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 2960f0de..51dbc672 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -780,6 +780,50 @@ class ListBisectTasksAPI(MethodView): return jsonify({"error": "内部服务器错误"}), 500 +class ListTasksByStatusAPI(MethodView): + def __init__(self): + self.bisect_db = BisectDB( + host=os.environ.get('MANTICORE_HOST', 'localhost'), + port=os.environ.get('MANTICORE_PORT', '9306'), + database="bisect", + pool_size=15 + ) + + def get(self): + try: + # 获取查询参数，默认为 'completed' + status = request.args.get('status', 'completed') + + # 查询指定状态的任务 + sql = f""" + SELECT id, bad_job_id, error_id, bisect_status + FROM bisect + WHERE bisect_status = '{status}' + ORDER BY id DESC + """ + tasks = self.bisect_db.execute_query(sql) + + # 格式化输出 + formatted_tasks = [] + for task in tasks: + formatted_tasks.append({ + "TASK ID": task['id'], + "BAD JOB ID": task['bad_job_id'], + "ERROR ID": task['error_id'], + "STATUS": {'wait': 'wait', 'processing': 'processing', 'completed': 'finish', 'failed': 'failed'}.get(task['bisect_status'], 'unknown'), + }) + + response_data = { + "total": len(formatted_tasks), + "tasks": formatted_tasks + } + return json.dumps(response_data, indent=2, ensure_ascii=False), 200, {'Content-Type': 'application/json; charset=utf-8'} + + except Exception as e: + logger.error(f"获取任务列表失败: {str(e)}") + return jsonify({"error": "内部服务器错误"}), 500 + + class DeleteFailedTasksAPI(MethodView): def __init__(self): self.bisect_db = BisectDB( @@ -814,6 +858,7 @@ def run_flask(): """使用生产级 WSGI 服务器，带开发服务器回退""" app.add_url_rule('/new_bisect_task', view_func=BisectAPI.as_view('bisect_api')) app.add_url_rule('/list_bisect_tasks', view_func=ListBisectTasksAPI.as_view('list_bisect_tasks')) + app.add_url_rule('/list_tasks_by_status', view_func=ListTasksByStatusAPI.as_view('list_tasks_by_status')) app.add_url_rule('/delete_failed_tasks', view_func=DeleteFailedTasksAPI.as_view('delete_failed_tasks')) port = int(os.environ.get('BISECT_API_PORT', 9999)) -- Gitee From faa201cb355b85abfa35ba4b12dc996e7b9ae1c4 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 10:54:17 +0800 Subject: [PATCH 17/39] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=9C=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E5=90=8D=E7=A7=B0=20'process=5Fregression=5F?= =?UTF-8?q?db'=20=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 51dbc672..29e7767f 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -79,13 +79,19 @@ class BisectTask: @staticmethod def _init_process_resources(config): """子进程资源初始化""" - global process_bisect_db, process_client + global process_bisect_db, process_client, process_regression_db process_bisect_db = BisectDB( host=config['manticore_host'], port=config['manticore_port'], database=config['bisect_db'], pool_size=1 - ) + ) + process_regression_db = BisectDB( + host=config['manticore_host'], + port=config['manticore_port'], + database=config['regression_db'], + pool_size=1 + ) process_client = ManticoreClient( host=config['manticore_host'], port=9308 -- Gitee From 8190c06652425eeb994bede7cfc4aa7d6f185add Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 13:34:18 +0800 Subject: [PATCH 18/39] =?UTF-8?q?feat:=20=E4=BD=BF=E7=94=A8=20ManticoreCli?= =?UTF-8?q?ent=20=E7=9A=84=20search=20=E6=96=B9=E6=B3=95=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index 29e7767f..d71bdfc9 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -134,10 +134,13 @@ class BisectTask: task_result_root = BisectTask._generate_task_path(config, task) try: - # 检查任务状态 - status_check = process_bisect_db.execute_query( - f"SELECT bisect_status FROM bisect WHERE id = {task_id} AND bisect_status = 'wait'" + # 使用 ManticoreClient 的 search 方法检查任务状态 + status_check = process_client.search( + table="bisect", + query={"match": {"id": task_id, "bisect_status": "wait"}}, + limit=1 ) + if not status_check: logger.warning(f"跳过无效任务 | ID: {task_id}") return -- Gitee From 9b64abfec6bde398fda50dda497fb21e39169d1b Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 13:37:54 +0800 Subject: [PATCH 19/39] =?UTF-8?q?feat:=20=E4=BD=BF=E7=94=A8=20ManticoreCli?= =?UTF-8?q?ent=20=E6=9B=B4=E6=96=B0=20update=5Fregression=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 76 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index d71bdfc9..a0210231 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -466,7 +466,7 @@ class BisectTask: logger.error(f"结果处理异常: {str(e)}") def update_regression(self, task, result): - """Update regression database with bisect results""" + """使用 ManticoreClient 更新回归数据库""" try: # 参数校验 if not task.get('error_id') or not task.get('bad_job_id'): @@ -479,12 +479,18 @@ class BisectTask: error_id = task['error_id'].replace("'", "''") # 转义单引号 # 检查是否已存在有效记录 - existing = process_regression_db.execute_query( - f"SELECT id, bisect_count, related_jobs " - f"FROM regression " - f"WHERE record_type = 'errid' " - f" AND errid = '{error_id}' " - f" AND valid = 'true'" + existing = process_client.search( + table="regression", + query={ + "bool": { + "must": [ + {"match": {"record_type": "errid"}}, + {"match": {"errid": error_id}}, + {"match": {"valid": "true"}} + ] + } + }, + limit=1 ) if not existing: @@ -492,43 +498,35 @@ class BisectTask: new_id = int(f"{current_time}{randint(1000,9999)}") # 生成唯一ID category = result.get('category', 'unknown').replace("'", "''") related_jobs_json = json.dumps([bad_job_id]) # 初始化为数组 - # TODO replace by manticore - insert_sql = f""" - INSERT INTO regression - (id, record_type, errid, category, - first_seen, last_seen, bisect_count, - related_jobs, valid) - VALUES ( - {new_id}, - 'errid', - '{error_id}', - '{category}', - {current_time}, - {current_time}, - 1, - '{related_jobs_json}', - 'true' - ) - """ - self.regression_db.execute_write(insert_sql) + + new_record = { + "id": new_id, + "record_type": "errid", + "errid": error_id, + "category": category, + "first_seen": current_time, + "last_seen": current_time, + "bisect_count": 1, + "related_jobs": related_jobs_json, + "valid": "true" + } + + if not process_client.insert("regression", new_id, new_record): + logger.error(f"插入新记录失败 | ErrorID: {error_id}") else: # 更新现有记录 record = existing[0] new_count = record['bisect_count'] + 1 record_id = record['id'] - - update_sql = f""" - UPDATE regression - SET bisect_count = {new_count}, - last_seen = {current_time}, - related_jobs = JSON_ARRAY_APPEND( - related_jobs, - '$', - '{bad_job_id}' - ) - WHERE id = %s - """ - process_regression_db.execute_update(update_sql, (record_id,)) + + update_record = { + "bisect_count": new_count, + "last_seen": current_time, + "related_jobs": json.dumps(record['related_jobs'] + [bad_job_id]) + } + + if not process_client.update("regression", record_id, update_record): + logger.error(f"更新记录失败 | ErrorID: {error_id}") logger.info(f"Regression updated | ErrorID: {error_id}") -- Gitee From a38e777d7306e39ffbebca0e710f62eb4783468c Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 13:41:48 +0800 Subject: [PATCH 20/39] refactor: migrate bisect-task to use ManticoreClient --- container/bisect/bisect-task.py | 18 +------------- lib/manticore_simple.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index a0210231..e08c9cbd 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -79,19 +79,7 @@ class BisectTask: @staticmethod def _init_process_resources(config): """子进程资源初始化""" - global process_bisect_db, process_client, process_regression_db - process_bisect_db = BisectDB( - host=config['manticore_host'], - port=config['manticore_port'], - database=config['bisect_db'], - pool_size=1 - ) - process_regression_db = BisectDB( - host=config['manticore_host'], - port=config['manticore_port'], - database=config['regression_db'], - pool_size=1 - ) + global process_client process_client = ManticoreClient( host=config['manticore_host'], port=9308 @@ -278,13 +266,9 @@ class BisectTask: ) logger.info(f"主进程 RegressionDB 连接 ID: {id(self.regression_db)}") - # 子进程配置 self._config = { "manticore_host": os.environ.get('MANTICORE_HOST', 'localhost'), "manticore_port": os.environ.get('MANTICORE_PORT', '9306'), - "bisect_db": "bisect", - "jobs_db": "jobs", - "regression_db": "regression" } self.process_pool = ProcessPoolExecutor( diff --git a/lib/manticore_simple.py b/lib/manticore_simple.py index 5f399533..c0a9d3e9 100644 --- a/lib/manticore_simple.py +++ b/lib/manticore_simple.py @@ -19,6 +19,48 @@ class ManticoreClient: """部分更新文档（仅修改指定字段）""" return self._request("update", index, id, document) + def search(self, + table: str, + query: dict, + limit: int = 100, + options: Optional[dict] = None) -> Optional[List[Dict]]: + """ + Manticore 官方标准搜索方法 + + :param table: 要查询的表名 + :param query: 查询 DSL (支持 query_string/match/bool 等) + :param limit: 返回结果数量 (默认100) + :param options: 高级选项 (scroll/列过滤等) + :return: 文档内容字典列表 + """ + try: + request_body = { + "table": table, + "query": query, + "limit": limit + } + + if options: + request_body["options"] = options + + resp = requests.post( + f"{self.base_url}/search", + json=request_body, + timeout=5 + ) + + if resp.status_code != 200: + return None + + result = resp.json() + return [ + hit.get('_source', {}) + for hit in result.get('hits', {}).get('hits', []) + ] + + except requests.exceptions.RequestException: + return None + def replace_with_retry(self, index: str, id: int, document: dict, retries: int = 3) -> bool: """带重试机制的替换操作""" for i in range(retries): -- Gitee From baca6e090a04e61fd19ec815c00d85e1fe58498a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 13:58:41 +0800 Subject: [PATCH 21/39] =?UTF-8?q?chore:=20=E6=9B=B4=E6=96=B0=20bisect=20?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=95=B0=E6=8D=AE=E5=BA=93=E8=AE=BE=E8=AE=A1?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E7=BB=93=E6=9E=84=E5=92=8C=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/readme.md | 41 ++++++++------------------------------ 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/container/bisect/readme.md b/container/bisect/readme.md index 391f5fcb..8a229112 100644 --- a/container/bisect/readme.md +++ b/container/bisect/readme.md @@ -48,38 +48,13 @@ submit-jobs # Bisect 任务数据库设计文档 -## 主表结构 (`bisect_tasks`) - -| 字段名称 | 类型 | 是否可为空 | 默认值 | 描述 | 索引建议 | -|----------------------|------------|------------|--------|----------------------------------------------------------------------|-------------------| -| id | keyword | NOT NULL | - | 任务唯一标识符（UUIDv4） | 主键(PK) | -| bad_job_id | keyword | NOT NULL | - | 需要二分法分析的原始Job ID | 普通索引(IDX_1) | -| error_id | keyword | NOT NULL | - | 错误标识符（需配合去重机制） | 唯一索引(UQ_1) | -| bisect_metrics | keyword | NULL | - | 性能指标名称（仅性能测试场景使用） | 条件索引(IDX_5) | -| priority_level | integer | NOT NULL | 1 | 优先级级别：0-低/1-中/2-高 | 排序索引(IDX_8) | -| bisect_status | keyword | NOT NULL | pending| 任务状态枚举：pending/running/paused/success/failed/retrying | 状态索引(IDX_2) | -| first_bad_commit | keyword | NULL | - | 通过bisect定位的首个问题提交 | 覆盖索引(IDX_6) | -| project | keyword | NOT NULL | - | 所属项目名称（格式：org/repo） | 组合索引(IDX_3) | -| pkgbuild_repo | keyword | NULL | - | 软件包构建仓库地址 | - | -| git_url | keyword | NULL | - | 上游代码仓库URL | - | -| bisect_suite | keyword | NOT NULL | - | 测试套件标识符（用于跨任务分析） | 组合索引(IDX_3) | -| first_bad_job_id | keyword | NULL | - | 问题提交对应的首个失败Job ID | 外键索引(FK_1) | -| first_result_root | text | NULL | - | 首次失败结果存储路径（OSS路径格式） | - | -| work_dir | text | NOT NULL | - | 任务工作目录（格式：/bisect/yyyy-mm-uuid/） | 前缀索引(IDX_4) | -| start_time | date | NULL | - | 任务实际开始时间（ISO8601） | 范围索引(IDX_7) | -| end_time | date | NULL | - | 任务结束时间（成功/失败时更新） | 范围索引(IDX_7) | -| commit_history | text | NOT NULL | - | 提交范围（格式：commit1...commit2） | - | -| timeout | integer | NOT NULL | 3600 | 超时阈值（秒） | - | - -## 嵌套表 (`job_commit_mappings`) - -| 字段名称 | 类型 | 是否可为空 | 描述 | 索引建议 | -|-----------------|------------|------------|----------------------------------------------------------------------|-------------------| -| job_id | keyword | NOT NULL | 关联的CI Job标识符 | 联合主键(PK_2) | -| commit_hash | keyword | NOT NULL | Git提交哈希（完整40位） | 覆盖索引(IDX_9) | -| metric_value | text | NULL | 性能指标数值（JSON格式存储多维度指标） | - | -| result_root | keyword | NOT NULL | 结果存储路径（OSS路径） | 前缀索引(IDX_10) | -| status | keyword | NOT NULL | 判定状态：bad/good/skip | 位图索引(BIT_1) | -| timestamp | date | NOT NULL | 任务执行时间戳（精确到毫秒） | 排序索引(IDX_11) | +## 主表结构 (`bisect`) +CCI_SRC/sbin/manti-table-bisect.sql + + +## 回归检查表 (`regression`) +CCI_SRC/sbin/manti-table-regression.sql + + -- Gitee From da69ccaf0838c313f5772517a6654dd9e66881ff Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 13:58:42 +0800 Subject: [PATCH 22/39] =?UTF-8?q?docs:=20=E6=9B=B4=E6=96=B0=20bisect=5Ftas?= =?UTF-8?q?k=20=E6=96=87=E6=A1=A3=E4=BB=A5=E5=A2=9E=E5=BC=BA=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=92=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E8=AE=BE=E8=AE=A1?= =?UTF-8?q?=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/readme.md | 96 ++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/container/bisect/readme.md b/container/bisect/readme.md index 8a229112..6b5c29f4 100644 --- a/container/bisect/readme.md +++ b/container/bisect/readme.md @@ -1,60 +1,76 @@ -## bisect submit +# Bisect 任务管理 -### way1: - run in special testbox +## 概述 -ensure security: -- account/token -- visit services +`bisect_task` 是一个用于自动化处理 bisect 任务的服务。它通过与 Manticore 数据库交互，管理任务的提交、处理和状态更新。 +## 功能 -common testbox: - submit upload lkp-tests tar ball, run in testbox +- **任务提交**：通过 API 接口提交新的 bisect 任务。 +- **任务处理**：自动从数据库中获取待处理任务，并执行 bisect 操作。 +- **状态更新**：实时更新任务状态，并将结果存储到数据库中。 +- **回归检查**：更新回归数据库以记录任务的执行结果。 -secure testbox: - submit NO UPLOAD lkp-tests tar ball - run only selected list of programs - run in a dedicated host machine, FW can visit services +## 数据库设计 +### Bisect 表结构 -### way2: -submit job1 - submit job2, reuse same account info -problem: consumes his credit / machine time +`bisect` 表用于存储所有的 bisect 任务信息。其结构如下： -### way3: -compass ci container/ run as service -refer to delimiter service - system bisect account +- **`id`**: `bigint`，任务的唯一标识符。 +- **`bad_job_id`**: `string`，标记为不良的作业ID。 +- **`error_id`**: `string`，任务相关的错误标识符。 +- **`bisect_status`**: `string`，任务的当前状态（如 `wait`、`processing`、`completed`、`failed`）。 +- **其他字段**：用于存储任务的详细信息。 - provide API/new-bisect-task - add to ES - - loop: - consume one task from ES - fork process, start bisect - bisect step - submit job - change bisect-task state=finish +### 回归检查表结构 -submit-jobs +`regression` 表用于存储回归检查的结果。其结构如下： -### regression db -### bisect task queue -### bisect tasks management and dashboard +- **`id`**: `bigint`，唯一标识符。 +- **`record_type`**: `string`，记录类型，通常为 `errid`。 +- **`errid`**: `string`，错误标识符。 +- **`category`**: `string`，错误类别。 +- **其他字段**：用于存储回归检查的详细信息。 - bisect task name - suite start_time/run_time step group_id +## 任务处理流程 -# Bisect 任务数据库设计文档 +### 生产者流程 -## 主表结构 (`bisect`) -CCI_SRC/sbin/manti-table-bisect.sql +- **任务发现**：定期从 `jobs` 数据库中获取新的故障任务。 +- **任务过滤**：根据错误标识符的白名单过滤任务。 +- **任务提交**：将符合条件的任务提交到 `bisect` 数据库。 +### 消费者流程 -## 回归检查表 (`regression`) -CCI_SRC/sbin/manti-table-regression.sql +- **任务获取**：从 `bisect` 数据库中获取状态为 `wait` 的任务。 +- **任务执行**：执行 bisect 操作以查找故障提交。 +- **结果更新**：更新任务状态为 `completed` 或 `failed`，并记录结果。 +- **回归检查**：更新 `regression` 数据库以记录任务的执行结果。 +## 历史设计讨论 +### Bisect 提交 +#### 方式1: +- 在特定测试机上运行 +- 确保安全性：账户/令牌，访问服务 + +#### 方式2: +- 提交 job1 +- 提交 job2，重用相同的账户信息 +- 问题：消耗信用/机器时间 + +#### 方式3: +- compass ci 容器运行为服务 +- 参考 delimiter 服务 +- 提供 API/new-bisect-task，添加到 ES +- 循环：从 ES 消费一个任务，fork 进程，开始 bisect + +### 回归数据库 +### Bisect 任务队列 +### Bisect 任务管理和仪表板 + +- Bisect 任务名称 +- 套件，开始时间/运行时间，步骤，组ID -- Gitee From 0b177c0e8f428ed75447a29706126816eefd0643 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 14:09:44 +0800 Subject: [PATCH 23/39] =?UTF-8?q?bisect-task:=20=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E5=A4=96=E9=83=A8=E6=98=A0=E5=B0=84=E7=9A=84=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/start | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container/bisect/start b/container/bisect/start index 349ff395..74ef941a 100755 --- a/container/bisect/start +++ b/container/bisect/start @@ -26,6 +26,8 @@ DEFAULT_CONFIG_DIR = '/etc/compass-ci/defaults' DEFAULT_USER_CONFIG_DIR = File.expand_path("~/.config/compass-ci/") DEFAULT_BISECT_CONFIG_DIR = '/home/bisect/.config/compass-ci/' MANTICORE_HOST='172.17.0.1' +BISECT_LOG_DIR='/home/bisect' +HOST_LOG_DIR='/tmp/.bisect' docker_rm 'bisect' cmd = %w[ -- Gitee From 3106ed0ae620c20564dc1f75087a54956be5424d Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 14:14:36 +0800 Subject: [PATCH 24/39] =?UTF-8?q?bisect-task:=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=9B=AE=E5=BD=95=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/bisect/start b/container/bisect/start index 74ef941a..25e71cb9 100755 --- a/container/bisect/start +++ b/container/bisect/start @@ -39,6 +39,7 @@ cmd = %w[ -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} -e MANTICORE_HOST=#{MANTICORE_HOST} + -v #{HOST_LOG_DIR}:#{BISECT_LOG_DIR} -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro -v #{DEFAULT_USER_CONFIG_DIR}:#{DEFAULT_BISECT_CONFIG_DIR}:ro -v /etc/localtime:/etc/localtime:ro -- Gitee From 38735156e5a70efb78f25bbf144d59a8d51f53a1 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 14:16:37 +0800 Subject: [PATCH 25/39] =?UTF-8?q?bisect-task:=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=B5=81=E7=A8=8B=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/readme.md | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/container/bisect/readme.md b/container/bisect/readme.md index 6b5c29f4..c8827c1d 100644 --- a/container/bisect/readme.md +++ b/container/bisect/readme.md @@ -4,6 +4,52 @@ `bisect_task` 是一个用于自动化处理 bisect 任务的服务。它通过与 Manticore 数据库交互，管理任务的提交、处理和状态更新。 +### bisect-task 流程 + ++---------+ get_new_bisect_task_from_jobs +-----------------+ add_bisect_task +-----------------------------+ +| jobs db | -------------------------------> | bisect producer | -----------------> | bisect db | ++---------+ +-----------------+ +-----------------------------+ + | + | get_tasks_from_bisect_task + v + +-----------------------------+ +-----------+ +-------------------+ + | bisect consumer | --> | run bisct | -+-----> | bisect process .. | + +-----------------------------+ +-----------+ | +-------------------+ + | +-------------------+ + +-----> | bisect process 1 | + | +-------------------+ + | +-------------------+ + +-----> | bisect process n | + +-------------------+ +### 在 compass-ci 中 + ++-----------------+ +------------+ +-------------------------+ +| upstream update | -+-----> | benchmark | ------+-> | jobs db | ++-----------------+ | +------------+ | +-------------------------+ + | | | + | | | bad_job + | | v + | +------------+ | +-------------------------+ valid errid + +-----> | build | ------+ | bisect-task producer | <---------------------------------------------------+ + | +------------+ | +-------------------------+ | + | | | | + | | | | + | | v | + | +------------+ | +-------------------------+ | + +-----> | functional | ------+ | bisect db | <+ | + +------------+ +-------------------------+ | | + | | | + | task wait to be bisect | found first bad commit | + v | | + +----------------------------------------------------+ sucess bisect errid +---------------+ + | bisect-task consumer | ---------------------> | regression db | + +----------------------------------------------------+ +---------------+ + | ^ + | task | return result + v | + +-------------------------+ | + | bisect-py | -+ + +-------------------------+ ## 功能 - **任务提交**：通过 API 接口提交新的 bisect 任务。 @@ -74,3 +120,77 @@ - Bisect 任务名称 - 套件，开始时间/运行时间，步骤，组ID +## bisect submit 归档设计部分 + +### way1: + run in special testbox + +ensure security: +- account/token +- visit services + + +common testbox: + submit upload lkp-tests tar ball, run in testbox + +secure testbox: + submit NO UPLOAD lkp-tests tar ball + run only selected list of programs + run in a dedicated host machine, FW can visit services + + +### way2: +submit job1 + submit job2, reuse same account info +problem: consumes his credit / machine time + +### way3: +compass ci container/ run as service +refer to delimiter service + system bisect account + + provide API/new-bisect-task + add to ES + + loop: + consume one task from ES + fork process, start bisect + bisect step + submit job + change bisect-task state=finish + +submit-jobs + +### regression db +### bisect task queue +### bisect tasks management and dashboard + + bisect task name + suite start_time/run_time step group_id + +## 附录 + +### 流程图源码 + +``` +############################################################## +# compass-ci bisect 流程图 v 0.5 + +[ upstream update ] -> { start: front, 0; } [ build ], [ functional ], [ benchmark ] -> { end: back,0; } [ jobs db ] +[ jobs db ] {flow: south;} - bad_job -> [ bisect-task producer ] -> {flow: south;} [ bisect db ] {flow: south;} - task wait to be bisect -> [ bisect-task consumer ] - task -> [ bisect-py ] +[ bisect-py ] - return result -> [ bisect-task consumer ] - found first bad commit -> [ bisect db ] +[ regression db ] - valid errid -> [ bisect-task producer ] +[ bisect-task consumer ] - sucess bisect errid -> [ regression db ] +``` + +``` +############################################################## +# bisect-task 流程图 v 0.5 +# {flow: south;} 向下 +# { end: back,0; } 多个聚合为一 +# { start: front, 0; } 一分为多 +[ jobs db ] - get_new_bisect_task_from_jobs -> [ bisect producer ] - add_bisect_task -> [ bisect db ] +[ bisect db ] - get_tasks_from_bisect_task -> {flow: south;} [ bisect consumer ] -> [ run bisct ] +[ run bisct ] -> { start: front, 0; } [ bisect process 1 ], [ bisect process .. ], [ bisect process n ] +``` + -- Gitee From 5e18ecd90a3d4e277a96c75acf1b3531cef6ec39 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 14:29:54 +0800 Subject: [PATCH 26/39] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20bisect=5Ftas?= =?UTF-8?q?k=20=E7=9A=84=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E5=92=8C?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_bisect_task_functional.py | 37 +++++++++++++++++++ tests/bisect-tasks/test_bisect_task_unit.py | 37 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/bisect-tasks/test_bisect_task_functional.py create mode 100644 tests/bisect-tasks/test_bisect_task_unit.py diff --git a/tests/bisect-tasks/test_bisect_task_functional.py b/tests/bisect-tasks/test_bisect_task_functional.py new file mode 100644 index 00000000..6660a9c5 --- /dev/null +++ b/tests/bisect-tasks/test_bisect_task_functional.py @@ -0,0 +1,37 @@ +import requests +import unittest + +class TestBisectTaskAPI(unittest.TestCase): + + BASE_URL = "http://localhost:9999" + + def test_add_bisect_task(self): + url = f"{self.BASE_URL}/new_bisect_task" + data = { + "bad_job_id": "12345", + "error_id": "error_67890" + } + response = requests.post(url, json=data) + self.assertEqual(response.status_code, 200) + self.assertIn("Task added successfully", response.json().get("message", "")) + + def test_list_bisect_tasks(self): + url = f"{self.BASE_URL}/list_bisect_tasks" + response = requests.get(url) + self.assertEqual(response.status_code, 200) + self.assertIn("tasks", response.json()) + + def test_list_tasks_by_status(self): + url = f"{self.BASE_URL}/list_tasks_by_status?status=completed" + response = requests.get(url) + self.assertEqual(response.status_code, 200) + self.assertIn("tasks", response.json()) + + def test_delete_failed_tasks(self): + url = f"{self.BASE_URL}/delete_failed_tasks" + response = requests.delete(url) + self.assertEqual(response.status_code, 200) + self.assertIn("deleted_count", response.json()) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/bisect-tasks/test_bisect_task_unit.py b/tests/bisect-tasks/test_bisect_task_unit.py new file mode 100644 index 00000000..43e69430 --- /dev/null +++ b/tests/bisect-tasks/test_bisect_task_unit.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import patch, MagicMock +from container.bisect.bisect_task import BisectTask + +class TestBisectTask(unittest.TestCase): + + @patch('container.bisect.bisect_task.ManticoreClient') + def setUp(self, MockManticoreClient): + self.mock_client = MockManticoreClient.return_value + self.bisect_task = BisectTask() + + def test_generate_task_id(self): + bad_job_id = "12345" + error_id = "error_67890" + task_id = self.bisect_task.generate_task_id(bad_job_id, error_id) + self.assertIsInstance(task_id, int) + self.assertTrue(0 <= task_id < 2**63) + + def test_add_bisect_task_success(self): + task = {"bad_job_id": "12345", "error_id": "error_67890"} + self.mock_client.insert.return_value = True + result = self.bisect_task.add_bisect_task(task) + self.assertTrue(result) + + def test_add_bisect_task_failure(self): + task = {"bad_job_id": "12345", "error_id": "error_67890"} + self.mock_client.insert.return_value = False + result = self.bisect_task.add_bisect_task(task) + self.assertFalse(result) + + def test_set_priority_level(self): + job_info = {"suite": "check_abi", "repo": "linux", "error_id": "some_error_id"} + priority = self.bisect_task.set_priority_level(job_info) + self.assertEqual(priority, 6) # 根据权重配置计算 + +if __name__ == '__main__': + unittest.main() -- Gitee From 554177c35ee9d446fbebfa713424c597ccfa1f3a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 29 May 2025 14:32:31 +0800 Subject: [PATCH 27/39] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=8F=90=E4=BA=A4=E6=96=B9=E6=B3=95=E4=BB=A5=E6=B8=85?= =?UTF-8?q?=E7=90=86=20Git=20=E7=9B=AE=E5=BD=95=E5=B9=B6=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=A4=8D=E7=94=A8=E5=BB=BA=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container/bisect/bisect-task.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index e08c9cbd..e8dcc1d8 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -439,16 +439,30 @@ class BisectTask: for future in as_completed(futures, timeout=72000): # 2小时超时 try: result = future.result() + task_id = result['id'] if result['status'] == 'success': - logger.info(f"任务完成: {result['id']}") + logger.info(f"任务完成: {task_id}") else: - logger.error(f"任务失败: {result['id']} - {result['error']}") + logger.error(f"任务失败: {task_id} - {result['error']}") + + # 删除 Git 目录以避免文件越来越大 + git_dir = self._generate_task_path(self._config, {'id': task_id}) + if os.path.exists(git_dir): + try: + shutil.rmtree(git_dir) + logger.info(f"成功删除 Git 目录: {git_dir}") + except Exception as e: + logger.error(f"删除 Git 目录失败 {git_dir}: {str(e)}") + except TimeoutError: logger.error("任务处理超时，可能发生死锁") future.cancel() except Exception as e: logger.error(f"结果处理异常: {str(e)}") + # 注释：后续可以考虑对 Git 目录进行复用，以减少重复克隆的开销。 + # 例如，可以在任务开始时检查是否存在可复用的 Git 目录，并在任务结束后更新其状态。 + def update_regression(self, task, result): """使用 ManticoreClient 更新回归数据库""" try: -- Gitee From e16ae256eb2984847e4a0f4827f8b7abeef79477 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Mon, 9 Jun 2025 16:24:53 +0800 Subject: [PATCH 28/39] =?UTF-8?q?docs:=20=E6=B7=BB=E5=8A=A0=E8=B0=83?= =?UTF-8?q?=E5=BA=A6=E5=99=A8API=E6=8E=A5=E5=8F=A3=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api.cr | 53 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/api.cr b/src/api.cr index 955254e6..d2100d37 100644 --- a/src/api.cr +++ b/src/api.cr @@ -9,25 +9,48 @@ require "./sched" require "./host" require "./account" -# ------------------------------------------------------------------------------------------- -# end_user: -# - restful API [post "/submit_job"] to submit a job to scheduler -# -- json formated [job] in the request data +# API Documentation +# ------------------------------------------------------------------------------- +# 基础接口: +# GET / - 健康检查,返回存活状态和版本号 +# GET /scheduler/v1/health - 新版健康检查(RESTful风格) # -# ------------------------------------------------------------------------------------------- -# runner: -# - restful API [get "/boot.ipxe/mac/52-54-00-12-34-56"] to get a job for ipxe qemu-runner -# -- when find then return <#!ipxe and job.cgz kernal initrd> -# -- when no job return <#!ipxe no job messages> +# 作业管理接口: +# POST /submit_job - 提交新作业(旧版) +# POST /scheduler/v1/jobs/submit - 提交新作业(新版RESTful) +# GET /boot.:boot_type/:parameter/:value - 物理机作业分配 +# GET /scheduler/v1/jobs/dispatch - 新版作业分配 +# GET /scheduler/v1/jobs/:job_id - 查询作业详情 +# POST /scheduler/v1/jobs/wait - 批量等待作业 +# POST /scheduler/v1/jobs/:job_id/update - 更新作业状态 +# POST /scheduler/v1/jobs/:job_id/cancel - 取消作业 +# POST /scheduler/v1/jobs/:job_id/terminate - 强制终止作业 # -# - restful API [get "/job_initrd_tmpfs/11/job.cgz"] to download job(11) job.cgz file -# - restful API [get "/scheduler/job/update"] report job var that should be append +# 主机管理接口: +# GET /scheduler/v1/hosts/:hostname - 查询主机信息 +# POST /scheduler/v1/hosts/:hostname - 注册/更新主机 # -# ------------------------------------------------------------------------------------------- -# scheduler: -# - use [redis incr] as job_id, a 64bit int number -# - restful API [get "/"] default echo +# 账户管理接口: +# POST /scheduler/v1/accounts/:my_account - 注册新账户(仅本地) # +# 事件上报接口: +# POST /report_event - 通用事件上报 +# POST /~lkp/cgi-bin/report_ssh_info - SSH信息上报 +# +# 文件操作接口: +# GET /srv/*path - 下载服务器文件 +# POST /result/*path - 上传结果文件 +# POST /srv/*path - 上传服务器文件 +# +# 仪表盘接口: +# GET /scheduler/v1/dashboard/jobs/pending - 待处理作业列表 +# GET /scheduler/v1/dashboard/jobs/running - 运行中作业列表 +# GET /scheduler/v1/dashboard/hosts - 主机列表 +# GET /scheduler/v1/dashboard/accounts - 账户列表 +# +# WebSocket接口: +# WS /scheduler/v1/client - 客户端实时通信 +# WS /scheduler/v1/vm-container-provider/:host - 资源提供者通信 # Struct to represent a result with success/failure status struct Result -- Gitee From c4678aac0e5d251985b606c9028a99325c52dfc3 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Tue, 10 Jun 2025 16:26:26 +0800 Subject: [PATCH 29/39] =?UTF-8?q?docs:=20=E5=AE=8C=E5=96=84API=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api.cr | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/api.cr b/src/api.cr index d2100d37..982a040f 100644 --- a/src/api.cr +++ b/src/api.cr @@ -16,15 +16,41 @@ require "./account" # GET /scheduler/v1/health - 新版健康检查(RESTful风格) # # 作业管理接口: -# POST /submit_job - 提交新作业(旧版) -# POST /scheduler/v1/jobs/submit - 提交新作业(新版RESTful) -# GET /boot.:boot_type/:parameter/:value - 物理机作业分配 -# GET /scheduler/v1/jobs/dispatch - 新版作业分配 +# POST /scheduler/v1/jobs/submit - 提交新作业 +# 请求体: {"suite": "...", "os": "...", ...} +# 返回: {"job_id": "...", "status": "..."} +# # GET /scheduler/v1/jobs/:job_id - 查询作业详情 +# 参数: job_id - 作业ID +# 可选参数: fields - 指定返回字段,多个用逗号分隔 +# +# GET /scheduler/v1/jobs/dispatch - 新版作业分配 +# 用于调度器分配作业给执行节点 +# # POST /scheduler/v1/jobs/wait - 批量等待作业 +# 请求体: {"job_ids": ["id1", "id2"]} +# 返回: {"finished": ["id1"], "running": ["id2"]} +# # POST /scheduler/v1/jobs/:job_id/update - 更新作业状态 +# 参数: job_id - 作业ID +# 请求体: {"state": "...", "result": "..."} +# # POST /scheduler/v1/jobs/:job_id/cancel - 取消作业 +# 参数: job_id - 作业ID +# 需要认证 +# # POST /scheduler/v1/jobs/:job_id/terminate - 强制终止作业 +# 参数: job_id - 作业ID +# 需要认证 +# +# 仪表盘接口: +# GET /scheduler/v1/dashboard/jobs/pending - 查询待处理作业列表 +# GET /scheduler/v1/dashboard/jobs/running - 查询运行中作业列表 +# +# 注意: +# 1. 所有POST请求需要设置 Content-Type: application/json +# 2. 涉及修改操作的接口需要进行账户认证 +# 3. 默认服务端口为3000 # # 主机管理接口: # GET /scheduler/v1/hosts/:hostname - 查询主机信息 -- Gitee From 6e771647b48254c706f5b5d6e03da747b6415908 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Wed, 11 Jun 2025 22:30:33 +0800 Subject: [PATCH 30/39] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0ES=E6=97=A0?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E6=97=B6=E8=87=AA=E5=8A=A8=E5=88=87=E6=8D=A2?= =?UTF-8?q?Manticore=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/compare.rb | 21 +++++++++++++++++---- lib/manticore.rb | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index bc935563..57dc27d7 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -4,6 +4,7 @@ require_relative './es_query.rb' require_relative './matrix2.rb' +require_relative './manticore.rb' require_relative './compare_matrixes.rb' require_relative './constants.rb' require 'yaml' @@ -30,6 +31,20 @@ def compare_matrices_list(argv, common_conditions, options) compare_matrixes(matrices_list, suite_list, nil, titles, options: options) end +def safe_multi_field_query(query, **opts) + es = ESQuery.new + begin + result = es.multi_field_query(query, **opts) + if result.nil? || result['hits'].nil? || result['hits']['hits'].empty? + raise 'No ES result' + end + result + rescue + manti = Manticore::Client.new + manti.multi_field_query(query, **opts) + end +end + def parse_argv(argv, common_conditions) conditions = [] common_items = common_conditions.split(' ') @@ -45,9 +60,8 @@ def create_matrices_list(conditions, options) matrices_list = [] suite_list = [] titles = [] - es = ESQuery.new conditions.each do |condition| - query_results = es.multi_field_query(condition[1], desc_keyword: 'start_time') + query_results = safe_multi_field_query(condition[1], desc_keyword: 'start_time') matrix, suites = Matrix.combine_query_data(query_results, options) next unless matrix @@ -82,8 +96,7 @@ def compare_group(argv, dimensions, options) end def create_groups_matrices_list(conditions, dims, options) - es = ESQuery.new - query_results = es.multi_field_query(conditions, desc_keyword: 'start_time') + query_results = safe_multi_field_query(conditions, desc_keyword: 'start_time') Matrix.combine_group_query_data(query_results['hits']['hits'], dims, options) end diff --git a/lib/manticore.rb b/lib/manticore.rb index 0725397b..ef234e53 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -38,6 +38,29 @@ module Manticore # result_hash = JSON.parse(response.body) end + def self.multi_field_query(query, desc_keyword: nil, size: 10_000, **opts) + # 将查询条件转换为SQL + sql = build_sql_from_query(query, desc_keyword, size) + result = execute_select(sql) + + # 将结果转换为ES兼容格式 + hits = JSON.parse(result.body)['data'].map do |row| + { '_source' => row } + end + + { 'hits' => { 'hits' => hits } } + end + + private + + def self.build_sql_from_query(query, desc_keyword, size) + # 简单实现,实际应该根据query参数构建更复杂的SQL + sql = "SELECT * FROM #{DEFAULT_INDEX}" + sql += " ORDER BY #{desc_keyword} DESC" if desc_keyword + sql += " LIMIT #{size}" + sql + end + # return format is "hits": { "hits": [ { "_source": {job hash}}]} def self.search(query) uri = URI("http://#{HOST}:#{PORT}/search") -- Gitee From 84b6a4bb71af7f312eeb760300d628fafe890a6a Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 13:36:59 +0800 Subject: [PATCH 31/39] =?UTF-8?q?refactor:=20=E5=9C=A8=20safe=5Fmulti=5Ffi?= =?UTF-8?q?eld=5Fquery=20=E4=B8=AD=E7=94=A8=20QueryBuilder=20=E6=9B=BF?= =?UTF-8?q?=E4=BB=A3=20multi=5Ffield=5Fquery=20=E5=AE=9E=E7=8E=B0=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/compare.rb | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 57dc27d7..f5aebc27 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -40,8 +40,28 @@ def safe_multi_field_query(query, **opts) end result rescue - manti = Manticore::Client.new - manti.multi_field_query(query, **opts) + # 直接使用 QueryBuilder 构建查询 + builder = Manticore::QueryBuilder.new(index: Manticore::DEFAULT_INDEX, size: opts[:size] || 10_000) + + # 处理查询条件 + if query.is_a?(Hash) + query.each do |k, v| + builder.add_filter(k, Array(v)) + end + elsif query.is_a?(String) + # 处理 key=val1,val2 格式的查询字符串 + field, values = query.split('=', 2) + builder.add_filter(field, values.split(',')) if field && values + end + + # 添加排序 + builder.sort(opts[:desc_keyword] || 'submit_time', order: 'desc') + + # 执行查询并格式化结果 + response = Manticore::Client.search(builder.build) + body = JSON.parse(response.body) + hits = (body['hits'] && body['hits']['hits']) || [] + { 'hits' => { 'hits' => hits } } end end -- Gitee From 62d268a368baa10858ff152f1be1b40e73c1a3b7 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 13:43:35 +0800 Subject: [PATCH 32/39] Restore manticore.rb Add search method for manticore python --- container/bisect/bisect-task.py | 8 +++++++- lib/manticore.rb | 23 ----------------------- lib/manticore_simple.py | 3 +++ 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/container/bisect/bisect-task.py b/container/bisect/bisect-task.py index e8dcc1d8..da937a00 100755 --- a/container/bisect/bisect-task.py +++ b/container/bisect/bisect-task.py @@ -123,9 +123,15 @@ class BisectTask: try: # 使用 ManticoreClient 的 search 方法检查任务状态 + # { "match": {"id": task_id}}, status_check = process_client.search( table="bisect", - query={"match": {"id": task_id, "bisect_status": "wait"}}, + query={ "bool": { + "must": [ + { "match": {"bisect_status": "wait"}} + ] + } + }, limit=1 ) diff --git a/lib/manticore.rb b/lib/manticore.rb index ef234e53..0725397b 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -38,29 +38,6 @@ module Manticore # result_hash = JSON.parse(response.body) end - def self.multi_field_query(query, desc_keyword: nil, size: 10_000, **opts) - # 将查询条件转换为SQL - sql = build_sql_from_query(query, desc_keyword, size) - result = execute_select(sql) - - # 将结果转换为ES兼容格式 - hits = JSON.parse(result.body)['data'].map do |row| - { '_source' => row } - end - - { 'hits' => { 'hits' => hits } } - end - - private - - def self.build_sql_from_query(query, desc_keyword, size) - # 简单实现,实际应该根据query参数构建更复杂的SQL - sql = "SELECT * FROM #{DEFAULT_INDEX}" - sql += " ORDER BY #{desc_keyword} DESC" if desc_keyword - sql += " LIMIT #{size}" - sql - end - # return format is "hits": { "hits": [ { "_source": {job hash}}]} def self.search(query) uri = URI("http://#{HOST}:#{PORT}/search") diff --git a/lib/manticore_simple.py b/lib/manticore_simple.py index c0a9d3e9..12331e71 100644 --- a/lib/manticore_simple.py +++ b/lib/manticore_simple.py @@ -1,6 +1,7 @@ import requests import json import time +from log_config import logger, StructuredLogger from typing import Dict, List, Union, Optional class ManticoreClient: @@ -50,12 +51,14 @@ class ManticoreClient: ) if resp.status_code != 200: + logger.error(f"Search request failed with status code {resp.status_code}: {resp.text}") return None result = resp.json() return [ hit.get('_source', {}) for hit in result.get('hits', {}).get('hits', []) + if '_source' in hit ] except requests.exceptions.RequestException: -- Gitee From d35910501623d0516a5f344d4165921855fb1c51 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 13:48:23 +0800 Subject: [PATCH 33/39] Restore api.cr --- src/api.cr | 79 +++++++++++------------------------------------------- 1 file changed, 15 insertions(+), 64 deletions(-) diff --git a/src/api.cr b/src/api.cr index 982a040f..955254e6 100644 --- a/src/api.cr +++ b/src/api.cr @@ -9,74 +9,25 @@ require "./sched" require "./host" require "./account" -# API Documentation -# ------------------------------------------------------------------------------- -# 基础接口: -# GET / - 健康检查,返回存活状态和版本号 -# GET /scheduler/v1/health - 新版健康检查(RESTful风格) +# ------------------------------------------------------------------------------------------- +# end_user: +# - restful API [post "/submit_job"] to submit a job to scheduler +# -- json formated [job] in the request data # -# 作业管理接口: -# POST /scheduler/v1/jobs/submit - 提交新作业 -# 请求体: {"suite": "...", "os": "...", ...} -# 返回: {"job_id": "...", "status": "..."} +# ------------------------------------------------------------------------------------------- +# runner: +# - restful API [get "/boot.ipxe/mac/52-54-00-12-34-56"] to get a job for ipxe qemu-runner +# -- when find then return <#!ipxe and job.cgz kernal initrd> +# -- when no job return <#!ipxe no job messages> # -# GET /scheduler/v1/jobs/:job_id - 查询作业详情 -# 参数: job_id - 作业ID -# 可选参数: fields - 指定返回字段,多个用逗号分隔 +# - restful API [get "/job_initrd_tmpfs/11/job.cgz"] to download job(11) job.cgz file +# - restful API [get "/scheduler/job/update"] report job var that should be append # -# GET /scheduler/v1/jobs/dispatch - 新版作业分配 -# 用于调度器分配作业给执行节点 +# ------------------------------------------------------------------------------------------- +# scheduler: +# - use [redis incr] as job_id, a 64bit int number +# - restful API [get "/"] default echo # -# POST /scheduler/v1/jobs/wait - 批量等待作业 -# 请求体: {"job_ids": ["id1", "id2"]} -# 返回: {"finished": ["id1"], "running": ["id2"]} -# -# POST /scheduler/v1/jobs/:job_id/update - 更新作业状态 -# 参数: job_id - 作业ID -# 请求体: {"state": "...", "result": "..."} -# -# POST /scheduler/v1/jobs/:job_id/cancel - 取消作业 -# 参数: job_id - 作业ID -# 需要认证 -# -# POST /scheduler/v1/jobs/:job_id/terminate - 强制终止作业 -# 参数: job_id - 作业ID -# 需要认证 -# -# 仪表盘接口: -# GET /scheduler/v1/dashboard/jobs/pending - 查询待处理作业列表 -# GET /scheduler/v1/dashboard/jobs/running - 查询运行中作业列表 -# -# 注意: -# 1. 所有POST请求需要设置 Content-Type: application/json -# 2. 涉及修改操作的接口需要进行账户认证 -# 3. 默认服务端口为3000 -# -# 主机管理接口: -# GET /scheduler/v1/hosts/:hostname - 查询主机信息 -# POST /scheduler/v1/hosts/:hostname - 注册/更新主机 -# -# 账户管理接口: -# POST /scheduler/v1/accounts/:my_account - 注册新账户(仅本地) -# -# 事件上报接口: -# POST /report_event - 通用事件上报 -# POST /~lkp/cgi-bin/report_ssh_info - SSH信息上报 -# -# 文件操作接口: -# GET /srv/*path - 下载服务器文件 -# POST /result/*path - 上传结果文件 -# POST /srv/*path - 上传服务器文件 -# -# 仪表盘接口: -# GET /scheduler/v1/dashboard/jobs/pending - 待处理作业列表 -# GET /scheduler/v1/dashboard/jobs/running - 运行中作业列表 -# GET /scheduler/v1/dashboard/hosts - 主机列表 -# GET /scheduler/v1/dashboard/accounts - 账户列表 -# -# WebSocket接口: -# WS /scheduler/v1/client - 客户端实时通信 -# WS /scheduler/v1/vm-container-provider/:host - 资源提供者通信 # Struct to represent a result with success/failure status struct Result -- Gitee From 9eb286fb79517aeadfad6bf4164b0ef273e72a92 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 13:58:55 +0800 Subject: [PATCH 34/39] =?UTF-8?q?feat:=20=E5=9C=A8=20Manticore::Client=20?= =?UTF-8?q?=E4=B8=AD=E6=B7=BB=E5=8A=A0=20multi=5Ffield=5Fquery=20=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=AE=9E=E7=8E=B0=E5=A4=9A=E5=AD=97=E6=AE=B5=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/manticore.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/manticore.rb b/lib/manticore.rb index 0725397b..b1ccec35 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -45,6 +45,24 @@ module Manticore # result_hash = JSON.parse(response.body) end + # Compatible with ESQuery interface + def self.multi_field_query(items, size: 10_000, desc_keyword: nil, **opts) + builder = Manticore::QueryBuilder.new(index: Manticore::DEFAULT_INDEX, size: size) + if items.is_a?(Hash) + items.each do |k, v| + builder.add_filter(k, Array(v)) + end + elsif items.is_a?(String) + field, values = items.split('=', 2) + builder.add_filter(field, values.split(',')) if field && values + end + builder.sort(desc_keyword || 'submit_time', order: 'desc') + response = Manticore::Client.search(builder.build) + body = JSON.parse(response.body) + hits = (body['hits'] && body['hits']['hits']) || [] + { 'hits' => { 'hits' => hits } } + end + end class QueryBuilder -- Gitee From eaee88710c128e65274edb5f73455a79df233b8e Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 14:02:19 +0800 Subject: [PATCH 35/39] =?UTF-8?q?refactor:=20=E7=BB=9F=E4=B8=80=20safe=5Fm?= =?UTF-8?q?ulti=5Ffield=5Fquery=20=E4=B8=AD=E6=9F=A5=E8=AF=A2=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E4=B8=BA=20multi=5Ffield=5Fquery=20=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/compare.rb | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index f5aebc27..2702bc7d 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -40,28 +40,7 @@ def safe_multi_field_query(query, **opts) end result rescue - # 直接使用 QueryBuilder 构建查询 - builder = Manticore::QueryBuilder.new(index: Manticore::DEFAULT_INDEX, size: opts[:size] || 10_000) - - # 处理查询条件 - if query.is_a?(Hash) - query.each do |k, v| - builder.add_filter(k, Array(v)) - end - elsif query.is_a?(String) - # 处理 key=val1,val2 格式的查询字符串 - field, values = query.split('=', 2) - builder.add_filter(field, values.split(',')) if field && values - end - - # 添加排序 - builder.sort(opts[:desc_keyword] || 'submit_time', order: 'desc') - - # 执行查询并格式化结果 - response = Manticore::Client.search(builder.build) - body = JSON.parse(response.body) - hits = (body['hits'] && body['hits']['hits']) || [] - { 'hits' => { 'hits' => hits } } + Manticore::Client.multi_field_query(query, size: opts[:size] || 10_000, desc_keyword: opts[:desc_keyword]) end end -- Gitee From e1fd92f64b69208c62bbec553d77848b7a0990fb Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 14:06:38 +0800 Subject: [PATCH 36/39] =?UTF-8?q?style:=20=E5=9C=A8lib/compare.rb=E7=9A=84?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=96=B9=E6=B3=95=E4=B8=AD=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=AF=A6=E7=BB=86=E8=B0=83=E8=AF=95=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/compare.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 2702bc7d..87e0a9af 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -34,13 +34,19 @@ end def safe_multi_field_query(query, **opts) es = ESQuery.new begin + puts "[DEBUG] Try ES search with query: #{query.inspect}, opts: #{opts.inspect}" result = es.multi_field_query(query, **opts) + puts "[DEBUG] ES result: #{result.inspect}" if result.nil? || result['hits'].nil? || result['hits']['hits'].empty? + puts "[DEBUG] No result from ES, will try Manticore." raise 'No ES result' end result - rescue - Manticore::Client.multi_field_query(query, size: opts[:size] || 10_000, desc_keyword: opts[:desc_keyword]) + rescue => e + puts "[DEBUG] Try Manticore search, query: #{query.inspect}, opts: #{opts.inspect}, error: #{e.inspect}" + m_result = Manticore::Client.multi_field_query(query, size: opts[:size] || 10_000, desc_keyword: opts[:desc_keyword]) + puts "[DEBUG] Manticore result: #{m_result.inspect}" + m_result end end @@ -60,8 +66,11 @@ def create_matrices_list(conditions, options) suite_list = [] titles = [] conditions.each do |condition| + puts "[DEBUG] Condition: #{condition.inspect}" query_results = safe_multi_field_query(condition[1], desc_keyword: 'start_time') + puts "[DEBUG] Query results: #{query_results.inspect}" matrix, suites = Matrix.combine_query_data(query_results, options) + puts "[DEBUG] Matrix: #{matrix.inspect}, Suites: #{suites.inspect}" next unless matrix matrices_list << matrix @@ -69,6 +78,7 @@ def create_matrices_list(conditions, options) suite_list.concat(suites) end + puts "[DEBUG] Matrices list size: #{matrices_list.size}, Suite list: #{suite_list.inspect}, Titles: #{titles.inspect}" return matrices_list, suite_list, titles end -- Gitee From 5802893738de8eeeeef38c92577375b39a2b84f7 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 14:10:31 +0800 Subject: [PATCH 37/39] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9QueryBuilder?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0id=E5=A4=9A=E5=80=BC=E6=97=B6=E4=BD=BF?= =?UTF-8?q?=E7=94=A8should(OR)=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/manticore.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/manticore.rb b/lib/manticore.rb index b1ccec35..d2513657 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -82,6 +82,9 @@ module Manticore elsif field == 'errid' @full_text_terms[:errid] ||= [] @full_text_terms[:errid] += values.map { |v| v.to_s } + elsif field == 'id' && values.size > 1 + @should_clauses ||= [] + @should_clauses += values.map { |v| { equals: { field => convert_value(v) } } } else @equals_clauses += values.map do |v| v = convert_value(v) @@ -104,12 +107,17 @@ module Manticore bool[:must] << { match: @full_text_terms } unless @full_text_terms.empty? bool[:must] += @equals_clauses unless @equals_clauses.empty? + + if defined?(@should_clauses) && @should_clauses && !@should_clauses.empty? + bool[:should] = @should_clauses + bool[:minimum_should_match] = 1 + end @ranges.each do |field, range| bool[:must] << { range: { field => range } } end - query[:query] = { bool: bool } unless bool[:must].empty? + query[:query] = { bool: bool } unless bool[:must].empty? && (!bool[:should] || bool[:should].empty?) query[:sort] = [@sort] unless @sort.empty? query -- Gitee From 31c9ec3b3d5fb0f605e4e3eaa840cffc234bfe35 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 14:17:56 +0800 Subject: [PATCH 38/39] =?UTF-8?q?fix:=20=E6=B7=BB=E5=8A=A0=E8=B0=83?= =?UTF-8?q?=E8=AF=95=E6=97=A5=E5=BF=97=E6=89=93=E5=8D=B0=20Manticore=20?= =?UTF-8?q?=E5=8E=9F=E5=A7=8B=E5=93=8D=E5=BA=94=E5=86=85=E5=AE=B9=E4=BE=BF?= =?UTF-8?q?=E4=BA=8E=E6=8E=92=E6=9F=A5=E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/manticore.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/manticore.rb b/lib/manticore.rb index d2513657..a5dd34a5 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -58,6 +58,7 @@ module Manticore end builder.sort(desc_keyword || 'submit_time', order: 'desc') response = Manticore::Client.search(builder.build) + puts "[DEBUG] Manticore raw response: #{response.body}" body = JSON.parse(response.body) hits = (body['hits'] && body['hits']['hits']) || [] { 'hits' => { 'hits' => hits } } -- Gitee From 9894c5b960d5aee27a451e2ef4d9362bc23d5925 Mon Sep 17 00:00:00 2001 From: jacknichao Date: Thu, 12 Jun 2025 14:36:08 +0800 Subject: [PATCH 39/39] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9multi=5Ffield=5F?= =?UTF-8?q?query=E5=85=BC=E5=AE=B9manticore=E8=BF=94=E5=9B=9E=E7=9A=84j?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/manticore.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/manticore.rb b/lib/manticore.rb index a5dd34a5..15eea089 100644 --- a/lib/manticore.rb +++ b/lib/manticore.rb @@ -60,7 +60,21 @@ module Manticore response = Manticore::Client.search(builder.build) puts "[DEBUG] Manticore raw response: #{response.body}" body = JSON.parse(response.body) - hits = (body['hits'] && body['hits']['hits']) || [] + if body['hits'] && body['hits']['hits'].is_a?(Array) && !body['hits']['hits'].empty? + hits = body['hits']['hits'].map do |h| + if h.key?('_source') + h + elsif h.key?('j') + { '_source' => h['j'] } + else + { '_source' => h } + end + end + elsif body['data'] + hits = body['data'].map { |row| { '_source' => row } } + else + hits = [] + end { 'hits' => { 'hits' => hits } } end -- Gitee