From 62fe1fe28643e01b4b2091d4cc4b10c0cbbd5d1e Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 20 Oct 2025 11:18:06 +0800 Subject: [PATCH 1/4] modify majun domain and param --- cve-vulner-manager/conf/product_app.conf | 1 + .../cve-ddd/infrastructure/majunimpl/impl.go | 23 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 0a6f62e..1166844 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -285,6 +285,7 @@ token = "${GITEE_MAJUN_TOKEN||xxx}" api_token = "${MAJUN_API_TOKEN||xxx}" app_id = "${MAJUN_APP_ID||xxx}" secret_key = "${MAJUN_SECRET_KEY||xxx}" +host = https://www.openlibing.com [testResult] host = https://dailybuild.openeuler.openatom.cn diff --git a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go index 42f82de..60f2919 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go @@ -18,8 +18,9 @@ import ( ) const ( - urlReleaseVersion = "https://majun.osinfra.cn/api/http/majun-platform-release/publish/externalInterface/queryReleaseVersion" - urlCollectCallback = "https://majun.osinfra.cn/api/http/majun-platform-release/publish/externalInterface/saveCveCallbackResult" + userAgent = "Apifox/1.0.0 (https://apifox.com)" + urlReleaseVersion = "/gateway/openlibing-platform-release/publish/externalInterface/queryReleaseVersion" + urlCollectCallback = "/gateway/openlibing-platform-release/publish/externalInterface/saveCveCallbackResult" ) type config struct { @@ -49,7 +50,8 @@ type releaseResponse struct { } func (impl *majunImpl) GetReleasedBranch() ([]string, error) { - req, err := impl.generateRequest(urlReleaseVersion, nil) + url := beego.AppConfig.String("majun::host") + urlReleaseVersion + req, err := impl.generateRequest(url, nil) if err != nil { return nil, err } @@ -65,15 +67,6 @@ func (impl *majunImpl) GetReleasedBranch() ([]string, error) { } return v.Result, nil - - //return []string{ - // "openEuler-20.03-LTS-SP1", - // "openEuler-20.03-LTS-SP4", - // "openEuler-22.03-LTS", - // "openEuler-22.03-LTS-SP1", - // "openEuler-22.03-LTS-SP2", - // "openEuler-22.03-LTS-SP3", - //}, nil } type callbackBody struct { @@ -102,7 +95,8 @@ func (impl *majunImpl) CollectCallback(id, data string) error { return err } - req, err := impl.generateRequest(urlCollectCallback, payload) + url := beego.AppConfig.String("majun::host") + urlCollectCallback + req, err := impl.generateRequest(url, payload) if err != nil { return err } @@ -134,8 +128,9 @@ func (impl *majunImpl) generateRequest(url string, body []byte) (req *http.Reque } req.Header.Set("timestamp", timestamp) - req.Header.Set("appId", impl.cfg.AppId) + req.Header.Set("accountid", impl.cfg.AppId) req.Header.Set("sign", sign) + req.Header.Set("User-Agent", userAgent) return } -- Gitee From 88ee95506c2de697d0b65104b3d2c21825a202ee Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 24 Oct 2025 15:17:59 +0800 Subject: [PATCH 2/4] modify sync issue plan date api --- .../cve-ddd/infrastructure/majunimpl/impl.go | 44 ++++++++++++++ cve-vulner-manager/task/issue.go | 59 ++++--------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go index 60f2919..2cb29ae 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go @@ -19,6 +19,7 @@ import ( const ( userAgent = "Apifox/1.0.0 (https://apifox.com)" + urlGetPlanDate = "/gateway/openlibing-vulnerability/openlibing-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" urlReleaseVersion = "/gateway/openlibing-platform-release/publish/externalInterface/queryReleaseVersion" urlCollectCallback = "/gateway/openlibing-platform-release/publish/externalInterface/saveCveCallbackResult" ) @@ -114,6 +115,48 @@ func (impl *majunImpl) CollectCallback(id, data string) error { return nil } +type PlanDataOfMaJun struct { + IssueNum string `json:"issueId"` + CveNum string `json:"cveNum"` + CvssScore float64 `json:"cvssScore"` + BeginTime string `json:"cveRepairStartTime"` + EndTime string `json:"issuePlanClosedTIme"` +} + +type PlanResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result struct { + Data []PlanDataOfMaJun `json:"data"` + } +} + +type PlanRequest struct { + Organization string `json:"organization"` + PageNum int `json:"pageNum"` + PageSize int `json:"pageSize"` +} + +func (impl *majunImpl) GetPlanDate(page, pageSize int) ([]PlanDataOfMaJun, error) { + param := PlanRequest{ + Organization: "openeuler", + PageNum: page, + PageSize: pageSize, + } + payload, _ := json.Marshal(param) + url := beego.AppConfig.String("majun::host") + urlGetPlanDate + + req, err := impl.generateRequest(url, payload) + if err != nil { + return nil, err + } + + var ret PlanResponse + _, err = impl.client.ForwardTo(req, &ret) + + return ret.Result.Data, err +} + func (impl *majunImpl) generateRequest(url string, body []byte) (req *http.Request, err error) { timestamp := strconv.FormatInt(time.Now().UnixMilli(), 10) sign := impl.sign(timestamp) @@ -131,6 +174,7 @@ func (impl *majunImpl) generateRequest(url string, body []byte) (req *http.Reque req.Header.Set("accountid", impl.cfg.AppId) req.Header.Set("sign", sign) req.Header.Set("User-Agent", userAgent) + req.Header.Set("Content-Type", "application/json") return } diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index bdf3a03..f4a94eb 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -1,16 +1,12 @@ package task import ( - "bytes" - "encoding/json" - "net/http" "strconv" "strings" "time" - "github.com/opensourceways/server-common-lib/utils" - "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/majunimpl" "cvevulner/models" "cvevulner/taskhandler" @@ -353,61 +349,26 @@ func SyncPlanDateOfIssueFromMaJun() error { return nil } -type PlanDataOfMaJun struct { - IssueNum string `json:"issueId"` - CveNum string `json:"cveNum"` - CvssScore float64 `json:"cvssScore"` - BeginTime string `json:"cveRepairStartTime"` - EndTime string `json:"issuePlanClosedTIme"` -} - -type PlanResponse struct { - Code int `json:"code"` - Message string `json:"message"` - Result struct { - Data []PlanDataOfMaJun `json:"data"` - } -} - -type PlanRequest struct { - Organization string `json:"organization"` - PageNum int `json:"pageNum"` - PageSize int `json:"pageSize"` -} - -func getPlanData() []PlanDataOfMaJun { - url := "https://majun.osinfra.cn/api/http/majun-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" - token := beego.AppConfig.String("majun::api_token") +func getPlanData() []majunimpl.PlanDataOfMaJun { + cli := majunimpl.NewMajunImpl() pageNum := 1 pageSize := 1000 - cli := utils.NewHttpClient(3) - var data []PlanDataOfMaJun + var allData []majunimpl.PlanDataOfMaJun for { - var ret PlanResponse - param := PlanRequest{ - Organization: "openeuler", - PageNum: pageNum, - PageSize: pageSize, - } - b, _ := json.Marshal(param) - req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(b)) - - req.Header.Add("access_token", token) - - if _, err := cli.ForwardTo(req, &ret); err != nil { - logs.Error("get plan data error: ", err.Error()) + data, err := cli.GetPlanDate(pageNum, pageSize) + if err != nil { + logs.Error("get plan data error: ", err.Error(), pageNum) break } - data = append(data, ret.Result.Data...) - - if len(ret.Result.Data) < pageSize { + allData = append(allData, data...) + if len(data) < pageSize { break } pageNum++ } - return data + return allData } -- Gitee From 209d563c07e418fa4c3007b51211c94cb2a6fbfc Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 29 Oct 2025 10:29:20 +0800 Subject: [PATCH 3/4] rerun InitIssueTask --- cve-vulner-manager/conf/product_app.conf | 4 +-- cve-vulner-manager/task/grabissue.go | 8 +----- cve-vulner-manager/taskhandler/grabissue.go | 28 +++++++++------------ 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 1166844..072811c 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -33,7 +33,7 @@ saFileDir = "download/sa" gaussFileDir = "download/gauss" # release package download url rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" -communityName = "src-openeuler:1,opengauss:2,mindspore:3,openlookeng:4" +communityName = "src-openeuler:1" [timedtask] @@ -78,7 +78,7 @@ cveflag = 1 getcve = 0 */40 * * * * oricveflag = 1 oricvecheck = 0 0 2 * * * -getissueflag = 2 +getissueflag = 1 getissue = 0 20 1 * * * issueflag = 1 createissue = 0 0 6 * * * diff --git a/cve-vulner-manager/task/grabissue.go b/cve-vulner-manager/task/grabissue.go index 08ee36d..c8f9590 100644 --- a/cve-vulner-manager/task/grabissue.go +++ b/cve-vulner-manager/task/grabissue.go @@ -13,7 +13,7 @@ import ( "github.com/astaxie/beego/logs" ) -//GetIssueData get the issue data +// GetIssueData get the issue data func GetIssueData() error { defer common.Catchs() logs.Info("The task of getting the data source of the created issue starts...") @@ -64,12 +64,6 @@ func GetIssueData() error { if !oki { logs.Error("ProcCveOriginData, GetCveIssueData, err: ", err) } - // Synchronous Data - duplicationDate, ok := BConfig.Int("cve::de_duplication_date") - if ok != nil { - duplicationDate = -30 - } - taskhandler.ProcIssueTemplateBranch(duplicationDate) } } logs.Info("End of syncing gitee data to cve vulnerability database") diff --git a/cve-vulner-manager/taskhandler/grabissue.go b/cve-vulner-manager/taskhandler/grabissue.go index 44d9279..82b5dd0 100644 --- a/cve-vulner-manager/taskhandler/grabissue.go +++ b/cve-vulner-manager/taskhandler/grabissue.go @@ -21,7 +21,7 @@ import ( var wg sync.WaitGroup var issueLock sync.Mutex -//OrgInfo +// OrgInfo type OrgInfo struct { ID int32 `json:"id,omitempty"` Login string `json:"login,omitempty"` @@ -38,26 +38,26 @@ type OrgInfo struct { PrivateRepos int64 `json:"private_repos,omitempty"` } -//Branch Get all branches +// Branch Get all branches type Branch struct { Name string `json:"name,omitempty"` Protected bool `json:"protected,omitempty"` ProtectionURL string `json:"protection_url,omitempty"` } -//PackageInfo package info model +// PackageInfo package info model type PackageInfo struct { Code string Msg string Data Info } -//Info cve info +// Info cve info type Info struct { Description string } -//GrabIssueByOrg grab issue by org name +// GrabIssueByOrg grab issue by org name func GrabIssueByOrg(accToken, org string) error { logs.Info("Synchronize gitee's issue start......") orgInfo, err := GetOrgInfo(accToken, org) @@ -79,17 +79,15 @@ func GrabIssueByOrg(accToken, org string) error { var i int64 for i = 1; i <= pageSize; i++ { - time.Sleep(time.Second * 20) - go GetOrgRepos(accToken, org, i) + GetOrgRepos(accToken, org, i) } - wg.Wait() logs.Info("Synchronize gitee's issue finish...") return nil } -//GrabIssueByRepo grab issue by repository +// GrabIssueByRepo grab issue by repository func GrabIssueByRepo(accToken, owner, repo, state string) { page := 1 product, err := GetInfProduct(accToken, owner, repo) @@ -201,14 +199,14 @@ func handleIssueList(list []models.GitIssue, product, desc string) { } } -//GetOrgInfo get organization information +// GetOrgInfo get organization information func GetOrgInfo(accToken, org string) (OrgInfo, error) { oi := OrgInfo{} var ( resp *http.Response err error ) - + for i := 0; i < 4; i++ { resp, err = http.Get(fmt.Sprintf(GiteOrgInfoURL, org, accToken)) if err == nil { @@ -230,10 +228,8 @@ func GetOrgInfo(accToken, org string) (OrgInfo, error) { return oi, err } -//GetOrgRepos get organization repository +// GetOrgRepos get organization repository func GetOrgRepos(accToken, org string, page int64) { - wg.Add(1) - defer wg.Done() resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) if err != nil { logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) @@ -256,7 +252,7 @@ func GetOrgRepos(accToken, org string, page int64) { } } -//GetIssueList get the repository issue list +// GetIssueList get the repository issue list func GetIssueList(accToken, owner, repo, state string, page int) (issueList []models.GitIssue, err error) { giteUrl := fmt.Sprintf(GiteRepoIssuesURL, owner, repo, accToken, state, page, perPage) resp, err := http.Get(giteUrl) @@ -279,7 +275,7 @@ func GetIssueList(accToken, owner, repo, state string, page int) (issueList []mo return } -//GetRepoDescription get repository description +// GetRepoDescription get repository description func GetRepoDescription(repo string) (desc string) { if repo == "" { return "" -- Gitee From e2e1706e4249cc35a9d7a39382fc9af50cc74f83 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 31 Oct 2025 14:52:30 +0800 Subject: [PATCH 4/4] fix ci --- .../cve-ddd/infrastructure/majunimpl/impl.go | 14 +++++++++++--- cve-vulner-manager/task/issue.go | 6 +++--- cve-vulner-manager/taskhandler/grabissue.go | 6 +----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go index 2cb29ae..2f90f2c 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go @@ -18,8 +18,9 @@ import ( ) const ( - userAgent = "Apifox/1.0.0 (https://apifox.com)" - urlGetPlanDate = "/gateway/openlibing-vulnerability/openlibing-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" + userAgent = "Apifox/1.0.0 (https://apifox.com)" + urlGetPlanDate = "/gateway/openlibing-vulnerability/" + + "openlibing-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" urlReleaseVersion = "/gateway/openlibing-platform-release/publish/externalInterface/queryReleaseVersion" urlCollectCallback = "/gateway/openlibing-platform-release/publish/externalInterface/saveCveCallbackResult" ) @@ -115,6 +116,7 @@ func (impl *majunImpl) CollectCallback(id, data string) error { return nil } +// PlanDataOfMaJun plan data of majun type PlanDataOfMaJun struct { IssueNum string `json:"issueId"` CveNum string `json:"cveNum"` @@ -123,6 +125,7 @@ type PlanDataOfMaJun struct { EndTime string `json:"issuePlanClosedTIme"` } +// PlanResponse response of majun type PlanResponse struct { Code int `json:"code"` Message string `json:"message"` @@ -131,6 +134,7 @@ type PlanResponse struct { } } +// PlanRequest request param type PlanRequest struct { Organization string `json:"organization"` PageNum int `json:"pageNum"` @@ -143,7 +147,11 @@ func (impl *majunImpl) GetPlanDate(page, pageSize int) ([]PlanDataOfMaJun, error PageNum: page, PageSize: pageSize, } - payload, _ := json.Marshal(param) + payload, err := json.Marshal(param) + if err != nil { + return nil, err + } + url := beego.AppConfig.String("majun::host") + urlGetPlanDate req, err := impl.generateRequest(url, payload) diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index f4a94eb..7ec0b61 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -5,13 +5,13 @@ import ( "strings" "time" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "cvevulner/common" "cvevulner/cve-ddd/infrastructure/majunimpl" "cvevulner/models" "cvevulner/taskhandler" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" ) // Verify whether the issue on gitee has been deleted diff --git a/cve-vulner-manager/taskhandler/grabissue.go b/cve-vulner-manager/taskhandler/grabissue.go index 82b5dd0..284f7af 100644 --- a/cve-vulner-manager/taskhandler/grabissue.go +++ b/cve-vulner-manager/taskhandler/grabissue.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "net/http" "strings" - "sync" "time" "github.com/astaxie/beego" @@ -18,10 +17,7 @@ import ( "cvevulner/util" ) -var wg sync.WaitGroup -var issueLock sync.Mutex - -// OrgInfo +// OrgInfo org detail type OrgInfo struct { ID int32 `json:"id,omitempty"` Login string `json:"login,omitempty"` -- Gitee