diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 0a6f62e681619b915bb380cf9ae2ab516169d7b4..072811cb19753f264aed86f17a0c2baf3fed0dab 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -33,7 +33,7 @@ saFileDir = "download/sa" gaussFileDir = "download/gauss" # release package download url rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" -communityName = "src-openeuler:1,opengauss:2,mindspore:3,openlookeng:4" +communityName = "src-openeuler:1" [timedtask] @@ -78,7 +78,7 @@ cveflag = 1 getcve = 0 */40 * * * * oricveflag = 1 oricvecheck = 0 0 2 * * * -getissueflag = 2 +getissueflag = 1 getissue = 0 20 1 * * * issueflag = 1 createissue = 0 0 6 * * * @@ -285,6 +285,7 @@ token = "${GITEE_MAJUN_TOKEN||xxx}" api_token = "${MAJUN_API_TOKEN||xxx}" app_id = "${MAJUN_APP_ID||xxx}" secret_key = "${MAJUN_SECRET_KEY||xxx}" +host = https://www.openlibing.com [testResult] host = https://dailybuild.openeuler.openatom.cn diff --git a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go index 42f82def4c6ae6f609ed1472578ba19535d2292c..2f90f2c4e11ebd503909ed8bce68aabca6cefb7b 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/majunimpl/impl.go @@ -18,8 +18,11 @@ import ( ) const ( - urlReleaseVersion = "https://majun.osinfra.cn/api/http/majun-platform-release/publish/externalInterface/queryReleaseVersion" - urlCollectCallback = "https://majun.osinfra.cn/api/http/majun-platform-release/publish/externalInterface/saveCveCallbackResult" + userAgent = "Apifox/1.0.0 (https://apifox.com)" + urlGetPlanDate = "/gateway/openlibing-vulnerability/" + + "openlibing-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" + urlReleaseVersion = "/gateway/openlibing-platform-release/publish/externalInterface/queryReleaseVersion" + urlCollectCallback = "/gateway/openlibing-platform-release/publish/externalInterface/saveCveCallbackResult" ) type config struct { @@ -49,7 +52,8 @@ type releaseResponse struct { } func (impl *majunImpl) GetReleasedBranch() ([]string, error) { - req, err := impl.generateRequest(urlReleaseVersion, nil) + url := beego.AppConfig.String("majun::host") + urlReleaseVersion + req, err := impl.generateRequest(url, nil) if err != nil { return nil, err } @@ -65,15 +69,6 @@ func (impl *majunImpl) GetReleasedBranch() ([]string, error) { } return v.Result, nil - - //return []string{ - // "openEuler-20.03-LTS-SP1", - // "openEuler-20.03-LTS-SP4", - // "openEuler-22.03-LTS", - // "openEuler-22.03-LTS-SP1", - // "openEuler-22.03-LTS-SP2", - // "openEuler-22.03-LTS-SP3", - //}, nil } type callbackBody struct { @@ -102,7 +97,8 @@ func (impl *majunImpl) CollectCallback(id, data string) error { return err } - req, err := impl.generateRequest(urlCollectCallback, payload) + url := beego.AppConfig.String("majun::host") + urlCollectCallback + req, err := impl.generateRequest(url, payload) if err != nil { return err } @@ -120,6 +116,55 @@ func (impl *majunImpl) CollectCallback(id, data string) error { return nil } +// PlanDataOfMaJun plan data of majun +type PlanDataOfMaJun struct { + IssueNum string `json:"issueId"` + CveNum string `json:"cveNum"` + CvssScore float64 `json:"cvssScore"` + BeginTime string `json:"cveRepairStartTime"` + EndTime string `json:"issuePlanClosedTIme"` +} + +// PlanResponse response of majun +type PlanResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result struct { + Data []PlanDataOfMaJun `json:"data"` + } +} + +// PlanRequest request param +type PlanRequest struct { + Organization string `json:"organization"` + PageNum int `json:"pageNum"` + PageSize int `json:"pageSize"` +} + +func (impl *majunImpl) GetPlanDate(page, pageSize int) ([]PlanDataOfMaJun, error) { + param := PlanRequest{ + Organization: "openeuler", + PageNum: page, + PageSize: pageSize, + } + payload, err := json.Marshal(param) + if err != nil { + return nil, err + } + + url := beego.AppConfig.String("majun::host") + urlGetPlanDate + + req, err := impl.generateRequest(url, payload) + if err != nil { + return nil, err + } + + var ret PlanResponse + _, err = impl.client.ForwardTo(req, &ret) + + return ret.Result.Data, err +} + func (impl *majunImpl) generateRequest(url string, body []byte) (req *http.Request, err error) { timestamp := strconv.FormatInt(time.Now().UnixMilli(), 10) sign := impl.sign(timestamp) @@ -134,8 +179,10 @@ func (impl *majunImpl) generateRequest(url string, body []byte) (req *http.Reque } req.Header.Set("timestamp", timestamp) - req.Header.Set("appId", impl.cfg.AppId) + req.Header.Set("accountid", impl.cfg.AppId) req.Header.Set("sign", sign) + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Content-Type", "application/json") return } diff --git a/cve-vulner-manager/task/grabissue.go b/cve-vulner-manager/task/grabissue.go index 08ee36d496a112a9510af187df7ced7caf00d9e6..c8f9590414195dbf39593756ff87e417458ac6f0 100644 --- a/cve-vulner-manager/task/grabissue.go +++ b/cve-vulner-manager/task/grabissue.go @@ -13,7 +13,7 @@ import ( "github.com/astaxie/beego/logs" ) -//GetIssueData get the issue data +// GetIssueData get the issue data func GetIssueData() error { defer common.Catchs() logs.Info("The task of getting the data source of the created issue starts...") @@ -64,12 +64,6 @@ func GetIssueData() error { if !oki { logs.Error("ProcCveOriginData, GetCveIssueData, err: ", err) } - // Synchronous Data - duplicationDate, ok := BConfig.Int("cve::de_duplication_date") - if ok != nil { - duplicationDate = -30 - } - taskhandler.ProcIssueTemplateBranch(duplicationDate) } } logs.Info("End of syncing gitee data to cve vulnerability database") diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index bdf3a03842111c6bae76a90503a3868192822ef5..7ec0b61fb2e1bfbc7d01bd30776b4bae74826266 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -1,21 +1,17 @@ package task import ( - "bytes" - "encoding/json" - "net/http" "strconv" "strings" "time" - "github.com/opensourceways/server-common-lib/utils" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/majunimpl" "cvevulner/models" "cvevulner/taskhandler" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" ) // Verify whether the issue on gitee has been deleted @@ -353,61 +349,26 @@ func SyncPlanDateOfIssueFromMaJun() error { return nil } -type PlanDataOfMaJun struct { - IssueNum string `json:"issueId"` - CveNum string `json:"cveNum"` - CvssScore float64 `json:"cvssScore"` - BeginTime string `json:"cveRepairStartTime"` - EndTime string `json:"issuePlanClosedTIme"` -} - -type PlanResponse struct { - Code int `json:"code"` - Message string `json:"message"` - Result struct { - Data []PlanDataOfMaJun `json:"data"` - } -} - -type PlanRequest struct { - Organization string `json:"organization"` - PageNum int `json:"pageNum"` - PageSize int `json:"pageSize"` -} - -func getPlanData() []PlanDataOfMaJun { - url := "https://majun.osinfra.cn/api/http/majun-vulnerability-view/admin/ci-portal/ci-admin/cve/getTime/details" - token := beego.AppConfig.String("majun::api_token") +func getPlanData() []majunimpl.PlanDataOfMaJun { + cli := majunimpl.NewMajunImpl() pageNum := 1 pageSize := 1000 - cli := utils.NewHttpClient(3) - var data []PlanDataOfMaJun + var allData []majunimpl.PlanDataOfMaJun for { - var ret PlanResponse - param := PlanRequest{ - Organization: "openeuler", - PageNum: pageNum, - PageSize: pageSize, - } - b, _ := json.Marshal(param) - req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(b)) - - req.Header.Add("access_token", token) - - if _, err := cli.ForwardTo(req, &ret); err != nil { - logs.Error("get plan data error: ", err.Error()) + data, err := cli.GetPlanDate(pageNum, pageSize) + if err != nil { + logs.Error("get plan data error: ", err.Error(), pageNum) break } - data = append(data, ret.Result.Data...) - - if len(ret.Result.Data) < pageSize { + allData = append(allData, data...) + if len(data) < pageSize { break } pageNum++ } - return data + return allData } diff --git a/cve-vulner-manager/taskhandler/grabissue.go b/cve-vulner-manager/taskhandler/grabissue.go index 44d92798343a66833a2a14fb0dc96eeb0492bf75..284f7afa7e1008e81a7a9dbcf7d9bbda3d69fca1 100644 --- a/cve-vulner-manager/taskhandler/grabissue.go +++ b/cve-vulner-manager/taskhandler/grabissue.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "net/http" "strings" - "sync" "time" "github.com/astaxie/beego" @@ -18,10 +17,7 @@ import ( "cvevulner/util" ) -var wg sync.WaitGroup -var issueLock sync.Mutex - -//OrgInfo +// OrgInfo org detail type OrgInfo struct { ID int32 `json:"id,omitempty"` Login string `json:"login,omitempty"` @@ -38,26 +34,26 @@ type OrgInfo struct { PrivateRepos int64 `json:"private_repos,omitempty"` } -//Branch Get all branches +// Branch Get all branches type Branch struct { Name string `json:"name,omitempty"` Protected bool `json:"protected,omitempty"` ProtectionURL string `json:"protection_url,omitempty"` } -//PackageInfo package info model +// PackageInfo package info model type PackageInfo struct { Code string Msg string Data Info } -//Info cve info +// Info cve info type Info struct { Description string } -//GrabIssueByOrg grab issue by org name +// GrabIssueByOrg grab issue by org name func GrabIssueByOrg(accToken, org string) error { logs.Info("Synchronize gitee's issue start......") orgInfo, err := GetOrgInfo(accToken, org) @@ -79,17 +75,15 @@ func GrabIssueByOrg(accToken, org string) error { var i int64 for i = 1; i <= pageSize; i++ { - time.Sleep(time.Second * 20) - go GetOrgRepos(accToken, org, i) + GetOrgRepos(accToken, org, i) } - wg.Wait() logs.Info("Synchronize gitee's issue finish...") return nil } -//GrabIssueByRepo grab issue by repository +// GrabIssueByRepo grab issue by repository func GrabIssueByRepo(accToken, owner, repo, state string) { page := 1 product, err := GetInfProduct(accToken, owner, repo) @@ -201,14 +195,14 @@ func handleIssueList(list []models.GitIssue, product, desc string) { } } -//GetOrgInfo get organization information +// GetOrgInfo get organization information func GetOrgInfo(accToken, org string) (OrgInfo, error) { oi := OrgInfo{} var ( resp *http.Response err error ) - + for i := 0; i < 4; i++ { resp, err = http.Get(fmt.Sprintf(GiteOrgInfoURL, org, accToken)) if err == nil { @@ -230,10 +224,8 @@ func GetOrgInfo(accToken, org string) (OrgInfo, error) { return oi, err } -//GetOrgRepos get organization repository +// GetOrgRepos get organization repository func GetOrgRepos(accToken, org string, page int64) { - wg.Add(1) - defer wg.Done() resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) if err != nil { logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) @@ -256,7 +248,7 @@ func GetOrgRepos(accToken, org string, page int64) { } } -//GetIssueList get the repository issue list +// GetIssueList get the repository issue list func GetIssueList(accToken, owner, repo, state string, page int) (issueList []models.GitIssue, err error) { giteUrl := fmt.Sprintf(GiteRepoIssuesURL, owner, repo, accToken, state, page, perPage) resp, err := http.Get(giteUrl) @@ -279,7 +271,7 @@ func GetIssueList(accToken, owner, repo, state string, page int) (issueList []mo return } -//GetRepoDescription get repository description +// GetRepoDescription get repository description func GetRepoDescription(repo string) (desc string) { if repo == "" { return ""