From 1a53ead8290064cf070ca45123d995b40cec0f74 Mon Sep 17 00:00:00 2001 From: tushenmei Date: Wed, 14 Oct 2020 09:35:45 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=9D=E5=AD=98"sig"=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E5=88=B0=E6=95=B0=E6=8D=AE=E5=BA=93=20=EF=BC=881=EF=BC=89?= =?UTF-8?q?=E6=96=B0=E5=A2=9Eadd=5Fsig=5Finfo.py=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=EF=BC=8C=E5=85=B6=E4=B8=ADread=5Fsigs=5Fcontent()=20=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E4=BB=8Eyaml=E6=96=87=E4=BB=B6=E8=AF=BB=E5=8F=96sig?= =?UTF-8?q?=E4=BF=A1=E6=81=AF;=20parse=5Fsigs=5Fdata()=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E8=AF=BB=E5=8F=96=E7=9A=84sig=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=EF=BC=8C=E8=BF=94=E5=9B=9E=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E4=B8=BAdict,key=E4=B8=BA=E5=8C=85=E5=90=8D?= =?UTF-8?q?=EF=BC=8Cvalue=E4=B8=BAsig=E4=BF=A1=E6=81=AF=EF=BC=9Bsave=5Fsig?= =?UTF-8?q?s=5Fdata()=E5=87=BD=E6=95=B0=E5=B0=86sig=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E5=9C=A8=E6=95=B0=E6=8D=AE=E5=BA=93=E4=B8=AD?= =?UTF-8?q?=EF=BC=8C=E5=BD=93=E8=AF=A5=E5=8C=85=E5=90=8D=E5=9C=A8=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E4=B8=AD=EF=BC=8C=E5=88=99=E6=9B=B4=E6=96=B0?= =?UTF-8?q?sig=E4=BF=A1=E6=81=AF=EF=BC=8C=E5=90=A6=E5=88=99=E5=B0=86?= =?UTF-8?q?=E5=8C=85=E5=90=8D=E5=92=8Csig=E4=BF=A1=E6=81=AF=E5=90=8C?= =?UTF-8?q?=E6=97=B6=E6=B7=BB=E5=8A=A0=E5=88=B0=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E4=B8=AD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit （2）修改constant.py文件，添加了EMPTY_PARSED_DATA这个ResponseCode及其对应的msg。 （3）修改package.py文件，在packages_maintainer这张表中添加了sig字段。 （4）修改_init_.py文件，将save_sigs_data()函数添加到定时任务中，定时保存sig的信息。 （5）修改setup.py文件，添加了packageship.application.apps.package.function.add_sig_info这个新增的文件的py_module。 --- .../packageship/application/__init__.py | 4 + .../apps/package/function/add_sig_info.py | 121 ++++++++++++++++++ .../apps/package/function/constants.py | 5 +- .../packageship/application/models/package.py | 1 + packageship/setup.py | 1 + .../test/common_files/dbs/lifecycle.db | Bin 9990144 -> 9990144 bytes .../common_files/operate_dbs/lifecycle.db | Bin 19021824 -> 19021824 bytes 7 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 packageship/packageship/application/apps/package/function/add_sig_info.py diff --git a/packageship/packageship/application/__init__.py b/packageship/packageship/application/__init__.py index 6a57a2ef..a259fbc2 100644 --- a/packageship/packageship/application/__init__.py +++ b/packageship/packageship/application/__init__.py @@ -20,6 +20,7 @@ def _timed_task(app): # disable=import-outside-toplevel Avoid circular import problems,so import inside the function # pylint: disable=import-outside-toplevel from packageship.application.apps.lifecycle.function.download_yaml import update_pkg_info + from packageship.application.apps.package.function.add_sig_info import save_sigs_data _readconfig = ReadConfig(system_config.SYS_CONFIG_PATH) try: @@ -45,6 +46,9 @@ def _timed_task(app): minute=_minute, args=(False,)) + app.apscheduler.add_job( # pylint: disable=no-member + func=save_sigs_data, id="save_sigs_data", trigger="cron", hour=_hour - 1, minute=_minute) + def init_app(operation): """ diff --git a/packageship/packageship/application/apps/package/function/add_sig_info.py b/packageship/packageship/application/apps/package/function/add_sig_info.py new file mode 100644 index 00000000..451a3877 --- /dev/null +++ b/packageship/packageship/application/apps/package/function/add_sig_info.py @@ -0,0 +1,121 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +read yaml file, parse sig info, save sig info in database +""" +import requests +import yaml +from sqlalchemy.exc import SQLAlchemyError + +from packageship.application.apps.package.function.constants import ResponseCode +from packageship.libs.log import Log +from packageship.application.apps.lifecycle.function.base import Base +from packageship.application.models.package import Packages +from packageship.application.models.package import PackagesMaintainer +from packageship.libs.dbutils import DBHelper + +LOGGER = Log(__name__) + + +def read_sigs_content(): + """ + Description: read sig data from yaml file + Args: + Returns: + sigs_dict + Raises: + FileNotFoundError: yaml file not found + YAMLError: yaml format error + """ + base_control = Base() + sig_url = "https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml" + try: + response = requests.get(sig_url, headers=base_control.headers) + if response.status_code != 200: + return None + sigs_content = yaml.load(response.content, Loader=yaml.FullLoader) + if sigs_content is None or not isinstance(sigs_content, dict): + return None + sigs_dict = {} + for sig in sigs_content['sigs']: + same_sig_pkg = [s for s in sig['repositories']] + for element in same_sig_pkg: + sigs_dict[element] = sig['name'] + return sigs_dict + except (FileNotFoundError, yaml.YAMLError) as error: + base_control.log.logger.error(error) + return None + + +def parse_sigs_data(): + """ + Description: parse sigs data + Args: + Returns: + parsed_sigs_dict: the parsed sig dict according yaml file + """ + parsed_sigs_dict = dict() + sigs_data = read_sigs_content() + if not sigs_data or not isinstance(sigs_data, dict): + return None + for key, value in sigs_data.items(): + if key.startswith('src-openeuler/'): + pkg_name = key.split('/')[1] + parsed_sigs_dict[pkg_name] = value + else: + if key.split('/')[1] not in parsed_sigs_dict.keys(): + pkg_name = key.split('/')[1] + parsed_sigs_dict[pkg_name] = value + return parsed_sigs_dict + + +def save_sigs_data(): + """ + Description: save sig info in database + Args: + Returns: + Raises: + SQLAlchemyError:sqlalchemy error + """ + parsed_sigs_data = parse_sigs_data() + if not parsed_sigs_data: + LOGGER.logger.error(ResponseCode.response_json(ResponseCode.EMPTY_PARSED_DATA)) + return ResponseCode.response_json(ResponseCode.EMPTY_PARSED_DATA) + + lifecycle_pkgname_list = [] + try: + with DBHelper(db_name="lifecycle") as database: + for table_name in filter( + lambda x: x not in ['packages_issue', 'packages_maintainer', 'databases_info'], + database.engine.table_names()): + cls_model = Packages.package_meta(table_name) + table_pkgname = database.session.query(cls_model.name).all() + table_pkgname_list = [x[0] for x in table_pkgname] + lifecycle_pkgname_list.extend(table_pkgname_list) + yaml_lifecycle_pkgname_intersection = { + key: value for key, + value in parsed_sigs_data.items() if key in lifecycle_pkgname_list} + for key, value in yaml_lifecycle_pkgname_intersection.items(): + packages_maintainer_obj = database.session.query( + PackagesMaintainer).filter_by(name=key).first() + if packages_maintainer_obj: + packages_maintainer_obj.sig = value + else: + database.add(PackagesMaintainer( + name=key, sig=value + )) + database.session.commit() + except SQLAlchemyError as error_msg: + LOGGER.logger.error(error_msg) + return ResponseCode.response_json(ResponseCode.UPDATA_OR_ADD_DATA_FAILED) + LOGGER.logger.error(ResponseCode.response_json(ResponseCode.SUCCESS)) diff --git a/packageship/packageship/application/apps/package/function/constants.py b/packageship/packageship/application/apps/package/function/constants.py index 872632b9..70d24d85 100644 --- a/packageship/packageship/application/apps/package/function/constants.py +++ b/packageship/packageship/application/apps/package/function/constants.py @@ -59,6 +59,8 @@ class ResponseCode(): TABLE_NAME_NOT_EXIST_IN_DATABASE = "60003" YAML_FILE_ERROR = " 70001" EMPTY_FOLDER = "70002" + EMPTY_PARSED_DATA = "70003" + CODE_MSG_MAP = { SUCCESS: "Successful Operation!", @@ -86,7 +88,8 @@ class ResponseCode(): TABLE_NAME_NOT_EXIST_IN_DATABASE: "the table name dose not match the existed database", NOT_FOUND_DATABASE_INFO: "Unable to get the generated database information", YAML_FILE_ERROR: "Data error in yaml file", - EMPTY_FOLDER: "This is an empty folder, no yaml file" + EMPTY_FOLDER: "This is an empty folder, no yaml file", + EMPTY_PARSED_DATA: "The parsed yaml data is empty" } @classmethod diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 2d9d2047..d334b045 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -202,6 +202,7 @@ class PackagesMaintainer(DBHelper.BASE): name = Column(String(200), nullable=True) maintainer = Column(String(200), nullable=True) maintainlevel = Column(Integer, nullable=True) + sig = Column(String(200), nullable=True) class DatabaseInfo(DBHelper.BASE): diff --git a/packageship/setup.py b/packageship/setup.py index 5df112c2..4c171691 100644 --- a/packageship/setup.py +++ b/packageship/setup.py @@ -18,6 +18,7 @@ setup( 'packageship.application.apps.package.url', 'packageship.application.apps.package.view', 'packageship.application.apps.package.function.be_depend', + 'packageship.application.apps.package.function.add_sig_info', 'packageship.application.apps.package.function.build_depend', 'packageship.application.apps.package.function.constants', 'packageship.application.apps.package.function.install_depend', diff --git a/packageship/test/common_files/dbs/lifecycle.db b/packageship/test/common_files/dbs/lifecycle.db index 457902d611737b640179d876a3a9e5c67b68c3fa..8df06f9216d5fdd1dd533fd66fdffc7b23837243 100644 GIT binary patch delta 786 zcmXBRNl#O87{%fHZ*K=mYb}V#q@d!&V6o0Pfg?)wb-;N7MZghS!C67%t%@jkomPaz z#EmWpm^5+W+Jzf;E-~pB(8Ps1f5yd=lasSNy;DEzUJMsSCM+u}Wm&#lvdOZns*%cA z$%Iv1`!nSBSzZ&jvj6xq-gXnWBc6mOVd752{cSRCJ{jBoYy#=B=%CLyIraxT`@Qv% zaXiqM4Ege7G1DJ%zStGU?fkYIoW?A_^EDW7#zTH*%4IvFp=x)sqs^)Dvcx3ahaOYf+1JSdR_Zh)t+NJvL(twqhH$V+R_r6T8reICf(X z1beX$`*8qGIEX_yj3YRTV>pfzIEhm@jb^mq49=nz=g@|Bbl^NL;36*JGOnb}Sm)4y z%a@lPF7N6re9I{+T`CEcq{>y5Zj~ODYbw`OZm8T;xutSj<&Mf-m3u0^Dt#*VRr*yP vs0^s6N=oIS$|IG>DuXIdREAWZsytH}RvA%wuHvY?PDj5 delta 962 zcmZY8$xqWk6bJB`uFz6`6vRRi{ZT{_5P!5TAZ`V#f>fSZ6xUj$3R?EX4V6V+RYa*j zx0B&yO!Pp)p@|3o2d_pGO$^~`^y1Ag5D$dIC-dgbOWq`JX5{-1(+EuGIcJ4PjSFGT z!rCl^C`pw0@@GYP^)H9fDl9T6?0-{!TG~yYWYFZ4K^fGY#?Ny>`B9q8lhT$f&?c=? z&oq6ojn><&E}zdZR;3S{OAXd+{hhha7Yek5${aRT^>ju$x&|QH=TTQ@W6_~#Z23S> z*xecS_u`wYu~Ci{4n$gG(Xc*h&Poq=$J)YP&8umZUTrCrzanL6h9;%Xi2kM`Br_utOR+ARQFQ04HQZ7Oa76SPL%5fn0Dy9;}0WSPvUOg#z$E zA#8*q*aVxQ7`8wOltLM7g>u*iT2g8ivL`NQD!>aq*bX~jC+vbssDj;44SQfO?1TMq z0Q^t`wQvvua0m{=5vYTDI0_9Agk#VM1jnHXPCy8n;Ut`b7B~%O;4GYj^Kb#ea1kQV z3YVY_qHr0mKs$86Rk#K*=!7olPRf~{sR@HHCZ~&fd)#Rb`~rhAFO=WPJjVY;aoY`M zj9e_w-!7Ln%MXC5m#L4bpJ{+;kZFkNI@1lNn@qQuZZi!t-C?@RbdPC-X_RSsG|BXsX^QCy(^IBtrUcV7CY|Xy(+j4TOt0S3YxN&z_$0po diff --git a/packageship/test/common_files/operate_dbs/lifecycle.db b/packageship/test/common_files/operate_dbs/lifecycle.db index 27f42bd18c02efbd6be91f531faa4825fae77025..66aeab0f0ec54a2affc730f1a24c5df7dc00d4f3 100644 GIT binary patch delta 3819 zcmZwI349dQ9S88)>}K|y-E1}?$0o1|AVe}q5=0RMLbw6}V#2K$XJ;pyA#*u1n-C3d zP+4kiD-_yjZ7aoEt*r|6NNG`P4_jM{RIT=+we~>3)@`-e+Qa_dzFD-NPd^{N{||ZZ z&HvqR$Hs2+kHNlplw+9kX$<3E!!W)>48tti-`rNqF|(VyxxkwY>p8}RR|L=d?qj!? z9`PLWj7JZR8%NKabVqjE)!wSMwvt;oTSzL>ftFzGKsXpH@f|)eHF&VO(UQ}RiB0W2 z9jn@VX130m)6~ob5{d4f&b945TN7)RZ%xc>lv9mOT%hUEBc%&VJq+UtOf;PIu&!Iq z2lt*2x}XHy;DJ(L!3#d{LjXwI5O5HN2$Vq-${_|7PzjTu3aX(7Y9S7FP!E$~3M8Nb z8eu9-gXwS)%z&BD1Q)|BXa*i;!zC~WTA&r$U@lw=^WZYL9OlCnumBdqm9PjF!xCtR zrO*M(U^%RSm9PprVKuCQE?5ieU_ErhRnP;yumLv0CfE#HU@L5ctKk~h4%b2-Tn7Rq zL4*`Y&<|-C02y|`ASj@M209pUJ(ysD4H+1MVb}>Ha0A>paXvWusO>79cyMfVQ{O~m zcP8YXm~%4oT*?27`?!;?Tz)i{-<8YXl*?yx`LSGncP@W(E`Lide`_wkCzsFV@_Tdn zk35mT&6CKc*f*F+K5`;*Jn}$fJaSVc73q#Fjx_ZIge_at{ecPF=-Q@IV?LaveXh0cXu4jl;{2;CGK2=#{Mg&IQc;M>7d z!4tvb!H0r(1#b*)4=xNg1>O(59yk$rJaA{AYb3BOFh5Y|f7gG?|BU}$|8BqO-{xQG zzuZ6B@AaMWz2rOYyW6+NXZWu5t?|wGHTc5bcf7B8pZ6Z~9`^3>n%=9utGrF#5c?iG z5uH==s;jsx`?wKTXIYA6;xRw7^9Fv0FeIb}y-ajUOv$o!110e!%QV%vn4P+r=6ANV z@EKcHEL+g=&s{ID43En3{ijE(veD~UrkF{{YPQ@)L5reF}mGNv#f zn1iHjE6Y@26vG3uIN%JudJD@;C1p}_P_|ps_F$V+6W#1&m6AzG69l4ASydMY zQ__$#jf!e^*2a|x<33x!{`VM8~+R`a=TIH)>sL${7{-@(vYO^y(4NWqmt`VwGhX+*hhysV%Yq$ z<-MpVUqH`gVFDf9?dwn$zk;6m3uTy;?VK9-%=s+SfwBK@CEqi?{@iGJHhMW-Xj;3n zlg!{UdQu8SOEZ?NVTJh4-lglD6|b9zbt-0DD%<=fNxZNhp1Movk{3R2SJJvE+XE`U zOz!U|!=Fp9kRh59u3zD-ySX;Hg`^N|%3A4dFhqm)=;2zN27*EEX2l#_1Tp|YOY17G zOyLm;UxL(R$w(<$)y;MWA`IFihQw>a_13_JBO2pbrXiN~Gu?tXi2Yw9+xRBlOjlAL zl1xLuv07zFzMpkQE@RHi*88TkfAwai8Ybzeuk9Vcmn6{AjxMs$x&!o z3U3UG=PUNu_HCZ&Tyog?+^=RJNy|&XCMSS`PSww{&kOaIU#Yeek}C`Dss>B~%j{H$mh23nJVDQ( zuF2_)nZW~#(N1BR>14)|$dgMyEEM*Vn@o33Qw4IYE9&WEqRC3XMSlWN8mprVt4X$z zENo{MFJL@QCyG)iIhaL=fq!yo`J1vFBXC87yYv@7N1oFd@A+J!h8hvB@nqU>r zNUX}~Nl43janT}^Xf8El+l9w1TA8I^k`g*+cU=WNO=?C#emTjThRTb&MjxlL7(KG8 z?L14Ka5+7Vs*R@o|J*R6W&)xeoND@qQV_t*HpnrS2yUW~k0R>SZ;|XH_6D7_|{_tbBR>kmL*hQC65NMuU;-k65 z?YYFqa*2=U5}(K=?#LzfY2#PPhv`19!t`;dAhLxCaiw7vM153tyZ#e)zsCG*{%FU3+4#J$px^UxNGL0r)aJ z2w#D(!b9*dJOW4HD0~ebg~#A=cmgKi>u?Od0pEn<@FYA1Ps6w18TdAQ2fhp6gJQ(<7HFPIR}H$scdZO=A1ga7krtUbIffz-@@i_bDQ{ko?O{JAHM%1&ppro zdG2{@PtR>!4;+XEIff}8%`n_@hH)KX7-rVp6B~wd%%mm9eV!MYA?yjp{XW0gbIJAK zkS%Oq?Sawy$#)0r2RCz{oI1hIX5G8DxSZjJhO#|v2cL0FD09Aiu;@5+aOsdrJCgMU zdxGO`z2w<{$zy{uu!93w7y?dk0S9g%b-myN9{do1Ae2K0DxeaoAPhqx0#S%TH4KB{ zFam0z7DmD-7!6}!ER2JzU_4w66JR0;FbO8Z6sUuGXn?734NQY;VLHr!nJ^1x!yK3k z^WZv|4-23X7Q!M}3`<}sG{G`h4$ZIvR>CT1fz_}ET461$gY~chHo_*j9&Uimum!e4 z8{7yY#6f}t$j}Z+=l}(_K_{dj4Jv4mf$gA!0VZUj3%X$ky&A8^gr6O zYkgbZhi#B>HZ%T<{sI3v|6~69{d@hL{*C^n{%QWP{viJ`|2BV~KgXZokMak2gBSVL z{2YD)AK^LQ2fo*R&-%{zj`;5IWqn(Hi+mG(0q>{YSG`YpANJnky~P{%uJqP&7B-7eYH?3&>k>#A^m=Dgq>a6aig?(B7{&MnU7{@Q`; zYi0dY2C~nW^-URYys^liV3}Bm+p$yFCU%KQQS(cs0$nytErWt+oMpyGY&$ePDeS1L z6SAg~GE7m!KYO*vGD5`GR~POL=L0vg%rK-%r}T^@NN8Hoh7vSwH!`}G9BdTWTB;|- zX)e;mmQoj1iJ+v#q+II4QL(v{LrjUPu&v%I4ctI;bQ@_=l~M|F1oPLk%!m-ztzhIG zqTWf$HnB`NV(acuqz-Gu;f*Xa3Pp0fQ!(q4W@m%-BCw&9QI5xDRq8y zdNQTUYC_g!9mW22EE9^@x|HW)H2#P(mEaBrPEeS&38}-oi3>a{%)tC;Z)8mMJi1xQ zXoQ7I68l?dT{_=de@FG)(pKIb?F~Dlb7;=KBhlWzodzb_*%ODv}8fn zP*lu@(A2tMr8U8^X_%9Ut@K_oh4r#@W%eDhYv|n4m-ACeO;^m0G`_XHoeXy>Jq#I1 zm+P(m?cN5uT%-&&{PlEaXQYfZJa1j;wU{9%EHnk18eNNOQcGK#-|m}?te6~9h&r)J zr4fjoW-cQOs<_?g5ZX1>H1M)qV42zw*CI;!POR{9#YACrBAqg=OV%?YzLi$C`Y)eA zPtB?|3vgDgve4CZ6k0}ByE3XZE9K*9W>RQkU8im%iLWXpMwwxxgiNPo)YVz_owegw zW*#<%|McQW9+fM`($ldDm77{tG+BK`$6QGvrGySy#2qkr5W}PC$st{eDZ$X%uNf`RjLOELP_m`$B$*i8mm55=EV3{!? zPLm~p9OG`0uA6r_T_!axlKVO|j9x3Ml4>_fmo*B@tLaRtvYCkw-doWa{W>)-n<+6a zr`rDS{M-2`8iY7Cp%~IMBnU+4PE|#6{b$K)28N>7h)orZ!M#u(E_EU%6>V^u_$rzw zoi$B*Cj}~LEArb~i&(yb?wNEph15!1&@*X4($vyD;}6k2nKrE(#^Ecc8!m0)_QNQA zLF=brklj=fpzD}6%=VO`4tC-5mnvmS*9Xs+#*|6c)i)$XmlpHUL)G`ua9orjFWq42 z6fO~S@UG@Pv`MoPUn6#tk9X70Se8xXtk2*%TB9*%c4#VU1YPJeVryKdD%LO@VJF=( zjk=7)P5=8FXRjK9l?ZW-S~`v(b+i#ZrnwaaW;ubD?-kp_nfedkV$fh2ow<@zz3dZ=txa zP%IRR`wPV{oGISssOfWLaw}|1|C&s0v<=Vh?YZ&fc~sB6NuE7s?kIVV?8;rR-@Xh-|wtf3|1s{Tk;W#`3 zkHR(@LTvDya~UDx8M(O0saVY z!=K>K@E3Ro-i5!yd+2tdv!P@`; -- Gitee