From b3138eee7c5139105ebb79f5d337df82b7f8ba4f Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 15 Jul 2020 19:24:12 +0800 Subject: [PATCH 1/8] Database initialization business logic changes --- .../application/initsystem/data_import.py | 645 ++++++------------ .../application/initsystem/datamerge.py | 377 ---------- .../packageship/application/models/package.py | 131 ++-- .../application/models/temporarydb.py | 86 --- 4 files changed, 304 insertions(+), 935 deletions(-) delete mode 100644 packageship/packageship/application/initsystem/datamerge.py delete mode 100644 packageship/packageship/application/models/temporarydb.py diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 94d76dad..66ebfe94 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -11,17 +11,10 @@ from sqlalchemy.exc import SQLAlchemyError, InternalError from packageship.libs.dbutils.sqlalchemy_helper import DBHelper from packageship.libs.exception import ContentNoneException from packageship.libs.exception import DatabaseRepeatException -from packageship.libs.exception import DataMergeException from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.log import Log -from packageship.application.models.package import bin_pack, src_pack, pack_requires, pack_provides -from packageship.application.initsystem.datamerge import MergeData -from packageship.application.models.temporarydb import src_package -from packageship.application.models.temporarydb import src_requires -from packageship.application.models.temporarydb import bin_package -from packageship.application.models.temporarydb import bin_requiresment -from packageship.application.models.temporarydb import bin_provides +from packageship.application.models.package import src_pack, bin_pack, bin_requires, src_requires, bin_provides, maintenance_info from packageship import system_config LOGGER = Log(__name__) @@ -48,26 +41,19 @@ class InitDataBase(): if self.config_file_path: # yaml configuration file content self.config_file_datas = self.__read_config_file() - self._read_config = ReadConfig() self.db_type = self._read_config.get_database('dbtype') - + self.sql = None + self._database = None + self.mainter_infos = dict() if self.db_type is None: self.db_type = 'mysql' if self.db_type not in ['mysql', 'sqlite']: LOGGER.logger.error("database type configuration error") raise Error('database type configuration error') - self._src_requires_dicts = dict() - self._bin_package_dicts = dict() - self._bin_package_name = dict() - self._bin_requires_dicts = dict() - self._bin_provides_dicts = dict() - self._src_packages = dict() - self._src_package_names = dict() self._sqlite_db = None - self.requires = dict() def __read_config_file(self): """ @@ -124,10 +110,14 @@ class InitDataBase(): # Create a database maintained by benchmark information if self.db_type == 'mysql': MysqlDatabaseOperations( - 'maintenance.information').create_datum_database() + db_name='maintenance.information', + tables=['maintenance_info'], + is_datum=True).create_database() else: SqliteDatabaseOperations( - 'maintenance.information').create_datum_database() + db_name='maintenance.information', + tables=['maintenance_info'], + is_datum=True).create_database() for database in self.config_file_datas: if not database.get('dbname'): @@ -154,15 +144,18 @@ class InitDataBase(): """ db_name = database.get('dbname') - self._sqlite_db = SqliteDatabaseOperations(db_name=db_name) - + tables = ['src_pack', 'bin_pack', + 'bin_requires', 'src_requires', 'bin_provides'] if self.db_type == 'mysql': - creatadatabase = MysqlDatabaseOperations(db_name) + creatadatabase = MysqlDatabaseOperations( + db_name=db_name, tables=tables) if not creatadatabase.create_database(): raise SQLAlchemyError("failed to create database or table") return db_name + self._sqlite_db = SqliteDatabaseOperations( + db_name=db_name, tables=tables) - sqltedb_file = self._sqlite_db.create_sqlite_database() + sqltedb_file = self._sqlite_db.create_database() if sqltedb_file is None: raise SQLAlchemyError( "failed to create database or table") @@ -179,35 +172,25 @@ class InitDataBase(): ContentNoneException: Exception with empty content TypeError: Data type error SQLAlchemyError: Abnormal database operation - DataMergeException: Error in data integration IOError: An error occurred while deleting the database information file """ try: - db_file = None # 1. create a database and related tables in the database db_name = self._create_database(database) # 2. get the data of binary packages and source packages - src_package_paths = database.get('src_db_file') - bin_package_paths = database.get('bin_db_file') + src_db_file = database.get('src_db_file') + bin_db_file = database.get('bin_db_file') - if src_package_paths is None or bin_package_paths is None: + if src_db_file is None or bin_db_file is None: raise ContentNoneException( - 'The configured database file path is empty') - if not isinstance(src_package_paths, list) \ - or not isinstance(bin_package_paths, list): - raise TypeError( - 'The source code and binary path types in the initialization file are abnormal') - # 3. Obtain temporary source package files and binary package files - db_file = self.file_merge( - src_package_paths, bin_package_paths) - # 4. dependencies between combined data - self.data_relationship(db_file) - # 5. save data - self.save_data(db_name) - - except (SQLAlchemyError, ContentNoneException, - DataMergeException, TypeError, Error) as error_msg: + 'The path to the sqlite file in the database initialization configuration \ + is incorrect ') + self._get_maintenance_info() + # # 3. Obtain temporary source package files and binary package files + self.__save_data(src_db_file, bin_db_file, db_name) + + except (SQLAlchemyError, ContentNoneException, TypeError, Error) as error_msg: # Delete the specified database try: if self.db_type == 'mysql': @@ -227,378 +210,255 @@ class InitDataBase(): } InitDataBase.__updata_settings_file( database_content=database_content) - finally: - try: - if os.path.exists(db_file): - os.remove(db_file) - except (IOError, UnboundLocalError) as error_msg: - LOGGER.logger.error(error_msg) - def _src_package_relation(self, src_package_data): + @staticmethod + def __columns(cursor): """ - Description: Mapping of data relations of source packages + Description: functional description:Returns all the column names + queried by the current cursor Args: - src_package_data: Source package data - Returns: + cursor: Cursor + Returns: + The first columns Raises: """ + return [col[0] for col in cursor.description] - _src_package_name = src_package_data.name - _src_package = { - "name": src_package_data.name, - "version": src_package_data.version, - "license": src_package_data.rpm_license, - "sourceURL": src_package_data.url, - "Maintaniner": src_package_data.maintaniner - } - if _src_package_name not in self._src_packages.keys(): - self._src_packages[_src_package_name] = _src_package - else: - # Determine the version number - if src_package_data.version > \ - self._src_packages[_src_package_name]['version']: - - self._src_packages[_src_package_name] = _src_package - # Delete previous version - for key in [names[0] for names in self._src_package_names.items() - if _src_package_name == names[1]]: - del self._src_package_names[key] - - self._src_package_names[src_package_data.pkgKey] = _src_package_name - - def _src_requires_relation(self, src_requires_data): + def __get_data(self): """ - Description: Source package dependent package data relationship mapping + According to different sql statements, query related table data + Args: - src_requires_data: Source package dependent package data + Returns: Raises: """ + if self.sql is None: + return None + try: + src_packages_data = self._database.session.execute(self.sql) + columns = InitDataBase.__columns( + src_packages_data.cursor) + return [dict(zip(columns, row)) for row in src_packages_data.fetchall()] + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) + return None - _src_package_name = self._src_package_names.get( - src_requires_data.pkgKey) - if _src_package_name: - if _src_package_name not in self._src_requires_dicts.keys(): - self._src_requires_dicts[_src_package_name] = [] - self._src_requires_dicts[_src_package_name].append({ - 'name': src_requires_data.name - }) - - def _bin_package_relation(self, bin_package_data): + def __save_data(self, src_db_file, bin_db_file, db_name): """ - Description: Binary package relationship mapping problem + Description: integration of multiple data files Args: - bin_package_data: Binary package data + src_package_paths: Source package database file + bin_package_paths: Binary package database file Returns: - + Path of the generated temporary database file Raises: """ + with DBHelper(db_name=src_db_file, db_type='sqlite:///', import_database=True) \ + as database: + self._database = database + # Save data related to source package + self._save_src_packages(db_name) + self._save_src_requires(db_name) - _bin_pkg_key = bin_package_data.pkgKey - self._bin_package_name[bin_package_data.name] = _bin_pkg_key - - src_package_name = bin_package_data.src_pack_name - _bin_package = { - 'name': bin_package_data.name, - 'version': bin_package_data.version, - 'relation': True - } - if src_package_name not in self._bin_package_dicts.keys(): - self._bin_package_dicts[src_package_name] = [] - - # Determine whether the version number is consistent with the source code package - # If they are the same, an association relationship is established. - for index, bin_package_item in enumerate(self._bin_package_dicts[src_package_name]): - if bin_package_item.get('name') == bin_package_data.name: - if bin_package_item.get('version') < bin_package_data.version: - self._bin_package_dicts[src_package_name][index]['relation'] = False + with DBHelper(db_name=bin_db_file, db_type='sqlite:///', import_database=True)\ + as database: + self._database = database + # Save binary package related data + self._save_bin_packages(db_name) + self._save_bin_requires(db_name) + self._save_bin_provides(db_name) - self._bin_package_dicts[src_package_name].append(_bin_package) - - def _bin_requires_relation(self, bin_requires_data): + def _save_src_packages(self, db_name): """ - Description: Binary package dependency package relationship mapping problem + Save the source package data + Args: - bin_requires_data: Binary package dependency package data + db_name: Saved database name Returns: Raises: """ + # Query all source packages + self.sql = " select * from packages " + packages_datas = self.__get_data() + if packages_datas is None: + raise ContentNoneException( + 'There is no relevant data in the source package provided') + try: + with DBHelper(db_name=db_name) as database: + database.batch_add(packages_datas, src_pack) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) - _bin_pkg_key = bin_requires_data.pkgKey - if _bin_pkg_key: - if _bin_pkg_key not in self._bin_requires_dicts: - self._bin_requires_dicts[_bin_pkg_key] = [] - - self._bin_requires_dicts[_bin_pkg_key].append({ - 'name': bin_requires_data.name - }) - - def _bin_provides_relation(self, bin_provides_data): + def _save_src_requires(self, db_name): """ - Description: Binary package provided by the relationship mapping problem + Args: - bin_provides_data: Component data provided by the binary package + Returns: Raises: """ + # Query all source packages + self.sql = " select * from requires " + requires_datas = self.__get_data() + if requires_datas is None: + LOGGER.logger.warning( + '{db_name}: The package data that the source package depends on is\ + empty'.format(db_name=db_name)) + try: + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, src_requires) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) - _bin_pkg_key = bin_provides_data.pkgKey - if _bin_pkg_key: - if _bin_pkg_key not in self._bin_provides_dicts: - self._bin_provides_dicts[_bin_pkg_key] = [] - self._bin_provides_dicts[_bin_pkg_key].append({ - 'name': bin_provides_data.name - }) - - def data_relationship(self, db_file): + def _save_bin_packages(self, db_name): """ - Description: dependencies between combined data + Save binary package data Args: - db_file: Temporary database file + Returns: Raises: - Error information - """ - self._bin_provides_dicts = dict() - self._bin_requires_dicts = dict() - self._bin_package_name = dict() - self._bin_package_dicts = dict() - self._src_requires_dicts = dict() - self._src_packages = dict() - self._src_package_names = dict() + """ + self.sql = " select * from packages " + bin_packaegs = self.__get_data() + if bin_packaegs is None: + raise ContentNoneException( + '{db_name}:There is no relevant data in the provided \ + binary package '.format(db_name=db_name)) + for index, bin_package_item in enumerate(bin_packaegs): + try: + src_package_name = bin_package_item.get('rpm_sourcerpm').split( + '-' + bin_package_item.get('version'))[0] + except AttributeError as exception_msg: + src_package_name = None + LOGGER.logger.warning(exception_msg) + finally: + bin_packaegs[index]['src_name'] = src_package_name try: - with DBHelper(db_name=db_file, db_type='sqlite:///') as database: - # source package data - for src_package_item in database.session.query(src_package).all(): - self._src_package_relation(src_package_item) - - # combine all dependent packages of source packages - for src_requires_item in database.session.query(src_requires).all(): - self._src_requires_relation(src_requires_item) - - # combine all binary packages - for bin_package_item in database.session.query(bin_package).all(): - self._bin_package_relation(bin_package_item) + with DBHelper(db_name=db_name) as database: + database.batch_add(bin_packaegs, bin_pack) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) - # combine all dependent packages under the current binary package - for bin_requires_item in database.session.query( - bin_requiresment).all(): - self._bin_requires_relation(bin_requires_item) - - # combine the packages provided by the current binary package - - for bin_provides_item in database.session.query(bin_provides).all(): - self._bin_provides_relation(bin_provides_item) - - except Error as error_msg: - LOGGER.logger.error(error_msg) - - def file_merge(self, src_package_paths, bin_package_paths): + def _save_bin_requires(self, db_name): """ - Description: integration of multiple data files + Args: - src_package_paths: Source package database file - bin_package_paths: Binary package database file + Returns: - Path of the generated temporary database file + Raises: - DataMergeException: Abnormal data integration + """ - _db_file = os.path.join( - self._sqlite_db.database_file_folder, 'temporary_database') - - if os.path.exists(_db_file): - os.remove(_db_file) - # create a temporary sqlite database - with DBHelper(db_name=_db_file, db_type='sqlite:///') as database: - tables = ['src_package', 'src_requires', - 'bin_package', 'bin_requiresment', 'bin_provides'] - database.create_table(tables) - - _src_package_key = 1 - # load all source package files and import the files into it - for src_file in src_package_paths: - load_sqlite_data = MergeData(db_file=src_file) - - # Combine data from all source packages - - _src_package_key, src_merge_result = load_sqlite_data.src_file_merge( - src_package_key=_src_package_key, db_file=_db_file) - if not src_merge_result: - raise DataMergeException( - 'abnormal multi-file database integration') - - # load binary package file - _bin_package_key = 1 - for bin_file in bin_package_paths: - load_sqlite_data = MergeData(db_file=bin_file) - - # Combine all binary package data - _bin_package_key, bin_merge_result = load_sqlite_data.bin_file_merge( - bin_package_key=_bin_package_key, db_file=_db_file) - if not bin_merge_result: - raise DataMergeException( - 'abnormal multi-file database integration') - return _db_file + self.sql = " select * from requires " + requires_datas = self.__get_data() + if requires_datas is None: + raise ContentNoneException( + '{db_name}:There is no relevant data in the provided binary \ + dependency package'.format(db_name=db_name)) + try: + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, bin_requires) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) - def __exists_repeat_database(self): + def _save_bin_provides(self, db_name): """ - Description: Determine if the same database name exists + Args: Returns: - True if there are duplicate databases, false otherwise + Raises: """ - db_names = [name.get('dbname') - for name in self.config_file_datas] - - if len(set(db_names)) != len(self.config_file_datas): - return True - - return False + self.sql = " select * from provides " + provides_datas = self.__get_data() + if provides_datas is None: + raise ContentNoneException( + '{db_name}:There is no relevant data in the provided \ + binary component'.format(db_name=db_name)) + try: + with DBHelper(db_name=db_name) as database: + database.batch_add(provides_datas, bin_provides) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) - def _save_bin_package(self, src_packages): + def _get_maintenance_info(self): """ - Description: Save binary package data + Description: Obtain the information of the maintainer Args: - src_packages: Source package data + Returns: - Binary package data + Maintainer related information Raises: - + SQLAlchemyError: An error occurred while executing the sql statement """ - bin_packages = [] - for package_data in src_packages: - try: - bin_package_datas = self._bin_package_dicts.pop( - package_data.name) - except KeyError: - bin_package_datas = None - - if bin_package_datas: - for bin_package_item in bin_package_datas: - bin_package_dict = { - 'name': bin_package_item.get('name'), - 'version': bin_package_item.get('version'), - 'srcIDkey': None - } - if bin_package_item.get('relation'): - bin_package_dict['srcIDkey'] = package_data.id - bin_packages.append(bin_package_dict) - - # source package dependency package - src_requires_datas = self._src_requires_dicts.get( - package_data.name) - if src_requires_datas: - for src_requires_item in src_requires_datas: - requires_name = src_requires_item.get('name') - if requires_name: - if requires_name not in self.requires.keys(): - self.requires[requires_name] = [] - self.requires[requires_name].append({ - 'name': src_requires_item.get('name'), - 'srcIDkey': package_data.id, - 'depProIDkey': None, - 'binIDkey': None - }) - - # organization independent binary package - - for bin_packs in self._bin_package_dicts.values(): - for bin_pack_item in bin_packs: - bin_packages.append({ - 'name': bin_pack_item.get('name'), - 'version': bin_pack_item.get('version'), - 'srcIDkey': None - }) - return bin_packages - - def _save_bin_provides(self, bin_packages): - """ - Description: Save package data provided by binary + try: + if not hasattr(self, 'mainter_infos'): + self.mainter_infos = dict() + with DBHelper(db_name='maintenance.information') as database: + for info in database.session.query(maintenance_info).all(): + if info.name not in self.mainter_infos.keys(): + self.mainter_infos[info.name] = [] + self.mainter_infos[info.name].append({ + 'version': info.version, + 'maintaniner': info.maintaniner, + 'maintainlevel': info.maintainlevel + }) + except SQLAlchemyError as sql_error: + LOGGER.logger.error(sql_error) + + def _get_mainter_info(self, src_package_name, version): + ''' + Get the maintainer information of the source package + Args: - bin_packages: Binary package data + src_package_name: Source package name + version: Source package version number Returns: - Package data provided by binary + Maintainer's name Raises: - """ - bin_provides_list = [] - for bin_pack_entity in bin_packages: - - # Get the pkgKey of the current binary package - pkg_key = self._bin_package_name.get(bin_pack_entity.name) - - if self._bin_requires_dicts.get(pkg_key): - for bin_requires_item in self._bin_requires_dicts.get(pkg_key): - requires_name = bin_requires_item.get('name') - if requires_name: - if requires_name not in self.requires.keys(): - self.requires[requires_name] = [] - self.requires[requires_name].append({ - 'name': bin_requires_item.get('name'), - 'binIDkey': bin_pack_entity.id, - 'depProIDkey': None, - 'srcIDkey': None - }) - - if self._bin_provides_dicts.get(pkg_key): - for bin_provides_item in self._bin_provides_dicts.get(pkg_key): - bin_provides_list.append({ - 'name': bin_provides_item.get('name'), - 'binIDkey': bin_pack_entity.id - }) - return bin_provides_list + ''' + maintenance_infos = self.mainter_infos.get(src_package_name) + maintaniner = None + if maintenance_infos: + for maintenance_item in maintenance_infos: + if maintenance_item.get('version') == version: + maintaniner = (maintenance_item.get( + 'maintaniner'), maintenance_item.get('maintainlevel')) + break + return maintaniner - def save_data(self, db_name): + def __exists_repeat_database(self): """ - Description: save related package data + Description: Determine if the same database name exists Args: - db_name: The name of the database - Returns: + Returns: + True if there are duplicate databases, false otherwise Raises: """ + db_names = [name.get('dbname') + for name in self.config_file_datas] - with DBHelper(db_name=db_name) as data_base: - # Add source package data - data_base.batch_add( - [src_package_item[1] for src_package_item in self._src_packages.items()], src_pack) - - self.requires = dict() - - # Save dependency data of binary packages and source packages - - data_base.batch_add(self._save_bin_package( - data_base.session.query(src_pack).all()), bin_pack) - - # Save all packages and dependent packages provided by the binary package - - data_base.batch_add(self._save_bin_provides( - data_base.session.query(bin_pack).all()), pack_provides) - - all_requires = [] - for provides_item in data_base.session.query(pack_provides).all(): - if provides_item.name in self.requires.keys(): - for requires_item in self.requires[provides_item.name]: - requires_item['depProIDkey'] = provides_item.id - all_requires.append(requires_item) + if len(set(db_names)) != len(self.config_file_datas): + return True - data_base.batch_add(all_requires, pack_requires) + return False @staticmethod def __updata_settings_file(**Kwargs): @@ -669,7 +529,8 @@ class InitDataBase(): if del_result: try: - file_read = open(system_config.DATABASE_FILE_INFO, 'r', encoding='utf-8') + file_read = open( + system_config.DATABASE_FILE_INFO, 'r', encoding='utf-8') _databases = yaml.load( file_read.read(), Loader=yaml.FullLoader) for database in _databases: @@ -693,7 +554,7 @@ class MysqlDatabaseOperations(): drop_database_sql: Delete the SQL statement of the database """ - def __init__(self, db_name): + def __init__(self, db_name, tables=None, is_datum=False): """ Description: Class instance initialization Args: @@ -704,6 +565,8 @@ class MysqlDatabaseOperations(): DEFAULT CHARACTER SET utf8mb4; '''.format(db_name=self.db_name) self.drop_database_sql = '''drop DATABASE if exists `{db_name}` '''.format( db_name=self.db_name) + self.tables = tables + self.is_datum = is_datum def create_database(self): """ @@ -720,9 +583,10 @@ class MysqlDatabaseOperations(): try: # create database - data_base.session.execute(self.drop_database_sql) + if not self.is_datum: + data_base.session.execute(self.drop_database_sql) data_base.session.execute(self.create_database_sql) - except SQLAlchemyError as exception_msg: + except (SQLAlchemyError, InternalError) as exception_msg: LOGGER.logger.error(exception_msg) return False else: @@ -766,9 +630,8 @@ class MysqlDatabaseOperations(): """ try: with DBHelper(db_name=self.db_name) as database: - tables = ['src_pack', 'bin_pack', 'pack_provides', - 'pack_requires'] - database.create_table(tables) + if self.tables: + database.create_table(self.tables) except SQLAlchemyError as exception_msg: LOGGER.logger.error(exception_msg) @@ -776,52 +639,6 @@ class MysqlDatabaseOperations(): else: return True - def create_datum_database(self): - """ - Description: Create a benchmark database to save the maintainer's information - Args: - - Returns: - True if successful, otherwise false - Raises: - SQLAlchemyError: An exception occurred while creating the database - """ - with DBHelper(db_name='mysql') as data_base: - # create database - try: - data_base.session.execute(self.create_database_sql) - except SQLAlchemyError as exception_msg: - LOGGER.logger.error(exception_msg) - return False - else: - # create tables - return self.__create_datum_tables() - - def __create_datum_tables(self): - """ - Description: Create a data table of maintainer information - rgs: - - Returns: - True if successful, otherwise false - Raises: - SQLAlchemyError: An exception occurred while creating the database - Error: Error information - """ - try: - with DBHelper(db_name=self.db_name) as database: - tables = ['maintenance_info'] - database.create_table(tables) - except InternalError as exists_table_err: - LOGGER.logger.error(exists_table_err) - return True - except (SQLAlchemyError, Error) as exception_msg: - LOGGER.logger.error(exception_msg) - return False - - else: - return True - class SqliteDatabaseOperations(): """ @@ -831,7 +648,7 @@ class SqliteDatabaseOperations(): database_file_folder: Database folder path """ - def __init__(self, db_name, **kwargs): + def __init__(self, db_name, tables=None, is_datum=False, ** kwargs): """ Description: Class instance initialization Args: @@ -844,6 +661,8 @@ class SqliteDatabaseOperations(): self._database_file_path() else: self.database_file_folder = kwargs.get('database_path') + self.tables = tables + self.is_datum = is_datum def _database_file_path(self): """ @@ -866,7 +685,7 @@ class SqliteDatabaseOperations(): LOGGER.logger.error(makedirs_error) self.database_file_folder = None - def create_sqlite_database(self): + def create_database(self): """ Description: create sqlite database and table Args: @@ -888,14 +707,13 @@ class SqliteDatabaseOperations(): os.remove(_db_file + '.db') # create a sqlite database - with DBHelper(db_name=_db_file) as database: - tables = ['src_pack', 'bin_pack', - 'pack_requires', 'pack_provides'] - try: - database.create_table(tables) - except SQLAlchemyError as create_table_err: - LOGGER.logger.error(create_table_err) - return None + if (self.is_datum and not os.path.exists(_db_file + '.db')) or not self.is_datum: + with DBHelper(db_name=_db_file) as database: + try: + database.create_table(self.tables) + except (SQLAlchemyError, InternalError) as create_table_err: + LOGGER.logger.error(create_table_err) + return None return _db_file @@ -920,32 +738,3 @@ class SqliteDatabaseOperations(): return False else: return True - - def create_datum_database(self): - """ - Description: create sqlite database and table - Args: - - Returns: - After successful generation, return the database file address, - otherwise return none - Raises: - FileNotFoundError: The specified database folder does not exist - SQLAlchemyError: An error occurred while generating the database - """ - if self.database_file_folder is None: - raise FileNotFoundError('Database folder does not exist') - - _db_file = os.path.join( - self.database_file_folder, self.db_name) - - if not os.path.exists(_db_file + '.db'): - # create a sqlite database - with DBHelper(db_name=_db_file) as database: - tables = ['maintenance_info'] - try: - database.create_table(tables) - except SQLAlchemyError as create_table_err: - LOGGER.logger.error(create_table_err) - return None - return _db_file diff --git a/packageship/packageship/application/initsystem/datamerge.py b/packageship/packageship/application/initsystem/datamerge.py deleted file mode 100644 index 85a6be7e..00000000 --- a/packageship/packageship/application/initsystem/datamerge.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/python3 -""" -Description: Integration of multiple sqlite file data, including reading - sqlite database and inserting data -Class: MergeData -""" -from sqlalchemy.exc import SQLAlchemyError -from packageship.application.models.temporarydb import src_package -from packageship.application.models.temporarydb import src_requires -from packageship.application.models.temporarydb import bin_package -from packageship.application.models.temporarydb import bin_requiresment -from packageship.application.models.temporarydb import bin_provides -from packageship.application.models.package import maintenance_info -from packageship.libs.dbutils import DBHelper -from packageship.libs.log import Log - -LOGGER = Log(__name__) - - -class MergeData(): - """ - Description: Load data from sqlite database - Attributes: - db_file: Database file - db_type: Connected database type - datum_database: Base database name - """ - - def __init__(self, db_file): - """ - Description: Class instance initialization - Args: - db_file: Database file - """ - self.db_file = db_file - self.db_type = 'sqlite:///' - self.datum_database = 'maintenance.information' - self.src_requires_dicts = dict() - self.src_package_datas = [] - self.bin_provides_dicts = dict() - self.bin_package_datas = [] - self.mainter_infos = dict() - self.bin_requires_dicts = dict() - - @staticmethod - def __columns(cursor): - """ - Description: functional description:Returns all the column names - queried by the current cursor - Args: - cursor: Cursor - - Returns: - The first columns - Raises: - - """ - return [col[0] for col in cursor.description] - - def get_package_data(self): - """ - Description: get binary package or source package data - Args: - - Returns: - All source package data queried - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - try: - with DBHelper(db_name=self.db_file, db_type=self.db_type, import_database=True) \ - as database: - src_packages_data = database.session.execute( - "select pkgKey,name,version,rpm_license,url,rpm_sourcerpm from packages") - columns = MergeData.__columns( - src_packages_data.cursor) - return [dict(zip(columns, row)) for row in src_packages_data.fetchall()] - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - return None - - def get_requires_data(self): - """ - Description: get dependent package data of binary package or source package - Args: - - Returns: - All dependent data queried - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - try: - with DBHelper(db_name=self.db_file, db_type=self.db_type, import_database=True) \ - as database: - requires = database.session.execute( - "select pkgKey,name from requires") - columns = MergeData.__columns(requires.cursor) - return [dict(zip(columns, row)) for row in requires.fetchall()] - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - return None - - def get_provides(self): - """ - Description: get the dependency package provided by the binary package - Args: - - Returns: - Query the component data provided by all binary packages - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - try: - with DBHelper(db_name=self.db_file, db_type=self.db_type, import_database=True) \ - as database: - requires = database.session.execute( - "select pkgKey,name from provides") - columns = MergeData.__columns(requires.cursor) - return [dict(zip(columns, row)) for row in requires.fetchall()] - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - return None - - def get_maintenance_info(self): - """ - Description: Obtain the information of the maintainer - Args: - - Returns: - Maintainer related information - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - try: - if not hasattr(self, 'mainter_infos'): - self.mainter_infos = dict() - with DBHelper(db_name=self.datum_database) as database: - for info in database.session.query(maintenance_info).all(): - if info.name not in self.mainter_infos.keys(): - self.mainter_infos[info.name] = [] - self.mainter_infos[info.name].append({ - 'version': info.version, - 'maintaniner': info.maintaniner - }) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - - def src_file_merge(self, src_package_key, db_file): - """ - Description: Source code related data integration - Args: - src_package_key: The relevant key value of the source package - db_file: Database file - Returns: - Key value after successful data combination - (0, False) or (src_package_key, True) - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - self.get_maintenance_info() - - self.__compose_src_package() - - self.__compose_src_rquires() - - # Combination of relationships between source packages and dependent packages - src_requires_data = [] - for src_package_item in self.src_package_datas: - src_package_key += 1 - requires = self.src_requires_dicts.get( - src_package_item.get('pkgKey')) - if requires: - for src_requires_item in requires: - src_requires_item['pkgKey'] = src_package_key - src_requires_data.append(src_requires_item) - src_package_item['pkgKey'] = src_package_key - - try: - with DBHelper(db_name=db_file, db_type=self.db_type) as data_base: - data_base.batch_add(self.src_package_datas, src_package) - data_base.batch_add(src_requires_data, src_requires) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - return (0, False) - else: - return (src_package_key, True) - - def __compose_src_package(self): - """ - Description: Combine source package data - Args: - - Returns: - - Raises: - - """ - if getattr(self, 'src_package_datas', None) is None: - self.src_package_datas = [] - - for src_package_item in self.get_package_data(): - src_package_name = src_package_item.get('name') - if src_package_name: - # Find the maintainer information of the current data - maintenance_infos = self.mainter_infos.get(src_package_name) - maintenance = [] - version = src_package_item.get('version') - if self.mainter_infos.get(src_package_name): - for maintenance_item in maintenance_infos: - if maintenance_item.get('version') == version: - maintenance.append(maintenance_item) - - self.src_package_datas.append( - { - "name": src_package_item.get('name'), - "version": version, - "rpm_license": src_package_item.get('rpm_license'), - "url": src_package_item.get('url'), - "pkgKey": src_package_item.get('pkgKey'), - 'maintaniner': - maintenance[0].get('maintaniner') if maintenance and len( - maintenance) > 0 else None - } - ) - - def __compose_src_rquires(self): - """ - Description: Combine source package dependent package data - Args: - - Returns: - - Raises: - - """ - if getattr(self, 'src_requires_dicts', None) is None: - self.src_requires_dicts = dict() - - for src_requires_item in self.get_requires_data(): - pkg_key = src_requires_item.get('pkgKey') - if pkg_key: - if pkg_key not in self.src_requires_dicts.keys(): - self.src_requires_dicts[pkg_key] = [] - self.src_requires_dicts[pkg_key].append( - { - 'name': src_requires_item.get('name'), - 'pkgKey': pkg_key - } - ) - - def __compose_bin_package(self): - """ - Description: Combine binary package data - Args: - - Returns: - - Raises: - AttributeError - """ - if getattr(self, 'bin_package_datas', None) is None: - self.bin_package_datas = [] - - for bin_package_item in self.get_package_data(): - try: - src_package_name = bin_package_item.get('rpm_sourcerpm').split( - '-' + bin_package_item.get('version'))[0] - except AttributeError as exception_msg: - src_package_name = None - LOGGER.logger.warning(exception_msg) - else: - self.bin_package_datas.append( - { - "name": bin_package_item.get('name'), - "version": bin_package_item.get('version'), - "license": bin_package_item.get('rpm_license'), - "sourceURL": bin_package_item.get('url'), - "src_pack_name": src_package_name, - "pkgKey": bin_package_item.get('pkgKey') - } - ) - - def __compose_bin_requires(self): - """ - Description: Combining binary dependent package data - Args: - - Returns: - - Raises: - """ - if getattr(self, 'bin_requires_dicts', None) is None: - self.bin_requires_dicts = dict() - - for bin_requires_item in self.get_requires_data(): - pkg_key = bin_requires_item.get('pkgKey') - if pkg_key: - if pkg_key not in self.bin_requires_dicts.keys(): - self.bin_requires_dicts[pkg_key] = [] - self.bin_requires_dicts[pkg_key].append({ - 'name': bin_requires_item.get('name'), - 'pkgKey': 0 - }) - - def __compose_bin_provides(self): - """ - Description: Combine binary package data - Args: - - Returns: - - Raises: - - """ - if getattr(self, 'bin_provides_dicts', None) is None: - self.bin_provides_dicts = dict() - - for bin_provides_item in self.get_provides(): - pkg_key = bin_provides_item.get('pkgKey') - if pkg_key: - if pkg_key not in self.bin_provides_dicts.keys(): - self.bin_provides_dicts[pkg_key] = [] - self.bin_provides_dicts[pkg_key].append({ - 'name': bin_provides_item.get('name'), - 'pkgKey': 0 - }) - - def bin_file_merge(self, bin_package_key, db_file): - """ - Description: Binary package related data integration - Args: - bin_package_key: Primary key of binary package - db_file: Database file - Returns: - Key value after successful data combination - (0, False) or (bin_package_key, True) - Raises: - SQLAlchemyError: An error occurred while executing the sql statement - """ - self.__compose_bin_package() - # binary package dependent package integration - - self.__compose_bin_requires() - - self.__compose_bin_provides() - - # integrate the id data of the binary package - bin_requires_datas = [] - bin_provides_datas = [] - for bin_package_item in self.bin_package_datas: - bin_package_key += 1 - # dependent packages - requires = self.bin_requires_dicts.get( - bin_package_item.get('pkgKey')) - if requires: - for bin_requires_item in requires: - bin_requires_item['pkgKey'] = bin_package_key - bin_requires_datas.append(bin_requires_item) - - provides = self.bin_provides_dicts.get( - bin_package_item.get('pkgKey')) - if provides: - for bin_provides_item in provides: - bin_provides_item['pkgKey'] = bin_package_key - bin_provides_datas.append(bin_provides_item) - bin_package_item['pkgKey'] = bin_package_key - # save binary package related data - try: - with DBHelper(db_name=db_file, db_type=self.db_type) as data_base: - data_base.batch_add(self.bin_package_datas, bin_package) - data_base.batch_add(bin_requires_datas, bin_requiresment) - data_base.batch_add(bin_provides_datas, bin_provides) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) - return (0, False) - else: - return (bin_package_key, True) diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index c0f8acb3..27a939bf 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -3,32 +3,42 @@ Description: Database entity model mapping """ from sqlalchemy import Column, Integer, String, ForeignKey -from sqlalchemy.orm import relationship from packageship.libs.dbutils.sqlalchemy_helper import DBHelper class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ - Description: functional description:Source package model + Source package model """ __tablename__ = 'src_pack' - id = Column(Integer, primary_key=True) - - name = Column(String(500), nullable=True) - - version = Column(String(200), nullable=True) - - license = Column(String(500), nullable=True) - - sourceURL = Column(String(200), nullable=True) - - downloadURL = Column(String(200), nullable=True) - - Maintaniner = Column(String(50), nullable=True) - - MaintainLevel = Column(String(20), nullable=True) + pkgKey = Column(Integer, primary_key=True) + pkgId = Column(String(500), nullable=True) + name = Column(String(200), nullable=True) + arch = Column(String(200), nullable=True) + version = Column(String(500), nullable=True) + epoch = Column(String(200), nullable=True) + release = Column(String(500), nullable=True) + summary = Column(String(500), nullable=True) + description = Column(String(500), nullable=True) + url = Column(String(500), nullable=True) + time_file = Column(Integer, nullable=True) + time_build = Column(Integer, nullable=True) + rpm_license = Column(String(500), nullable=True) + rpm_vendor = Column(String(500), nullable=True) + rpm_group = Column(String(500), nullable=True) + rpm_buildhost = Column(String(500), nullable=True) + rpm_sourcerpm = Column(String(500), nullable=True) + rpm_header_start = Column(Integer, nullable=True) + rpm_header_end = Column(Integer, nullable=True) + rpm_packager = Column(String(500), nullable=True) + size_package = Column(Integer, nullable=True) + size_installed = Column(Integer, nullable=True) + size_archive = Column(Integer, nullable=True) + location_href = Column(String(500), nullable=True) + location_base = Column(String(500), nullable=True) + checksum_type = Column(String(500), nullable=True) class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 @@ -37,48 +47,81 @@ class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 """ __tablename__ = 'bin_pack' - id = Column(Integer, primary_key=True) - + pkgKey = Column(Integer, primary_key=True) + pkgId = Column(String(500), nullable=True) name = Column(String(500), nullable=True) - - version = Column(String(200), nullable=True) - - srcIDkey = Column(Integer, ForeignKey('src_pack.id')) - - src_pack = relationship('src_pack', backref="bin_pack") - - -class pack_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 + arch = Column(String(500), nullable=True) + version = Column(String(500), nullable=True) + epoch = Column(String(500), nullable=True) + release = Column(String(500), nullable=True) + summary = Column(String(500), nullable=True) + description = Column(String(500), nullable=True) + url = Column(String(500), nullable=True) + time_file = Column(Integer, nullable=True) + time_build = Column(Integer, nullable=True) + rpm_license = Column(String(500), nullable=True) + rpm_vendor = Column(String(500), nullable=True) + rpm_group = Column(String(500), nullable=True) + rpm_buildhost = Column(String(500), nullable=True) + rpm_sourcerpm = Column(String(500), nullable=True) + rpm_header_start = Column(Integer, nullable=True) + rpm_header_end = Column(Integer, nullable=True) + rpm_packager = Column(String(500), nullable=True) + size_package = Column(Integer, nullable=True) + size_installed = Column(Integer, nullable=True) + size_archive = Column(Integer, nullable=True) + location_href = Column(String(500), nullable=True) + location_base = Column(String(500), nullable=True) + checksum_type = Column(String(500), nullable=True) + src_name = Column(String(500), nullable=True) + + +class bin_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 """ - functional description: + Binary package dependent package entity model """ - __tablename__ = 'pack_requires' + __tablename__ = 'bin_requires' id = Column(Integer, primary_key=True) + name = Column(String(200), nullable=True) + flags = Column(String(200), nullable=True) + epoch = Column(String(200), nullable=True) + version = Column(String(500), nullable=True) + release = Column(String(200), nullable=True) + pkgKey = Column(Integer, nullable=True) + pre = Column(String(20), nullable=True) - name = Column(String(500), nullable=True) - - # depProIDkey = Column(Integer, ForeignKey( - # 'pack_provides.id'), nullable=True) - depProIDkey = Column(Integer) - srcIDkey = Column(Integer, ForeignKey('src_pack.id'), nullable=True) +class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 + """ + Source entity package dependent package entity model + """ + __tablename__ = 'src_requires' - binIDkey = Column(Integer, ForeignKey('bin_pack.id'), nullable=True) + id = Column(Integer, primary_key=True) + name = Column(String(200), nullable=True) + flags = Column(String(200), nullable=True) + epoch = Column(String(200), nullable=True) + version = Column(String(500), nullable=True) + release = Column(String(200), nullable=True) + pkgKey = Column(Integer, nullable=True) + pre = Column(String(20), nullable=True) -class pack_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 +class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 """ - functional description: + Component entity model provided by binary package """ - __tablename__ = 'pack_provides' + __tablename__ = 'bin_provides' id = Column(Integer, primary_key=True) - - name = Column(String(500), nullable=True) - - binIDkey = Column(Integer, ForeignKey('bin_pack.id')) + name = Column(String(200), nullable=True) + flags = Column(String(200), nullable=True) + epoch = Column(String(200), nullable=True) + version = Column(String(500), nullable=True) + release = Column(String(200), nullable=True) + pkgKey = Column(Integer, nullable=True) class maintenance_info(DBHelper.BASE): # pylint: disable=C0103,R0903 diff --git a/packageship/packageship/application/models/temporarydb.py b/packageship/packageship/application/models/temporarydb.py deleted file mode 100644 index 07a2dd17..00000000 --- a/packageship/packageship/application/models/temporarydb.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/python3 -""" -Description: Database entity model mapping -""" -from sqlalchemy import Column, Integer, String -from packageship.libs.dbutils.sqlalchemy_helper import DBHelper - - -class src_package(DBHelper.BASE): # pylint: disable=C0103,R0903 - """ - Description: Temporary source package model - """ - - __tablename__ = 'src_package' - - pkgKey = Column(Integer, primary_key=True) - - name = Column(String(500), nullable=True) - - version = Column(String(200), nullable=True) - - rpm_license = Column(String(500), nullable=True) - - url = Column(String(200), nullable=True) - - maintaniner = Column(String(100), nullable=True) - - -class bin_package(DBHelper.BASE): # pylint: disable=C0103,R0903 - """ - Description: Temporary binary package model - """ - __tablename__ = 'bin_package' - - pkgKey = Column(Integer, primary_key=True) - - name = Column(String(500), nullable=True) - - version = Column(String(200), nullable=True) - - rpm_license = Column(String(500), nullable=True) - - url = Column(String(500), nullable=True) - - rpm_sourcerpm = Column(String(500), nullable=True) - - src_pack_name = Column(String(200), nullable=True) - - -class src_requires(DBHelper.BASE): # pylint: disable=C0103,R0903 - """ - Description: Temporary source package depends on package model - """ - __tablename__ = 'src_requires' - - id = Column(Integer, primary_key=True) - - pkgKey = Column(Integer) - - name = Column(String(500), nullable=True) - - -class bin_requiresment(DBHelper.BASE): # pylint: disable=C0103,R0903 - """ - Description: Dependency package model for temporary binary packages - """ - __tablename__ = 'bin_requiresment' - - id = Column(Integer, primary_key=True) - - pkgKey = Column(Integer) - - name = Column(String(500), nullable=True) - - -class bin_provides(DBHelper.BASE): # pylint: disable=C0103,R0903 - """ - Description: Provided package model for temporary binary packages - """ - __tablename__ = 'bin_provides' - - id = Column(Integer, primary_key=True) - - pkgKey = Column(Integer) - - name = Column(String(500), nullable=True) -- Gitee From 71771972595680567fde6fb14225b92ffae9f0db Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 15 Jul 2020 19:33:56 +0800 Subject: [PATCH 2/8] Change format of pylint --- .../packageship/application/initsystem/data_import.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 66ebfe94..03021730 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -14,7 +14,12 @@ from packageship.libs.exception import DatabaseRepeatException from packageship.libs.exception import Error from packageship.libs.configutils.readconfig import ReadConfig from packageship.libs.log import Log -from packageship.application.models.package import src_pack, bin_pack, bin_requires, src_requires, bin_provides, maintenance_info +from packageship.application.models.package import src_pack +from packageship.application.models.package import bin_pack +from packageship.application.models.package import bin_requires +from packageship.application.models.package import src_requires +from packageship.application.models.package import bin_provides +from packageship.application.models.package import maintenance_info from packageship import system_config LOGGER = Log(__name__) @@ -290,7 +295,8 @@ class InitDataBase(): packages_datas = self.__get_data() if packages_datas is None: raise ContentNoneException( - 'There is no relevant data in the source package provided') + '{db_name}:There is no relevant data in the source \ + package provided'.format(db_name=db_name)) try: with DBHelper(db_name=db_name) as database: database.batch_add(packages_datas, src_pack) -- Gitee From 82579b661768cd781ca2e45b17ce187e9a52ba3d Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 15 Jul 2020 19:58:44 +0800 Subject: [PATCH 3/8] When importing data, add maintainer information --- .../application/initsystem/data_import.py | 95 +++++++++++-------- .../packageship/application/models/package.py | 2 + 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 03021730..04c8c51e 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -29,6 +29,7 @@ class InitDataBase(): """ Description: Database initialization, generate multiple databases and data based on configuration files + Attributes: config_file_path: configuration file path config_file_datas: initialize the configuration content of the database @@ -38,6 +39,7 @@ class InitDataBase(): def __init__(self, config_file_path=None): """ Description: Class instance initialization + Args: config_file_path: Configuration file path """ @@ -62,10 +64,8 @@ class InitDataBase(): def __read_config_file(self): """ - Description: Read the contents of the configuration file load each - node data in the yaml configuration file as - a list to return - Args: + Read the contents of the configuration file load each + node data in the yaml configuration file as a list to return Returns: Initialize the contents of the database configuration file @@ -93,10 +93,7 @@ class InitDataBase(): def init_data(self): """ - Description: Initialization of the database - Args: - - Returns: + Initialization of the database Raises: IOError: An error occurred while deleting the database information file @@ -139,7 +136,8 @@ class InitDataBase(): def _create_database(self, database): """ - Description: create related databases + create related databases + Args: database: Initialize the configuration content of the database Returns: @@ -168,7 +166,8 @@ class InitDataBase(): def _init_data(self, database): """ - Description: data initialization operation + data initialization operation + Args: database: Initialize the configuration content of the database Returns: @@ -203,7 +202,6 @@ class InitDataBase(): database.get('dbname')) else: self._sqlite_db.drop_database() - except (IOError, Error) as exception_msg: LOGGER.logger.error(exception_msg) else: @@ -219,11 +217,11 @@ class InitDataBase(): @staticmethod def __columns(cursor): """ - Description: functional description:Returns all the column names - queried by the current cursor + functional description:Returns all the column names + queried by the current cursor + Args: cursor: Cursor - Returns: The first columns Raises: @@ -255,7 +253,8 @@ class InitDataBase(): def __save_data(self, src_db_file, bin_db_file, db_name): """ - Description: integration of multiple data files + integration of multiple data files + Args: src_package_paths: Source package database file bin_package_paths: Binary package database file @@ -296,7 +295,12 @@ class InitDataBase(): if packages_datas is None: raise ContentNoneException( '{db_name}:There is no relevant data in the source \ - package provided'.format(db_name=db_name)) + package provided '.format(db_name=db_name)) + for index, src_package_item in enumerate(packages_datas): + maintaniner, maintainlevel = self._get_mainter_info( + src_package_item.get('name'), src_package_item.get('version')) + packages_datas[index]['maintaniner'] = maintaniner + packages_datas[index]['maintainlevel'] = maintainlevel try: with DBHelper(db_name=db_name) as database: database.batch_add(packages_datas, src_pack) @@ -305,9 +309,10 @@ class InitDataBase(): def _save_src_requires(self, db_name): """ + Save the dependent package data of the source package Args: - + db_name：Name database Returns: Raises: @@ -329,8 +334,9 @@ class InitDataBase(): def _save_bin_packages(self, db_name): """ Save binary package data - Args: + Args: + db_name：Name database Returns: Raises: @@ -359,9 +365,10 @@ class InitDataBase(): def _save_bin_requires(self, db_name): """ + Save the dependent package data of the binary package Args: - + db_name：Name database Returns: Raises: @@ -381,9 +388,10 @@ class InitDataBase(): def _save_bin_provides(self, db_name): """ + Save the component data provided by the binary package Args: - + db_name：Name database Returns: Raises: @@ -404,7 +412,6 @@ class InitDataBase(): def _get_maintenance_info(self): """ Description: Obtain the information of the maintainer - Args: Returns: Maintainer related information @@ -446,12 +453,13 @@ class InitDataBase(): maintaniner = (maintenance_item.get( 'maintaniner'), maintenance_item.get('maintainlevel')) break + if maintaniner is None: + maintaniner = (None, None) return maintaniner def __exists_repeat_database(self): """ - Description: Determine if the same database name exists - Args: + Determine if the same database name exists Returns: True if there are duplicate databases, false otherwise @@ -469,7 +477,8 @@ class InitDataBase(): @staticmethod def __updata_settings_file(**Kwargs): """ - Description: update some configuration files related to the database in the system + update some configuration files related to the database in the system + Args: **Kwargs: data related to configuration file nodes database_name: Name database @@ -498,7 +507,8 @@ class InitDataBase(): @staticmethod def delete_settings_file(): """ - Description: Delete the configuration file of the database + Delete the configuration file of the database + Args: Returns: @@ -518,7 +528,8 @@ class InitDataBase(): def delete_db(self, db_name): """ - Description: elete the database + delete the database + Args: db_name: The name of the database Returns: @@ -553,7 +564,8 @@ class InitDataBase(): class MysqlDatabaseOperations(): """ - Description: Related to database operations, creating databases, creating tables + Related to database operations, creating databases, creating tables + Attributes: db_name: The name of the database create_database_sql: SQL statement to create a database @@ -562,7 +574,8 @@ class MysqlDatabaseOperations(): def __init__(self, db_name, tables=None, is_datum=False): """ - Description: Class instance initialization + Class instance initialization + Args: db_name: Database name """ @@ -576,8 +589,7 @@ class MysqlDatabaseOperations(): def create_database(self): """ - Description: create a database - Args: + create a mysql database Returns: True if successful, otherwise false @@ -602,7 +614,8 @@ class MysqlDatabaseOperations(): @classmethod def drop_database(cls, db_name): """ - Description: Delete the database according to the specified name + Delete the database according to the specified name + Args: db_name: The name of the database to be deleted Returns: @@ -626,8 +639,7 @@ class MysqlDatabaseOperations(): def __create_tables(self): """ - Description: Create the specified data table - Args: + Create the specified data table Returns: True if successful, otherwise false @@ -648,7 +660,8 @@ class MysqlDatabaseOperations(): class SqliteDatabaseOperations(): """ - Description: sqlite database related operations + sqlite database related operations + Attributes: db_name: Name database database_file_folder: Database folder path @@ -656,7 +669,8 @@ class SqliteDatabaseOperations(): def __init__(self, db_name, tables=None, is_datum=False, ** kwargs): """ - Description: Class instance initialization + Class instance initialization + Args: db_name: Database name kwargs: data related to configuration file nodes @@ -672,8 +686,8 @@ class SqliteDatabaseOperations(): def _database_file_path(self): """ - Description: Database file path - Args: + Database file path + Returns: Raises: @@ -693,8 +707,7 @@ class SqliteDatabaseOperations(): def create_database(self): """ - Description: create sqlite database and table - Args: + create sqlite database and table Returns: After successful generation, return the database file address, @@ -725,15 +738,13 @@ class SqliteDatabaseOperations(): def drop_database(self): """ - Description: Delete the specified sqlite database - Args: + Delete the specified sqlite database Returns: Return true after successful deletion, otherwise return false Raises: IOError: An io exception occurred while deleting the specified database file """ - try: db_path = os.path.join( self.database_file_folder, self.db_name + '.db') diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 27a939bf..3724657e 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -39,6 +39,8 @@ class src_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 location_href = Column(String(500), nullable=True) location_base = Column(String(500), nullable=True) checksum_type = Column(String(500), nullable=True) + maintaniner = Column(String(100), nullable=True) + maintainlevel = Column(String(100), nullable=True) class bin_pack(DBHelper.BASE): # pylint: disable=C0103,R0903 -- Gitee From 6bcfa42707ac327502fa52a0b85afde1d3c4f2e5 Mon Sep 17 00:00:00 2001 From: gongzt Date: Wed, 15 Jul 2020 20:15:51 +0800 Subject: [PATCH 4/8] Query maintainer information optimization --- packageship/packageship/application/initsystem/data_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 04c8c51e..be5953b9 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -121,6 +121,8 @@ class InitDataBase(): tables=['maintenance_info'], is_datum=True).create_database() + self._get_maintenance_info() + for database in self.config_file_datas: if not database.get('dbname'): continue @@ -190,7 +192,6 @@ class InitDataBase(): raise ContentNoneException( 'The path to the sqlite file in the database initialization configuration \ is incorrect ') - self._get_maintenance_info() # # 3. Obtain temporary source package files and binary package files self.__save_data(src_db_file, bin_db_file, db_name) -- Gitee From 050868398cb7227ce9978ce99f7756b23758545e Mon Sep 17 00:00:00 2001 From: gongzt Date: Thu, 16 Jul 2020 09:27:52 +0800 Subject: [PATCH 5/8] Supplement of abnormal information when initializing and saving data --- .../application/initsystem/data_import.py | 127 +++++++++--------- .../packageship/application/models/package.py | 2 +- packageship/setup.py | 2 - 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index be5953b9..ac2c6de4 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -58,8 +58,10 @@ class InitDataBase(): self.db_type = 'mysql' if self.db_type not in ['mysql', 'sqlite']: - LOGGER.logger.error("database type configuration error") - raise Error('database type configuration error') + _msg = "The database type is incorrectly configured.\ + The system temporarily supports only sqlite and mysql databases" + LOGGER.logger.error(_msg) + raise Error(_msg) self._sqlite_db = None def __read_config_file(self): @@ -76,7 +78,8 @@ class InitDataBase(): if not os.path.exists(self.config_file_path): raise FileNotFoundError( - 'system initialization configuration file does not exist') + 'system initialization configuration file \ + does not exist: %s' % self.config_file_path) # load yaml configuration file with open(self.config_file_path, 'r', encoding='utf-8') as file_context: init_database_config = yaml.load( @@ -100,7 +103,8 @@ class InitDataBase(): """ if getattr(self, 'config_file_datas', None) is None or \ self.config_file_datas is None: - raise ContentNoneException('Initialization file content is empty') + raise ContentNoneException('The content of the database initialization \ + configuration file is empty') if self.__exists_repeat_database(): raise DatabaseRepeatException( @@ -120,19 +124,23 @@ class InitDataBase(): db_name='maintenance.information', tables=['maintenance_info'], is_datum=True).create_database() - + # Obtain the maintenance information of the previous data of the system self._get_maintenance_info() for database in self.config_file_datas: if not database.get('dbname'): + LOGGER.logger.error( + 'The database name in the database initialization configuration file is empty') continue priority = database.get('priority') if not isinstance(priority, int) or priority < 0 or priority > 100: + LOGGER.logger.error('The priority value type in the database initialization \ + configuration file is incorrect') continue - status = database.get('status') - if status not in ['enable', 'disable']: + if database.get('status') not in ['enable', 'disable']: + LOGGER.logger.error('The database status value in the database \ + initialization configuration file is incorrect') continue - # Initialization data self._init_data(database) @@ -192,8 +200,16 @@ class InitDataBase(): raise ContentNoneException( 'The path to the sqlite file in the database initialization configuration \ is incorrect ') - # # 3. Obtain temporary source package files and binary package files - self.__save_data(src_db_file, bin_db_file, db_name) + # 3. Obtain temporary source package files and binary package files + if self.__save_data(src_db_file, bin_db_file, db_name): + # Update the configuration file of the database + database_content = { + 'database_name': database.get('dbname'), + 'priority': database.get('priority'), + 'status': database.get('status') + } + InitDataBase.__updata_settings_file( + database_content=database_content) except (SQLAlchemyError, ContentNoneException, TypeError, Error) as error_msg: # Delete the specified database @@ -205,15 +221,6 @@ class InitDataBase(): self._sqlite_db.drop_database() except (IOError, Error) as exception_msg: LOGGER.logger.error(exception_msg) - else: - # Update the configuration file of the database - database_content = { - 'database_name': database.get('dbname'), - 'priority': database.get('priority'), - 'status': database.get('status') - } - InitDataBase.__updata_settings_file( - database_content=database_content) @staticmethod def __columns(cursor): @@ -264,20 +271,26 @@ class InitDataBase(): Raises: """ - with DBHelper(db_name=src_db_file, db_type='sqlite:///', import_database=True) \ - as database: - self._database = database - # Save data related to source package - self._save_src_packages(db_name) - self._save_src_requires(db_name) - - with DBHelper(db_name=bin_db_file, db_type='sqlite:///', import_database=True)\ - as database: - self._database = database - # Save binary package related data - self._save_bin_packages(db_name) - self._save_bin_requires(db_name) - self._save_bin_provides(db_name) + try: + with DBHelper(db_name=src_db_file, db_type='sqlite:///', import_database=True) \ + as database: + self._database = database + # Save data related to source package + self._save_src_packages(db_name) + self._save_src_requires(db_name) + + with DBHelper(db_name=bin_db_file, db_type='sqlite:///', import_database=True)\ + as database: + self._database = database + # Save binary package related data + self._save_bin_packages(db_name) + self._save_bin_requires(db_name) + self._save_bin_provides(db_name) + except (SQLAlchemyError, ContentNoneException) as sql_error: + LOGGER.logger.error(sql_error) + return False + else: + return True def _save_src_packages(self, db_name): """ @@ -302,11 +315,9 @@ class InitDataBase(): src_package_item.get('name'), src_package_item.get('version')) packages_datas[index]['maintaniner'] = maintaniner packages_datas[index]['maintainlevel'] = maintainlevel - try: - with DBHelper(db_name=db_name) as database: - database.batch_add(packages_datas, src_pack) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) + + with DBHelper(db_name=db_name) as database: + database.batch_add(packages_datas, src_pack) def _save_src_requires(self, db_name): """ @@ -323,14 +334,10 @@ class InitDataBase(): self.sql = " select * from requires " requires_datas = self.__get_data() if requires_datas is None: - LOGGER.logger.warning( - '{db_name}: The package data that the source package depends on is\ - empty'.format(db_name=db_name)) - try: - with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, src_requires) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) + raise ContentNoneException('{db_name}: The package data that the source package \ + depends on is empty'.format(db_name=db_name)) + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, src_requires) def _save_bin_packages(self, db_name): """ @@ -358,11 +365,9 @@ class InitDataBase(): LOGGER.logger.warning(exception_msg) finally: bin_packaegs[index]['src_name'] = src_package_name - try: - with DBHelper(db_name=db_name) as database: - database.batch_add(bin_packaegs, bin_pack) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) + + with DBHelper(db_name=db_name) as database: + database.batch_add(bin_packaegs, bin_pack) def _save_bin_requires(self, db_name): """ @@ -381,11 +386,9 @@ class InitDataBase(): raise ContentNoneException( '{db_name}:There is no relevant data in the provided binary \ dependency package'.format(db_name=db_name)) - try: - with DBHelper(db_name=db_name) as database: - database.batch_add(requires_datas, bin_requires) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) + + with DBHelper(db_name=db_name) as database: + database.batch_add(requires_datas, bin_requires) def _save_bin_provides(self, db_name): """ @@ -403,12 +406,10 @@ class InitDataBase(): if provides_datas is None: raise ContentNoneException( '{db_name}:There is no relevant data in the provided \ - binary component'.format(db_name=db_name)) - try: - with DBHelper(db_name=db_name) as database: - database.batch_add(provides_datas, bin_provides) - except SQLAlchemyError as sql_error: - LOGGER.logger.error(sql_error) + binary component '.format(db_name=db_name)) + + with DBHelper(db_name=db_name) as database: + database.batch_add(provides_datas, bin_provides) def _get_maintenance_info(self): """ @@ -420,8 +421,6 @@ class InitDataBase(): SQLAlchemyError: An error occurred while executing the sql statement """ try: - if not hasattr(self, 'mainter_infos'): - self.mainter_infos = dict() with DBHelper(db_name='maintenance.information') as database: for info in database.session.query(maintenance_info).all(): if info.name not in self.mainter_infos.keys(): diff --git a/packageship/packageship/application/models/package.py b/packageship/packageship/application/models/package.py index 3724657e..725f4af7 100644 --- a/packageship/packageship/application/models/package.py +++ b/packageship/packageship/application/models/package.py @@ -2,7 +2,7 @@ """ Description: Database entity model mapping """ -from sqlalchemy import Column, Integer, String, ForeignKey +from sqlalchemy import Column, Integer, String from packageship.libs.dbutils.sqlalchemy_helper import DBHelper diff --git a/packageship/setup.py b/packageship/setup.py index 676e6ccb..b10dc708 100644 --- a/packageship/setup.py +++ b/packageship/setup.py @@ -30,9 +30,7 @@ setup( 'packageship.application.apps.package.function.searchdb', 'packageship.application.apps.package.function.self_depend', 'packageship.application.initsystem.data_import', - 'packageship.application.initsystem.datamerge', 'packageship.application.models.package', - 'packageship.application.models.temporarydb', 'packageship.application.settings', 'packageship.libs.__init__', 'packageship.libs.configutils.readconfig', -- Gitee From 4ff114526a528d9d0266e0b81bdb3ac8f7718100 Mon Sep 17 00:00:00 2001 From: gongzt Date: Tue, 21 Jul 2020 11:09:21 +0800 Subject: [PATCH 6/8] Perform fault-tolerant processing when the file does not exist and initialization fails --- .../application/initsystem/data_import.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index ac2c6de4..9ca67a08 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -200,6 +200,9 @@ class InitDataBase(): raise ContentNoneException( 'The path to the sqlite file in the database initialization configuration \ is incorrect ') + if os.path.exists(src_db_file) or os.path.exists(bin_db_file): + raise FileNotFoundError("sqlite file {src} or {bin} does not exist, please \ + check and try again".format(src=src_db_file, bin=bin_db_file)) # 3. Obtain temporary source package files and binary package files if self.__save_data(src_db_file, bin_db_file, db_name): # Update the configuration file of the database @@ -211,16 +214,20 @@ class InitDataBase(): InitDataBase.__updata_settings_file( database_content=database_content) - except (SQLAlchemyError, ContentNoneException, TypeError, Error) as error_msg: + except (SQLAlchemyError, ContentNoneException, TypeError, + Error, FileNotFoundError) as error_msg: + LOGGER.logger.error(error_msg) # Delete the specified database - try: - if self.db_type == 'mysql': - MysqlDatabaseOperations.drop_database( - database.get('dbname')) - else: - self._sqlite_db.drop_database() - except (IOError, Error) as exception_msg: - LOGGER.logger.error(exception_msg) + self.__del_fail_database(database.get('dbname')) + + def __del_fail_database(self, db_name): + try: + if self.db_type == 'mysql': + MysqlDatabaseOperations.drop_database(db_name) + else: + self._sqlite_db.drop_database() + except (IOError, Error) as exception_msg: + LOGGER.logger.error(exception_msg) @staticmethod def __columns(cursor): @@ -288,6 +295,7 @@ class InitDataBase(): self._save_bin_provides(db_name) except (SQLAlchemyError, ContentNoneException) as sql_error: LOGGER.logger.error(sql_error) + self.__del_fail_database(db_name) return False else: return True -- Gitee From dff80ec4ef35b61f01707e284a504b2a99aff236 Mon Sep 17 00:00:00 2001 From: gongzt Date: Tue, 21 Jul 2020 14:06:41 +0800 Subject: [PATCH 7/8] When deleting the database, return the ID of whether the deletion is successful --- packageship/packageship/application/initsystem/data_import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 9ca67a08..6606a635 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -566,8 +566,10 @@ class InitDataBase(): yaml.safe_dump(_databases, file_context) except (IOError, Error) as del_config_error: LOGGER.logger.error(del_config_error) + del_result = False finally: file_read.close() + return del_result class MysqlDatabaseOperations(): -- Gitee From 3420a7c5526e2b7e50604d93e5e238daf4e21795 Mon Sep 17 00:00:00 2001 From: gongzt Date: Tue, 21 Jul 2020 14:37:20 +0800 Subject: [PATCH 8/8] Delete database consistency changes --- .../application/initsystem/data_import.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/packageship/packageship/application/initsystem/data_import.py b/packageship/packageship/application/initsystem/data_import.py index 6606a635..5d0dd0e6 100644 --- a/packageship/packageship/application/initsystem/data_import.py +++ b/packageship/packageship/application/initsystem/data_import.py @@ -545,30 +545,32 @@ class InitDataBase(): Raises: IOError: File or network operation io abnormal """ - if self.db_type == 'mysql': - del_result = MysqlDatabaseOperations.drop_database(db_name) - else: - if not hasattr(self, '_sqlite_db') or getattr(self, '_sqlite_db') is None: - self._sqlite_db = SqliteDatabaseOperations(db_name=db_name) - del_result = self._sqlite_db.drop_database() + try: + del_result = True + file_read = open( + system_config.DATABASE_FILE_INFO, 'r', encoding='utf-8') + _databases = yaml.load( + file_read.read(), Loader=yaml.FullLoader) + for database in _databases: + if database.get('database_name') == db_name: + _databases.remove(database) + # Delete the successfully imported database configuration node + with open(system_config.DATABASE_FILE_INFO, 'w+', encoding='utf-8') as file_context: + yaml.safe_dump(_databases, file_context) + except (IOError, Error) as del_config_error: + LOGGER.logger.error(del_config_error) + del_result = False + finally: + file_read.close() if del_result: - try: - file_read = open( - system_config.DATABASE_FILE_INFO, 'r', encoding='utf-8') - _databases = yaml.load( - file_read.read(), Loader=yaml.FullLoader) - for database in _databases: - if database.get('database_name') == db_name: - _databases.remove(database) - # Delete the successfully imported database configuration node - with open(system_config.DATABASE_FILE_INFO, 'w+', encoding='utf-8') as file_context: - yaml.safe_dump(_databases, file_context) - except (IOError, Error) as del_config_error: - LOGGER.logger.error(del_config_error) - del_result = False - finally: - file_read.close() + if self.db_type == 'mysql': + del_result = MysqlDatabaseOperations.drop_database(db_name) + else: + if not hasattr(self, '_sqlite_db') or getattr(self, '_sqlite_db') is None: + self._sqlite_db = SqliteDatabaseOperations(db_name=db_name) + del_result = self._sqlite_db.drop_database() + return del_result -- Gitee