diff --git a/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h b/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h index 6652f535ef9938572e2d6c5f2ad603eb9045d027..300067bbb4b9e4f10b231f77aae1791185db99a6 100644 --- a/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h +++ b/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h @@ -75,8 +75,6 @@ public: static constexpr const char *FLAG_ONLY_MODE_NOTIFY = "RESERVE#ALL_CLOUDDATA"; - static constexpr const char *TRACKER_CHECK_PREFIX = "tracker_check_"; - // cloud data timestamp is utc ms precision // used for 100ns to ms when handle cloud data timestamp static constexpr const uint32_t TEN_THOUSAND = 10000; @@ -104,8 +102,6 @@ public: static constexpr const uint32_t ON_CHANGE_KNOWLEDGE = 0x4; static constexpr const uint32_t ON_CHANGE_CLOUD = 0x8; - static constexpr const int64_t ONE_DAY_MS = 24 * 3600 * 1000; - static constexpr std::chrono::milliseconds DFX_TIME_THRESHOLD = std::chrono::milliseconds(1000); }; } // namespace DistributedDB diff --git a/frameworks/libs/distributeddb/common/include/relational/tracker_table.h b/frameworks/libs/distributeddb/common/include/relational/tracker_table.h index 22da2f28a1cbfc673ecbc6ed06b7a9afe64a60ba..7dafdf7a806689f993adb7cf162b1ac01677d43f 100644 --- a/frameworks/libs/distributeddb/common/include/relational/tracker_table.h +++ b/frameworks/libs/distributeddb/common/include/relational/tracker_table.h @@ -27,10 +27,17 @@ using AfterBuildAction = std::function; namespace TriggerMode { enum class TriggerModeEnum; } +class TableInfo; class TrackerTable { public: TrackerTable() = default; ~TrackerTable() {}; + struct RepairInfo { + std::vector createTriggerSqls; + std::string misDataKeys; + bool existNullExtend = false; + bool existDirtyLog = false; + }; std::string GetTableName() const; const std::set &GetTrackerColNames() const; @@ -49,7 +56,6 @@ public: const std::string GetTempTriggerName(TriggerMode::TriggerModeEnum mode) const; const std::string GetTempUpdateTriggerSql(bool incFlag = false) const; const std::string GetTempDeleteTriggerSql(bool incFlag = false) const; - const std::string GetTempUpdateLogCursorTriggerSql() const; void SetTableName(const std::string &tableName); void SetExtendNames(const std::set &colNames); void SetExtendName(const std::string &colName); @@ -62,6 +68,12 @@ public: void SetTriggerObserver(bool isTriggerObserver); void SetKnowledgeTable(bool isKnowledgeTable); std::string GetOnChangeType() const; + void CheckMissingTrigger(const TableInfo &table, sqlite3 *db); + void CheckMismatchedDataKeys(sqlite3 *db); + void CheckNullExtendLog(sqlite3 *db); + void CheckExistDirtyLog(sqlite3 *db); + bool IsNeedRepair(); + void Repair(std::function &)> repairFunc); private: std::string tableName_; std::string extendColName_; @@ -70,6 +82,7 @@ private: bool isTrackerAction_ = false; bool isTriggerObserver_ = true; bool isKnowledgeTable_ = false; + RepairInfo repairInfo_; }; } // namespace DistributedDB diff --git a/frameworks/libs/distributeddb/common/src/relational/tracker_table.cpp b/frameworks/libs/distributeddb/common/src/relational/tracker_table.cpp index 3277b434c947da3a0aa73913c93dce979cd151a7..d02fb0c57edbb87e9366d9d9444e377e8eb9cccb 100644 --- a/frameworks/libs/distributeddb/common/src/relational/tracker_table.cpp +++ b/frameworks/libs/distributeddb/common/src/relational/tracker_table.cpp @@ -17,6 +17,8 @@ #include "db_common.h" #include "tracker_table.h" #include "schema_constant.h" +#include "simple_tracker_log_table_manager.h" +#include "sqlite_relational_utils.h" namespace DistributedDB { void TrackerTable::Init(const TrackerSchema &schema) @@ -284,20 +286,6 @@ const std::string TrackerTable::GetTempDeleteTriggerSql(bool incFlag) const return sql; } -const std::string TrackerTable::GetTempUpdateLogCursorTriggerSql() const -{ - std::string sql = "CREATE TEMP TRIGGER IF NOT EXISTS " + std::string(DBConstant::RELATIONAL_PREFIX) + tableName_; - sql += "LOG_ON_UPDATE_TEMP AFTER UPDATE ON " + DBCommon::GetLogTableName(tableName_); - sql += " WHEN (SELECT 1 FROM " + std::string(DBConstant::RELATIONAL_PREFIX) + "metadata" + - " WHERE key = 'log_trigger_switch' AND value = 'false')\n"; - sql += "BEGIN\n"; - sql += CloudStorageUtils::GetCursorIncSql(tableName_) + "\n"; - sql += "UPDATE " + DBCommon::GetLogTableName(tableName_) + " SET "; - sql += "cursor=" + CloudStorageUtils::GetSelectIncCursorSql(tableName_) + " WHERE data_key = OLD.data_key;\n"; - sql += "END;"; - return sql; -} - void TrackerTable::SetTableName(const std::string &tableName) { tableName_ = tableName; @@ -412,5 +400,56 @@ std::string TrackerTable::GetOnChangeType() const return isKnowledgeTable_ ? std::to_string(CloudDbConstant::ON_CHANGE_KNOWLEDGE) : std::to_string(CloudDbConstant::ON_CHANGE_TRACKER); } + +void TrackerTable::CheckMissingTrigger(const TableInfo &table, sqlite3 *db) +{ + auto tableManager = std::make_unique(); + tableManager->GetMissingTrigger(db, table, "", repairInfo_.createTriggerSqls); +} + +void TrackerTable::CheckMismatchedDataKeys(sqlite3 *db) +{ + std::string misDataKeys = SQLiteRelationalUtils::GetMismatchedDataKeys(db, tableName_); + if (misDataKeys.empty()) { + return; + } + repairInfo_.misDataKeys = misDataKeys; +} + +void TrackerTable::CheckNullExtendLog(sqlite3 *db) +{ + std::string sql = "SELECT COUNT(1) FROM " + DBCommon::GetLogTableName(tableName_) + + " WHERE (json_valid(extend_field) = 0 OR json_type(extend_field) IS NOT 'object' OR" + + " json_extract(extend_field, '$') = '{}') AND data_key != -1"; + if (!SQLiteRelationalUtils::ExecuteCheckSql(db, sql).second) { + return; + } + repairInfo_.existNullExtend = true; +} + +void TrackerTable::CheckExistDirtyLog(sqlite3 *db) +{ + auto [errCode, isExistDirtyLog] = SQLiteRelationalUtils::CheckExistDirtyLog(db, tableName_); + if (errCode != E_OK) { + LOGW("[RDBStore][ClearDirtyLog] Check dirty log failed %d", errCode); + return; + } + repairInfo_.existDirtyLog = isExistDirtyLog; +} + +bool TrackerTable::IsNeedRepair() +{ + return (!repairInfo_.createTriggerSqls.empty() || !repairInfo_.misDataKeys.empty() || + repairInfo_.existNullExtend || repairInfo_.existDirtyLog); +} + +void TrackerTable::Repair(std::function &)> repairFunc) +{ + if (!IsNeedRepair()) { + return; + } + repairFunc(repairInfo_, tableName_, extendColNames_); +} } #endif diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 508733a08d59bac1473085d781ee5cf592265124..4d05f7297506a90b67ba03b167008424a16df71c 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -466,7 +466,7 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName, if (errCode != E_OK) { LOGE("Create distributed table failed. %d", errCode); } else { - CleanDirtyLogIfNeed(tableName, nullptr); + CleanDirtyLogIfNeed(tableName); } if (schemaChanged) { LOGD("Notify schema changed."); @@ -1378,7 +1378,7 @@ int SQLiteRelationalStore::SetTrackerTable(const TrackerSchema &trackerSchema) if (isNoTableInSchema) { errCode = sqliteStorageEngine_->SetTrackerTable(trackerSchema, tableInfo, isFirstCreate); if (errCode == E_OK) { - CleanDirtyLogIfNeed(trackerSchema.tableName, nullptr); + CleanDirtyLogIfNeed(trackerSchema.tableName); } return errCode; } @@ -1943,16 +1943,13 @@ int32_t SQLiteRelationalStore::GetDeviceSyncTaskCount() const return syncAbleEngine_->GetDeviceSyncTaskCount(); } -void SQLiteRelationalStore::CleanDirtyLogIfNeed(const std::string &tableName, - SQLiteSingleVerRelationalStorageExecutor *handle) const +void SQLiteRelationalStore::CleanDirtyLogIfNeed(const std::string &tableName) const { int errCode = E_OK; + SQLiteSingleVerRelationalStorageExecutor *handle = GetHandle(true, errCode); if (handle == nullptr) { - handle = GetHandle(true, errCode); - if (handle == nullptr) { - LOGW("[RDBStore][ClearDirtyLog] Get handle failed %d", errCode); - return; - } + LOGW("[RDBStore][ClearDirtyLog] Get handle failed %d", errCode); + return; } ResFinalizer finalizer([this, handle]() { SQLiteSingleVerRelationalStorageExecutor *releaseHandle = handle; @@ -2000,76 +1997,73 @@ int SQLiteRelationalStore::SetProperty(const Property &property) return E_OK; } -bool SQLiteRelationalStore::IsDailyTrackerIntegrityRepair(const std::string &tableName) -{ - std::string keyStr = CloudDbConstant::TRACKER_CHECK_PREFIX + tableName; - const Key key(keyStr.begin(), keyStr.end()); - Value value; - int errCode = storageEngine_->GetMetaData(key, value); - if (errCode != E_OK && errCode != -E_NOT_FOUND) { - LOGW("[RDBStore] get tracker intergrity check time failed %d", errCode); - return false; - } - int64_t lastTimeMs = 0; - if (!value.empty()) { - int64_t result = std::strtoll(std::string(value.begin(), value.end()).c_str(), - nullptr, DBConstant::STR_TO_LL_BY_DEVALUE); - if (errno != ERANGE && result != LLONG_MIN && result != LLONG_MAX) { - lastTimeMs = result; - } - } - uint64_t curTimeNs = 0; - errCode = TimeHelper::GetSysCurrentRawTime(curTimeNs); - if (errCode != E_OK) { - LOGW("[RDBStore] get cur time failed %d", errCode); - return false; - } - int64_t curTimeMs = curTimeNs / CloudDbConstant::TEN_THOUSAND; - std::random_device rd; - std::mt19937_64 gen(rd()); - std::uniform_int_distribution dis(CloudDbConstant::ONE_DAY_MS, CloudDbConstant::ONE_DAY_MS + - CloudDbConstant::ONE_DAY_MS); - int64_t ranDelayMs = dis(gen); - if (std::abs(curTimeMs - lastTimeMs) < ranDelayMs) { - return false; - } - std::vector curTimeVal; - DBCommon::StringToVector(std::to_string(curTimeMs), curTimeVal); - errCode = storageEngine_->PutMetaData(key, curTimeVal); - if (errCode != E_OK) { - LOGW("[RDBStore] save tracker intergrity check time failed %d", errCode); - return false; - } - return true; -} - void SQLiteRelationalStore::TrackerIntegrityRepair(const TrackerSchema &trackerSchema, const TableInfo &tableInfo, bool isNoTableInSchema) { int errCode = E_OK; - auto *handle = GetHandle(true, errCode); + auto *handle = GetHandle(false, errCode); if (handle == nullptr) { - LOGW("[TrackerIntegrityRepair] get handle failed:%d", errCode); + LOGW("[TrackerIntegrityRepair] get read handle failed:%d", errCode); return; } - handle->CheckAndCreateTrigger(tableInfo); - ReleaseHandle(handle); - if (!IsDailyTrackerIntegrityRepair(trackerSchema.tableName)) { - return; - } - handle = GetHandle(true, errCode); - if (handle == nullptr) { - LOGW("[TrackerIntegrityRepair] get handle err:%d", errCode); + sqlite3 *dbHandle = nullptr; + if (handle->GetDbHandle(dbHandle) != E_OK) { + ReleaseHandle(handle); return; } + TrackerTable trackerTable = tableInfo.GetTrackerTable(); + trackerTable.CheckMissingTrigger(tableInfo, dbHandle); // Try clear historical mismatched log, which usually do not occur and apply to tracker table only. if (isNoTableInSchema) { - handle->ClearLogOfMismatchedData(trackerSchema.tableName); + trackerTable.CheckMismatchedDataKeys(dbHandle); + } + if (!trackerSchema.extendColNames.empty()) { + trackerTable.CheckNullExtendLog(dbHandle); + } + auto obj = GetSchemaObj(); + if (obj.IsSchemaValid()) { + trackerTable.CheckExistDirtyLog(dbHandle); } - handle->RecoverNullExtendLog(trackerSchema, tableInfo.GetTrackerTable()); - CleanDirtyLogIfNeed(trackerSchema.tableName, handle); + ReleaseHandle(handle); LOGI("[TrackerIntegrityRepair] check finish [%s length[%u]]", DBCommon::StringMiddleMasking(trackerSchema.tableName).c_str(), trackerSchema.tableName.length()); + TrackerRepairImpl(trackerTable, obj); +} + +void SQLiteRelationalStore::TrackerRepairImpl(TrackerTable &trackerTable, const RelationalSchemaObject &obj) +{ + trackerTable.Repair([this, &obj] (const TrackerTable::RepairInfo &repairInfo, const std::string &tableName, + const std::set &extendColNames) { + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + LOGW("[TrackerIntegrityRepair] get write handle failed:%d", errCode); + return; + } + sqlite3 *db = nullptr; + if (handle->GetDbHandle(db) != E_OK) { + return; + } + for (const auto &sql : repairInfo.createTriggerSqls) { + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGW("[%s [%zu]] Failed to recreate trigger, errCode=%d", + DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), errCode); + } + } + if (!repairInfo.misDataKeys.empty()) { + SQLiteRelationalUtils::DeleteMismatchLog(db, tableName, repairInfo.misDataKeys); + } + if (repairInfo.existNullExtend) { + handle->RecoverNullExtendLog(tableName, extendColNames, + SQLiteRelationalUtils::GetTempUpdateLogCursorTriggerSql(tableName)); + } + if (repairInfo.existDirtyLog) { + (void) SQLiteRelationalUtils::CleanDirtyLog(db, tableName, obj); + } + ReleaseHandle(handle); + LOGI("[TrackerIntegrityRepair] repair finish"); + }); } } // namespace DistributedDB #endif diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index f5fd3527f2be3184ac89b798f06406e05fa74281..7e9b31b278ffffe952acd169e311abb14e34cda9 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -211,12 +211,12 @@ protected: int OperateDataStatusInner(const std::vector &tables, uint64_t virtualTime) const; - void CleanDirtyLogIfNeed(const std::string &tableName, SQLiteSingleVerRelationalStorageExecutor *handle) const; + void CleanDirtyLogIfNeed(const std::string &tableName) const; void TrackerIntegrityRepair(const TrackerSchema &trackerSchema, const TableInfo &tableInfo, bool isNoTableInSchema); - bool IsDailyTrackerIntegrityRepair(const std::string &tableName); + void TrackerRepairImpl(TrackerTable &trackerTable, const RelationalSchemaObject &obj); RelationalSchemaObject GetSchemaObj() const; // use for sync Interactive diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp index 4d4779a9872ec837b8efeab6c872a1cb7fb7619a..b8310174645aac6bce6f3c986596f29d15c172f3 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp @@ -821,63 +821,6 @@ int SQLiteRelationalUtils::UpdateLocalDataModifyTime(sqlite3 *db, const std::str return errCode; } -std::pair SQLiteRelationalUtils::CheckExistDirtyLog(sqlite3 *db, const std::string &oriTable) -{ - std::pair res; - auto &[errCode, isExist] = res; - if (db == nullptr) { - errCode = -E_INVALID_DB; - return res; - } - auto logTable = DBCommon::GetLogTableName(oriTable); - bool isCreate = false; - errCode = SQLiteUtils::CheckTableExists(db, logTable, isCreate); - if (!isCreate) { - return res; - } - std::string sql = "SELECT count(1) FROM (SELECT data_key FROM " + logTable + " " - "WHERE data_key != -1 GROUP BY data_key HAVING count(1) > 1)"; - return ExecuteCheckSql(db, sql); -} - -std::pair SQLiteRelationalUtils::ExecuteCheckSql(sqlite3 *db, const std::string &sql) -{ - bool isExist = false; - int errCode = ExecuteSql(db, sql, [&isExist](sqlite3_stmt *stmt) { - auto count = static_cast(sqlite3_column_int64(stmt, 0)); - if (count > 0) { - isExist = true; - LOGW("[SQLiteRDBUtils] Exist %" PRId64 " duplicate log", count); - } - }); - return {errCode, isExist}; -} - -int SQLiteRelationalUtils::ExecuteSql(sqlite3 *db, const std::string &sql, - const std::function &checkFunc) -{ - sqlite3_stmt *stmt = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, stmt); - if (errCode != E_OK) { - LOGE("[SQLiteRDBUtils] Get check stmt failed %d", errCode); - return errCode; - } - ResFinalizer finalizer([stmt]() { - int resetRet = E_OK; - sqlite3_stmt *releaseStmt = stmt; - SQLiteUtils::ResetStatement(releaseStmt, true, resetRet); - }); - errCode = SQLiteUtils::StepNext(stmt); - if (errCode == E_OK) { - if (checkFunc != nullptr) { - checkFunc(stmt); - } - } else if (errCode == -E_FINISHED) { - errCode = E_OK; - } - return errCode; -} - int SQLiteRelationalUtils::CleanDirtyLog(sqlite3 *db, const std::string &oriTable, const RelationalSchemaObject &obj) { TableInfo tableInfo; @@ -897,6 +840,8 @@ int SQLiteRelationalUtils::CleanDirtyLog(sqlite3 *db, const std::string &oriTabl errCode = ExecuteSql(db, sql, nullptr); if (errCode == E_OK) { LOGI("[SQLiteRDBUtils] Clean %d dirty hash log", sqlite3_changes(db)); + } else { + LOGW("[SQLiteRDBUtils][ClearDirtyLog] failed %d", errCode); } return errCode; } @@ -1050,4 +995,29 @@ int SQLiteRelationalUtils::BindOneField(sqlite3_stmt *stmt, int bindIdx, const F return -E_INTERNAL_ERROR; } } + +void SQLiteRelationalUtils::DeleteMismatchLog(sqlite3 *db, const std::string &tableName, const std::string &misDataKeys) +{ + std::string delSql = "DELETE FROM " + DBCommon::GetLogTableName(tableName) + " WHERE data_key IN " + + misDataKeys; + int errCode = SQLiteUtils::ExecuteRawSQL(db, delSql); + if (errCode != E_OK) { + LOGW("[%s [%zu]] Failed to del mismatch log, errCode=%d", DBCommon::StringMiddleMasking(tableName).c_str(), + tableName.size(), errCode); + } +} + +const std::string SQLiteRelationalUtils::GetTempUpdateLogCursorTriggerSql(const std::string &tableName) +{ + std::string sql = "CREATE TEMP TRIGGER IF NOT EXISTS " + std::string(DBConstant::RELATIONAL_PREFIX) + tableName; + sql += "LOG_ON_UPDATE_TEMP AFTER UPDATE ON " + DBCommon::GetLogTableName(tableName); + sql += " WHEN (SELECT 1 FROM " + std::string(DBConstant::RELATIONAL_PREFIX) + "metadata" + + " WHERE key = 'log_trigger_switch' AND value = 'false')\n"; + sql += "BEGIN\n"; + sql += CloudStorageUtils::GetCursorIncSql(tableName) + "\n"; + sql += "UPDATE " + DBCommon::GetLogTableName(tableName) + " SET "; + sql += "cursor=" + CloudStorageUtils::GetSelectIncCursorSql(tableName) + " WHERE data_key = OLD.data_key;\n"; + sql += "END;"; + return sql; +} } // namespace DistributedDB \ No newline at end of file diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h index 1924ae26f02fae5046e3a23f5e7cea78992f69ef..8b63f15df3da076a887028b29a27467db28bcb50 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h @@ -119,6 +119,12 @@ public: VBucket &distributedPk); static int BindOneField(sqlite3_stmt *stmt, int bindIdx, const FieldInfo &fieldInfo, VBucket &distributedPk); + + static std::string GetMismatchedDataKeys(sqlite3 *db, const std::string &tableName); + + static void DeleteMismatchLog(sqlite3 *db, const std::string &tableName, const std::string &misDataKeys); + + static const std::string GetTempUpdateLogCursorTriggerSql(const std::string &tableName); private: static int BindExtendStatementByType(sqlite3_stmt *statement, int cid, Type &typeVal); diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils_client.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils_client.cpp index d56bbecefd7e3acbe658c3dbee1ebed1fecda678..56f327707b8fa42a0b116584291221dd463a9ff8 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils_client.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils_client.cpp @@ -332,4 +332,97 @@ int SQLiteRelationalUtils::GeneTimeStrForLog(const TableInfo &tableInfo, GenLogP } return E_OK; } + +std::string SQLiteRelationalUtils::GetMismatchedDataKeys(sqlite3 *db, const std::string &tableName) +{ + bool isLogTableExist = false; + int errCode = SQLiteUtils::CheckTableExists(db, DBCommon::GetLogTableName(tableName), isLogTableExist); + if (!isLogTableExist) { + return ""; + } + std::string sql = "SELECT data_key from " + DBCommon::GetLogTableName(tableName) + " WHERE data_key NOT IN " + + "(SELECT _rowid_ FROM " + tableName + ") AND data_key != -1"; + sqlite3_stmt *stmt = nullptr; + errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + LOGW("[RDBExecutor][ClearMisLog] Get stmt failed, %d", errCode); + return ""; + } + std::vector dataKeys; + std::string misDataKeys = "("; + errCode = SQLiteUtils::StepWithRetry(stmt, false); + while (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + int dataKey = sqlite3_column_int64(stmt, 0); + dataKeys.push_back(dataKey); + misDataKeys += std::to_string(dataKey) + ","; + errCode = SQLiteUtils::StepWithRetry(stmt, false); + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + stmt = nullptr; + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGW("[RDBExecutor][ClearMisLog] Step failed. %d", errCode); + return ""; + } + if (dataKeys.empty()) { + return ""; + } + misDataKeys.pop_back(); + misDataKeys += ")"; + LOGI("[RDBExecutor][ClearMisLog] Mismatched:%s", misDataKeys.c_str()); + return misDataKeys; +} + +std::pair SQLiteRelationalUtils::ExecuteCheckSql(sqlite3 *db, const std::string &sql) +{ + bool isExist = false; + int errCode = ExecuteSql(db, sql, [&isExist](sqlite3_stmt *stmt) { + auto count = static_cast(sqlite3_column_int64(stmt, 0)); + if (count > 0) { + isExist = true; + LOGW("[SQLiteRDBUtils] Exist %" PRId64 " duplicate log", count); + } + }); + return {errCode, isExist}; +} + +int SQLiteRelationalUtils::ExecuteSql(sqlite3 *db, const std::string &sql, + const std::function &checkFunc) +{ + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + LOGE("[SQLiteRDBUtils] Get check stmt failed %d", errCode); + return errCode; + } + errCode = SQLiteUtils::StepNext(stmt); + if (errCode == E_OK) { + if (checkFunc != nullptr) { + checkFunc(stmt); + } + } else if (errCode == -E_FINISHED) { + errCode = E_OK; + } + int resetRet = E_OK; + SQLiteUtils::ResetStatement(stmt, true, resetRet); + return errCode; +} + +std::pair SQLiteRelationalUtils::CheckExistDirtyLog(sqlite3 *db, const std::string &oriTable) +{ + std::pair res; + auto &[errCode, isExist] = res; + if (db == nullptr) { + errCode = -E_INVALID_DB; + return res; + } + auto logTable = DBCommon::GetLogTableName(oriTable); + bool isCreate = false; + errCode = SQLiteUtils::CheckTableExists(db, logTable, isCreate); + if (!isCreate) { + return res; + } + std::string sql = "SELECT count(1) FROM (SELECT data_key FROM " + logTable + " " + "WHERE data_key != -1 GROUP BY data_key HAVING count(1) > 1)"; + return ExecuteCheckSql(db, sql); +} } // namespace DistributedDB \ No newline at end of file diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h index c506a3b18622e33cd395abb12c1e22ffb9ded6fe..cd4d9eaed48847fb105e3de8f08a6892c58483be 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h @@ -260,6 +260,8 @@ public: void CheckAndCreateTrigger(const TableInfo &table); + bool CheckNullExtendLog(const TrackerTable &table); + int GetLockStatusByGid(const std::string &tableName, const std::string &gid, LockStatus &status); int CompareSchemaTableColumns(const std::string &tableName); @@ -275,7 +277,8 @@ public: int IsTableOnceDropped(const std::string &tableName, bool &onceDropped); - void RecoverNullExtendLog(const TrackerSchema &trackerSchema, const TrackerTable &table); + void RecoverNullExtendLog(const std::string &tableName, + const std::set &extendColNames, const std::string &sql); int ConvertLogToLocal(const std::string &tableName, const std::vector &gids); private: @@ -538,7 +541,8 @@ private: bool AbortGetDownloadAssetGidIfNeed(const TableSchema &tableSchema, const std::string &gid, bool abortWithLimit, uint32_t &count); - int RecoverNullExtendLogInner(const TrackerSchema &trackerSchema, const TrackerTable &table); + int RecoverNullExtendLogInner(const std::string &tableName, + const std::set &extendColNames, const std::string &sql); static constexpr const char *CONSISTENT_FLAG = "0x20"; static constexpr const char *UPDATE_FLAG_CLOUD = "flag = 0"; diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp index 6d67e5a4df82235b522468906941a948797201f6..21fd2234b386118022665a0a1e46e21d8d6727f8 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp @@ -1995,71 +1995,22 @@ void SQLiteSingleVerRelationalStorageExecutor::CheckAndCreateTrigger(const Table void SQLiteSingleVerRelationalStorageExecutor::ClearLogOfMismatchedData(const std::string &tableName) { - bool isLogTableExist = false; - int errCode = SQLiteUtils::CheckTableExists(dbHandle_, DBCommon::GetLogTableName(tableName), isLogTableExist); - if (!isLogTableExist) { + std::string misDataKeys = SQLiteRelationalUtils::GetMismatchedDataKeys(dbHandle_, tableName); + if (misDataKeys.empty()) { return; } - std::string sql = "SELECT data_key from " + DBCommon::GetLogTableName(tableName) + " WHERE data_key NOT IN " + - "(SELECT _rowid_ FROM " + tableName + ") AND data_key != -1"; - sqlite3_stmt *stmt = nullptr; - errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); - if (errCode != E_OK) { - LOGW("[RDBExecutor][ClearMisLog] Get stmt failed, %d", errCode); - return; - } - std::vector dataKeys; - std::string misDataKeys = "("; - errCode = SQLiteUtils::StepWithRetry(stmt, false); - while (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - int dataKey = sqlite3_column_int64(stmt, 0); - dataKeys.push_back(dataKey); - misDataKeys += std::to_string(dataKey) + ","; - errCode = SQLiteUtils::StepWithRetry(stmt, false); - } - SQLiteUtils::ResetStatement(stmt, true, errCode); - stmt = nullptr; - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGW("[RDBExecutor][ClearMisLog] Step failed. %d", errCode); - return; - } - if (dataKeys.empty()) { - return; - } - misDataKeys.pop_back(); - misDataKeys += ")"; - LOGW("[RDBExecutor][ClearMisLog] Mismatched:%s", misDataKeys.c_str()); - std::string delSql = "DELETE FROM " + DBCommon::GetLogTableName(tableName) + " WHERE data_key IN " + misDataKeys; - errCode = SQLiteUtils::GetStatement(dbHandle_, delSql, stmt); - if (errCode != E_OK) { - LOGW("[RDBExecutor][ClearMisLog] Get del stmt failed, %d", errCode); - return; - } - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGW("[RDBExecutor][ClearMisLog] Step del failed, %d", errCode); - } - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteRelationalUtils::DeleteMismatchLog(dbHandle_, tableName, misDataKeys); } -void SQLiteSingleVerRelationalStorageExecutor::RecoverNullExtendLog(const TrackerSchema &trackerSchema, - const TrackerTable &table) +void SQLiteSingleVerRelationalStorageExecutor::RecoverNullExtendLog(const std::string &tableName, + const std::set &extendColNames, const std::string &sql) { - if (trackerSchema.extendColNames.empty()) { - return; - } - std::string sql = "SELECT COUNT(1) FROM " + DBCommon::GetLogTableName(trackerSchema.tableName) + - " WHERE (json_valid(extend_field) = 0 OR json_type(extend_field) IS NOT 'object' OR" + - " json_extract(extend_field, '$') = '{}') AND data_key != -1"; - if (!SQLiteRelationalUtils::ExecuteCheckSql(dbHandle_, sql).second) { - return; - } auto errCode = SQLiteUtils::BeginTransaction(dbHandle_, TransactType::IMMEDIATE); if (errCode != E_OK) { LOGE("[RDBExecutor][RecoverTracker] Begin transaction fail %d", errCode); return; } - errCode = RecoverNullExtendLogInner(trackerSchema, table); + errCode = RecoverNullExtendLogInner(tableName, extendColNames, sql); int changeRow = 0; if (errCode == E_OK) { changeRow = sqlite3_changes(dbHandle_); @@ -2068,22 +2019,22 @@ void SQLiteSingleVerRelationalStorageExecutor::RecoverNullExtendLog(const Tracke (void)SQLiteUtils::RollbackTransaction(dbHandle_); } LOGI("[RDBExecutor][RecoverTracker] Recover tracker[%s][%zu] finished[%d] changeRow[%d]", - DBCommon::StringMiddleMasking(trackerSchema.tableName).c_str(), trackerSchema.tableName.size(), + DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), errCode, changeRow); } -int SQLiteSingleVerRelationalStorageExecutor::RecoverNullExtendLogInner(const TrackerSchema &trackerSchema, - const TrackerTable &table) +int SQLiteSingleVerRelationalStorageExecutor::RecoverNullExtendLogInner(const std::string &tableName, + const std::set &extendColNames, const std::string &sql) { std::vector> actions; actions.emplace_back([this]() { return SetLogTriggerStatus(false); }); - actions.emplace_back([this, &table]() { - return SQLiteUtils::ExecuteRawSQL(dbHandle_, table.GetTempUpdateLogCursorTriggerSql()); + actions.emplace_back([this, &sql]() { + return SQLiteUtils::ExecuteRawSQL(dbHandle_, sql); }); - actions.emplace_back([this, &trackerSchema]() { - return UpdateExtendField(trackerSchema.tableName, trackerSchema.extendColNames, + actions.emplace_back([this, &tableName, &extendColNames]() { + return UpdateExtendField(tableName, extendColNames, " AND (json_valid(log.extend_field) = 0 OR json_type(log.extend_field) IS NOT 'object' OR" " json_extract(extend_field, '$') = '{}')"); }); diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp index b612ab10efcd7f41a6beb3c3a2853507df1aed7c..b4b85c36be8175de743ebb4009fdde6bc8257039 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp @@ -238,6 +238,20 @@ int CheckTriggerExist(sqlite3 *db, const TableInfo &table, const std::string &tr void SqliteLogTableManager::CheckAndCreateTrigger(sqlite3 *db, const TableInfo &table, const std::string &identity) { std::vector sqls; + GetMissingTrigger(db, table, identity, sqls); + const std::string &tableName = table.GetTableName(); + for (const auto &sql : sqls) { + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGW("[%s [%zu]] Failed to recreate trigger, errCode=%d", DBCommon::StringMiddleMasking(tableName).c_str(), + tableName.size(), errCode); + } + } +} + +void SqliteLogTableManager::GetMissingTrigger(sqlite3 *db, const TableInfo &table, const std::string &identity, + std::vector &createTriggerSqls) +{ bool insertTriggerExist = false; const std::string &tableName = table.GetTableName(); if (CheckTriggerExist(db, table, "INSERT", insertTriggerExist) == E_OK && !insertTriggerExist) { @@ -245,7 +259,7 @@ void SqliteLogTableManager::CheckAndCreateTrigger(sqlite3 *db, const TableInfo & DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); std::string insertTriggerSql = GetInsertTrigger(table, identity); if (!insertTriggerSql.empty()) { - sqls.emplace_back(insertTriggerSql); + createTriggerSqls.emplace_back(insertTriggerSql); } } @@ -255,7 +269,7 @@ void SqliteLogTableManager::CheckAndCreateTrigger(sqlite3 *db, const TableInfo & DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); std::string updateTriggerSql = GetUpdateTrigger(table, identity); if (!updateTriggerSql.empty()) { - sqls.emplace_back(updateTriggerSql); + createTriggerSqls.emplace_back(updateTriggerSql); } } @@ -265,15 +279,7 @@ void SqliteLogTableManager::CheckAndCreateTrigger(sqlite3 *db, const TableInfo & DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); std::string deleteTriggerSql = GetDeleteTrigger(table, identity); if (!deleteTriggerSql.empty()) { - sqls.emplace_back(deleteTriggerSql); - } - } - - for (const auto &sql : sqls) { - int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); - if (errCode != E_OK) { - LOGW("[%s [%zu]] Failed to recreate trigger, errCode=%d", DBCommon::StringMiddleMasking(tableName).c_str(), - tableName.size(), errCode); + createTriggerSqls.emplace_back(deleteTriggerSql); } } } diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h index b583f3b2ed0ed884e5226d63efafab76971ff062..19c3d0cbe8c60b8df25fbb2c6229d5cb0e032c51 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h +++ b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h @@ -37,6 +37,9 @@ public: void CheckAndCreateTrigger(sqlite3 *db, const TableInfo &table, const std::string &identity); + void GetMissingTrigger(sqlite3 *db, const TableInfo &table, const std::string &identity, + std::vector &createTriggerSqls); + std::string GetCreateRelationalLogTableSql(const TableInfo &table, const std::string &extendStr = ""); virtual std::string GetConflictPkSql(const TableInfo &table); diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_client.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_client.cpp index 4825ffda9691f898a7041e1860908105a17182c3..7d508af6e25cd85cd0aae2ba6e3c6b4e7dfc96a8 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_client.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_client.cpp @@ -648,4 +648,18 @@ int SQLiteUtils::BindInt64ToStatement(sqlite3_stmt *statement, int index, int64_ return E_OK; } + +int SQLiteUtils::StepNext(sqlite3_stmt *stmt, bool isMemDb) +{ + if (stmt == nullptr) { + return -E_INVALID_ARGS; + } + int errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_FINISHED; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = E_OK; + } + return errCode; +} } // namespace DistributedDB diff --git a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_extend.cpp b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_extend.cpp index 7100ec7890170e218e6b8cf7a06877ed8eeec44f..ecb0db3cc36bc2b502340421544edbbf5b885d49 100644 --- a/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_extend.cpp +++ b/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils_extend.cpp @@ -643,20 +643,6 @@ int SQLiteUtils::UpdateCipherShaAlgo(sqlite3 *db, bool setWal, CipherType type, return Rekey(db, passwd); } -int SQLiteUtils::StepNext(sqlite3_stmt *stmt, bool isMemDb) -{ - if (stmt == nullptr) { - return -E_INVALID_ARGS; - } - int errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - errCode = -E_FINISHED; - } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - errCode = E_OK; - } - return errCode; -} - bool SQLiteUtils::IsStmtReadOnly(sqlite3_stmt *statement) { if (statement == nullptr) { diff --git a/frameworks/libs/distributeddb/test/unittest/common/storage/virtual_sqlite_relational_store.cpp b/frameworks/libs/distributeddb/test/unittest/common/storage/virtual_sqlite_relational_store.cpp index 024b3261a8f487e620691f088b995ef015dfc637..f187b6e030310e635bbdd2d6bc29080001d80ff3 100644 --- a/frameworks/libs/distributeddb/test/unittest/common/storage/virtual_sqlite_relational_store.cpp +++ b/frameworks/libs/distributeddb/test/unittest/common/storage/virtual_sqlite_relational_store.cpp @@ -28,7 +28,7 @@ int VirtualSqliteRelationalStore::CallCheckTrackerTable(const TrackerSchema &tra void VirtualSqliteRelationalStore::CallCleanDirtyLogIfNeed(const std::string &tableName) const { - CleanDirtyLogIfNeed(tableName, nullptr); + CleanDirtyLogIfNeed(tableName); } RelationalSchemaObject VirtualSqliteRelationalStore::CallGetSchemaObj() const diff --git a/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_upgrade_test.cpp b/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_upgrade_test.cpp index 5c8c5b103897d2b2e145716df43e1200ad651ac9..09e9317b3f1a0ebd7968ff5b705ec5ac48b2f438 100644 --- a/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_upgrade_test.cpp +++ b/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_upgrade_test.cpp @@ -147,15 +147,6 @@ HWTEST_F(DistributedDBRDBUpgradeTest, UpgradeTracker002, TestSize.Level0) EXPECT_EQ(SetTrackerTables(info1, {DEVICE_SYNC_TABLE}), E_OK); EXPECT_EQ(CountTableData(info1, DBCommon::GetLogTableName(DEVICE_SYNC_TABLE), " json_valid(extend_field) = 0 OR json_extract(extend_field, '$') = '{}'"), 0); - /** - * @tc.steps: step4. Set tracker again and check log. - * @tc.expected: step4. Due to not being processed within 24 hours since last time. - */ - sql = "UPDATE " + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + " SET extend_field=''"; - EXPECT_EQ(ExecuteSQL(sql, info1), E_OK); - EXPECT_EQ(SetTrackerTables(info1, {DEVICE_SYNC_TABLE}), E_OK); - EXPECT_EQ(CountTableData(info1, DBCommon::GetLogTableName(DEVICE_SYNC_TABLE), - " json_valid(extend_field) = 0 OR json_extract(extend_field, '$') = '{}'"), chkCnt); } /** @@ -190,4 +181,49 @@ HWTEST_F(DistributedDBRDBUpgradeTest, UpgradeTracker003, TestSize.Level0) EXPECT_EQ(CountTableData(info1, DBCommon::GetLogTableName(DEVICE_SYNC_TABLE), " json_extract(extend_field, '$') = '{}'"), chkCnt); } + +/** + * @tc.name: UpgradeTracker004 + * @tc.desc: Test concurrent upgrade + * @tc.type: FUNC + * @tc.author: bty + */ +HWTEST_F(DistributedDBRDBUpgradeTest, UpgradeTracker004, TestSize.Level2) +{ + /** + * @tc.steps: step1. Init delegate and set tracker schema. + * @tc.expected: step1. Ok + */ + ASSERT_NO_FATAL_FAILURE(InitUpgradeDelegate()); + auto info1 = GetStoreInfo1(); + EXPECT_EQ(SetTrackerTables(info1, {DEVICE_SYNC_TABLE}), E_OK); + /** + * @tc.steps: step2. Insert local data and log update extend_field to empty str. + * @tc.expected: step2. Ok + */ + int chkCnt = 10; + InsertLocalDBData(0, chkCnt, info1); + /** + * @tc.steps: step3. concurrent upgrade + * @tc.expected: step3. Ok + */ + size_t loopTimes = 200; + std::thread t1([this, &info1, loopTimes]() { + for (size_t i = 0; i < loopTimes; i++) { + std::string sql = "UPDATE " + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + " SET extend_field=''"; + EXPECT_EQ(ExecuteSQL(sql, info1), E_OK); + EXPECT_EQ(CreateDistributedTable(info1, DEVICE_SYNC_TABLE), E_OK); + } + }); + std::thread t2([this, &info1, loopTimes]() { + for (size_t i = 0; i < loopTimes; i++) { + EXPECT_EQ(SetTrackerTables(info1, {DEVICE_SYNC_TABLE}), E_OK); + } + }); + for (size_t i = 0; i < loopTimes; i++) { + EXPECT_EQ(SetTrackerTables(info1, {DEVICE_SYNC_TABLE}), E_OK); + } + t1.join(); + t2.join(); +} } \ No newline at end of file