From 36d841b821660687733ce13a2750e93bbb853aa6 Mon Sep 17 00:00:00 2001 From: yaohaolin Date: Fri, 7 Nov 2025 11:25:49 +0800 Subject: [PATCH] fix doc error --- README.md | 10 +- README_CN.md | 12 +- docs/.codecheck/check.yml | 9 + docs/README_CN.md | 2 +- docs/_ext/customdocumenter.txt | 245 ------ docs/_ext/myautosummary.py | 500 ------------ docs/_ext/overwriteautosummary_generate.txt | 716 ------------------ docs/_ext/overwriteobjectiondirective.txt | 440 ----------- docs/_ext/overwriteviewcode.txt | 382 ---------- docs/_ext/rename_include.py | 60 -- docs/_static/custom.css | 51 +- docs/_templates/autosummary/class.rst | 44 ++ docs/requirements.txt | 17 +- docs/source_zh_cn/api/api_cpp/common.md | 356 --------- docs/source_zh_cn/api/api_cpp/dsclient.md | 85 --- docs/source_zh_cn/api/api_cpp/hetero_cache.md | 402 ---------- docs/source_zh_cn/api/api_cpp/kv_cache.md | 391 ---------- docs/source_zh_cn/api/api_cpp/object_cache.md | 479 ------------ docs/source_zh_cn/appendix/index.md | 11 + docs/source_zh_cn/conf.py | 238 +++--- .../contributor_guide/contribution.md | 20 + .../{getting-started => deployment}/deploy.md | 37 +- .../{appendix => deployment}/dscli.md | 26 +- docs/source_zh_cn/deployment/index.md | 15 + .../k8s_configuration.md | 4 +- .../development-guide/api/cpp/Buffer.rst | 97 +++ .../development-guide/api/cpp/KVClient.rst | 247 ++++++ .../development-guide/api/cpp/Optional.rst | 60 ++ .../api/cpp/SensitiveValue.rst | 85 +++ .../development-guide/api/cpp/Status.rst | 123 +++ .../development-guide/api/cpp/StringView.rst | 71 ++ .../api/cpp/enum-CacheType.rst | 18 + .../api/cpp/enum-ConsistencyType.rst | 18 + .../api/cpp/enum-StatusCode.rst | 27 + .../api/cpp/enum-WriteMode.rst | 18 + .../development-guide/api/cpp/index.rst | 56 ++ .../api/cpp/struct-ConnectOptions.rst | 62 ++ .../api/cpp/struct-SetParam.rst | 23 + .../development-guide/api/index.md | 16 + .../python}/datasystem.DsClient.hetero.rst | 6 +- .../api/python}/datasystem.DsClient.init.rst | 2 +- .../api/python}/datasystem.DsClient.kv.rst | 6 +- .../python}/datasystem.DsClient.object.rst | 6 +- .../api/python}/datasystem.DsClient.rst | 21 +- ...system.DsTensorClient.async_dev_delete.rst | 0 ...tasystem.DsTensorClient.async_mget_h2d.rst | 0 ...tasystem.DsTensorClient.async_mset_d2h.rst | 0 .../datasystem.DsTensorClient.delete.rst | 0 .../datasystem.DsTensorClient.dev_delete.rst | 0 ...system.DsTensorClient.dev_local_delete.rst | 0 .../datasystem.DsTensorClient.dev_mget.rst | 0 .../datasystem.DsTensorClient.dev_mset.rst | 0 .../datasystem.DsTensorClient.dev_recv.rst | 0 .../datasystem.DsTensorClient.dev_send.rst | 0 ...nsorClient.get_page_attn_layerwise_d2d.rst | 2 +- .../datasystem.DsTensorClient.init.rst | 0 .../datasystem.DsTensorClient.mget_h2d.rst | 0 ...sorClient.mget_page_attn_blockwise_h2d.rst | 2 +- .../datasystem.DsTensorClient.mset_d2h.rst | 0 ...sorClient.mset_page_attn_blockwise_d2h.rst | 2 +- ...nsorClient.put_page_attn_layerwise_d2d.rst | 2 +- .../api/python}/datasystem.DsTensorClient.rst | 48 +- .../python}/datasystem.hetero_client.Blob.rst | 17 +- ...asystem.hetero_client.Blob.set_dev_ptr.rst | 0 ...datasystem.hetero_client.Blob.set_size.rst | 0 ...tero_client.DeviceBlobList.append_blob.rst | 0 ...ro_client.DeviceBlobList.get_blob_list.rst | 0 ...atasystem.hetero_client.DeviceBlobList.rst | 19 +- ...tero_client.DeviceBlobList.set_dev_idx.rst | 0 .../datasystem.hetero_client.Future.get.rst | 0 .../datasystem.hetero_client.Future.rst | 15 +- ...o_client.HeteroClient.async_dev_delete.rst | 17 + ...ero_client.HeteroClient.async_mget_h2d.rst | 2 +- ...ero_client.HeteroClient.async_mset_d2h.rst | 2 +- ...stem.hetero_client.HeteroClient.delete.rst | 0 ....hetero_client.HeteroClient.dev_delete.rst | 0 ...o_client.HeteroClient.dev_local_delete.rst | 0 ...em.hetero_client.HeteroClient.dev_mget.rst | 0 ...em.hetero_client.HeteroClient.dev_mset.rst | 0 ...hetero_client.HeteroClient.dev_publish.rst | 0 ...tero_client.HeteroClient.dev_subscribe.rst | 0 ...etero_client.HeteroClient.generate_key.rst | 0 ...tero_client.HeteroClient.get_meta_info.rst | 2 +- ...system.hetero_client.HeteroClient.init.rst | 0 ...em.hetero_client.HeteroClient.mget_h2d.rst | 0 ...em.hetero_client.HeteroClient.mset_d2h.rst | 0 .../datasystem.hetero_client.HeteroClient.rst | 44 +- .../datasystem.hetero_client.MetaInfo.rst | 6 +- .../datasystem.kv_client.KVClient.delete.rst | 0 .../datasystem.kv_client.KVClient.exist.rst | 2 +- .../datasystem.kv_client.KVClient.expire.rst | 2 +- ...system.kv_client.KVClient.generate_key.rst | 0 .../datasystem.kv_client.KVClient.get.rst | 0 ..._client.KVClient.get_read_only_buffers.rst | 0 .../datasystem.kv_client.KVClient.init.rst | 0 .../datasystem.kv_client.KVClient.mset.rst | 0 .../datasystem.kv_client.KVClient.msettx.rst | 0 .../datasystem.kv_client.KVClient.read.rst | 0 .../python}/datasystem.kv_client.KVClient.rst | 50 +- .../datasystem.kv_client.KVClient.set.rst | 0 ...atasystem.kv_client.KVClient.set_value.rst | 0 ...v_client.ReadOnlyBuffer.immutable_data.rst | 0 ...system.kv_client.ReadOnlyBuffer.rlatch.rst | 0 .../datasystem.kv_client.ReadOnlyBuffer.rst | 19 +- ...stem.kv_client.ReadOnlyBuffer.unrlatch.rst | 0 .../datasystem.kv_client.ReadParam.rst | 0 .../python}/datasystem.kv_client.SetParam.rst | 0 ...tasystem.object_client.Buffer.get_size.rst | 0 ...em.object_client.Buffer.immutable_data.rst | 0 ...object_client.Buffer.invalidate_buffer.rst | 0 ...ystem.object_client.Buffer.memory_copy.rst | 0 ...stem.object_client.Buffer.mutable_data.rst | 0 ...atasystem.object_client.Buffer.publish.rst | 0 ...datasystem.object_client.Buffer.rlatch.rst | 2 +- .../datasystem.object_client.Buffer.rst | 35 +- .../datasystem.object_client.Buffer.seal.rst | 0 ...tasystem.object_client.Buffer.unrlatch.rst | 2 +- ...tasystem.object_client.Buffer.unwlatch.rst | 2 +- ...datasystem.object_client.Buffer.wlatch.rst | 2 +- .../datasystem.object_client.CacheType.rst | 15 + ...tasystem.object_client.ConsistencyType.rst | 0 ...stem.object_client.ObjectClient.create.rst | 0 ...ect_client.ObjectClient.g_decrease_ref.rst | 0 ...ect_client.ObjectClient.g_increase_ref.rst | 0 ...lient.ObjectClient.generate_object_key.rst | 0 ...asystem.object_client.ObjectClient.get.rst | 0 ...system.object_client.ObjectClient.init.rst | 0 ...asystem.object_client.ObjectClient.put.rst | 2 +- ...ient.ObjectClient.query_global_ref_num.rst | 0 .../datasystem.object_client.ObjectClient.rst | 27 +- .../datasystem.object_client.WriteMode.rst | 0 .../development-guide/api/python/index.rst | 181 +++++ .../development-guide/{ => example}/hetero.md | 0 .../development-guide/example/index.md | 16 + .../development-guide/{ => example}/kv.md | 0 .../development-guide/{ => example}/object.md | 0 docs/source_zh_cn/development-guide/index.md | 20 + .../getting-started/getting_started.md | 358 +++++++++ docs/source_zh_cn/getting-started/overview.md | 78 -- .../image => images}/deployment.png | Bin .../image => images}/introduction.png | Bin .../image => images}/logical_architecture.png | Bin .../image => images}/logo-large.png | Bin docs/source_zh_cn/images/logo-small.png | Bin 0 -> 4466 bytes docs/source_zh_cn/index.rst | 130 ++-- .../installation_linux.md} | 6 +- scripts/build_thirdparty.sh | 66 +- 147 files changed, 2190 insertions(+), 4519 deletions(-) create mode 100644 docs/.codecheck/check.yml delete mode 100644 docs/_ext/customdocumenter.txt delete mode 100644 docs/_ext/myautosummary.py delete mode 100644 docs/_ext/overwriteautosummary_generate.txt delete mode 100644 docs/_ext/overwriteobjectiondirective.txt delete mode 100644 docs/_ext/overwriteviewcode.txt delete mode 100644 docs/_ext/rename_include.py create mode 100644 docs/_templates/autosummary/class.rst delete mode 100644 docs/source_zh_cn/api/api_cpp/common.md delete mode 100644 docs/source_zh_cn/api/api_cpp/dsclient.md delete mode 100644 docs/source_zh_cn/api/api_cpp/hetero_cache.md delete mode 100644 docs/source_zh_cn/api/api_cpp/kv_cache.md delete mode 100644 docs/source_zh_cn/api/api_cpp/object_cache.md create mode 100644 docs/source_zh_cn/appendix/index.md create mode 100644 docs/source_zh_cn/contributor_guide/contribution.md rename docs/source_zh_cn/{getting-started => deployment}/deploy.md (92%) rename docs/source_zh_cn/{appendix => deployment}/dscli.md (96%) create mode 100644 docs/source_zh_cn/deployment/index.md rename docs/source_zh_cn/{appendix => deployment}/k8s_configuration.md (99%) create mode 100644 docs/source_zh_cn/development-guide/api/cpp/Buffer.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/KVClient.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/Optional.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/SensitiveValue.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/Status.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/StringView.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/enum-CacheType.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/enum-ConsistencyType.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/enum-StatusCode.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/enum-WriteMode.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/index.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/struct-ConnectOptions.rst create mode 100644 docs/source_zh_cn/development-guide/api/cpp/struct-SetParam.rst create mode 100644 docs/source_zh_cn/development-guide/api/index.md rename docs/source_zh_cn/{api/api_python/ds_client/DsClient => development-guide/api/python}/datasystem.DsClient.hetero.rst (33%) rename docs/source_zh_cn/{api/api_python/ds_client/DsClient => development-guide/api/python}/datasystem.DsClient.init.rst (88%) rename docs/source_zh_cn/{api/api_python/ds_client/DsClient => development-guide/api/python}/datasystem.DsClient.kv.rst (36%) rename docs/source_zh_cn/{api/api_python/ds_client/DsClient => development-guide/api/python}/datasystem.DsClient.object.rst (34%) rename docs/source_zh_cn/{api/api_python/ds_client => development-guide/api/python}/datasystem.DsClient.rst (73%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.async_dev_delete.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.async_mget_h2d.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.async_mset_d2h.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.delete.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_delete.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_local_delete.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_mget.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_mset.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_recv.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.dev_send.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst (96%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.init.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.mget_h2d.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst (96%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.mset_d2h.rst (100%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst (95%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client/DsTensorClient => development-guide/api/python}/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst (96%) rename docs/source_zh_cn/{api/api_python/ds_tensor_client => development-guide/api/python}/datasystem.DsTensorClient.rst (36%) rename docs/source_zh_cn/{api/api_python/hetero_client => development-guide/api/python}/datasystem.hetero_client.Blob.rst (54%) rename docs/source_zh_cn/{api/api_python/hetero_client/Blob => development-guide/api/python}/datasystem.hetero_client.Blob.set_dev_ptr.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/Blob => development-guide/api/python}/datasystem.hetero_client.Blob.set_size.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/DeviceBlobList => development-guide/api/python}/datasystem.hetero_client.DeviceBlobList.append_blob.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/DeviceBlobList => development-guide/api/python}/datasystem.hetero_client.DeviceBlobList.get_blob_list.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client => development-guide/api/python}/datasystem.hetero_client.DeviceBlobList.rst (51%) rename docs/source_zh_cn/{api/api_python/hetero_client/DeviceBlobList => development-guide/api/python}/datasystem.hetero_client.DeviceBlobList.set_dev_idx.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/Future => development-guide/api/python}/datasystem.hetero_client.Future.get.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client => development-guide/api/python}/datasystem.hetero_client.Future.rst (49%) create mode 100644 docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_dev_delete.rst rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst (95%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst (95%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.delete.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_delete.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_local_delete.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_mget.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_mset.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_publish.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.dev_subscribe.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.generate_key.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.get_meta_info.rst (93%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.init.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.mget_h2d.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client/HeteroClient => development-guide/api/python}/datasystem.hetero_client.HeteroClient.mset_d2h.rst (100%) rename docs/source_zh_cn/{api/api_python/hetero_client => development-guide/api/python}/datasystem.hetero_client.HeteroClient.rst (45%) rename docs/source_zh_cn/{api/api_python/hetero_client => development-guide/api/python}/datasystem.hetero_client.MetaInfo.rst (73%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.delete.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.exist.rst (92%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.expire.rst (94%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.generate_key.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.get.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.get_read_only_buffers.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.init.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.mset.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.msettx.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.read.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client => development-guide/api/python}/datasystem.kv_client.KVClient.rst (52%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.set.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/KVClient => development-guide/api/python}/datasystem.kv_client.KVClient.set_value.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/ReadOnlyBuffer => development-guide/api/python}/datasystem.kv_client.ReadOnlyBuffer.immutable_data.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client/ReadOnlyBuffer => development-guide/api/python}/datasystem.kv_client.ReadOnlyBuffer.rlatch.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client => development-guide/api/python}/datasystem.kv_client.ReadOnlyBuffer.rst (40%) rename docs/source_zh_cn/{api/api_python/kv_client/ReadOnlyBuffer => development-guide/api/python}/datasystem.kv_client.ReadOnlyBuffer.unrlatch.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client => development-guide/api/python}/datasystem.kv_client.ReadParam.rst (100%) rename docs/source_zh_cn/{api/api_python/kv_client => development-guide/api/python}/datasystem.kv_client.SetParam.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.get_size.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.immutable_data.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.invalidate_buffer.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.memory_copy.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.mutable_data.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.publish.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.rlatch.rst (91%) rename docs/source_zh_cn/{api/api_python/object_client => development-guide/api/python}/datasystem.object_client.Buffer.rst (30%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.seal.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.unrlatch.rst (84%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.unwlatch.rst (84%) rename docs/source_zh_cn/{api/api_python/object_client/Buffer => development-guide/api/python}/datasystem.object_client.Buffer.wlatch.rst (91%) create mode 100644 docs/source_zh_cn/development-guide/api/python/datasystem.object_client.CacheType.rst rename docs/source_zh_cn/{api/api_python/object_client => development-guide/api/python}/datasystem.object_client.ConsistencyType.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.create.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.g_decrease_ref.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.g_increase_ref.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.generate_object_key.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.get.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.init.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.put.rst (96%) rename docs/source_zh_cn/{api/api_python/object_client/ObjectClient => development-guide/api/python}/datasystem.object_client.ObjectClient.query_global_ref_num.rst (100%) rename docs/source_zh_cn/{api/api_python/object_client => development-guide/api/python}/datasystem.object_client.ObjectClient.rst (57%) rename docs/source_zh_cn/{api/api_python/object_client => development-guide/api/python}/datasystem.object_client.WriteMode.rst (100%) create mode 100644 docs/source_zh_cn/development-guide/api/python/index.rst rename docs/source_zh_cn/development-guide/{ => example}/hetero.md (100%) create mode 100644 docs/source_zh_cn/development-guide/example/index.md rename docs/source_zh_cn/development-guide/{ => example}/kv.md (100%) rename docs/source_zh_cn/development-guide/{ => example}/object.md (100%) create mode 100644 docs/source_zh_cn/development-guide/index.md create mode 100644 docs/source_zh_cn/getting-started/getting_started.md delete mode 100644 docs/source_zh_cn/getting-started/overview.md rename docs/source_zh_cn/{getting-started/image => images}/deployment.png (100%) rename docs/source_zh_cn/{getting-started/image => images}/introduction.png (100%) rename docs/source_zh_cn/{getting-started/image => images}/logical_architecture.png (100%) rename docs/source_zh_cn/{getting-started/image => images}/logo-large.png (100%) create mode 100644 docs/source_zh_cn/images/logo-small.png rename docs/source_zh_cn/{getting-started/install.md => installation/installation_linux.md} (98%) diff --git a/README.md b/README.md index 76482bb..a524c28 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -![](./docs/source_zh_cn/getting-started/image/logo-large.png) +![](./docs/source_zh_cn/images/logo-large.png) openYuanrong 是一个 Serverless 分布式计算引擎,致力于以一套统一 Serverless 架构支持 AI、大数据、微服务等各类分布式应用。它提供多语言函数编程接口,以单机编程体验简化分布式应用开发;提供分布式动态调度和数据共享等能力,实现分布式应用的高性能运行和集群的高效资源利用。 ## 简介 -![](./docs/source_zh_cn/getting-started/image/introduction.png) +![](./docs/source_zh_cn/images/introduction.png) openYuanrong 由多语言函数运行时、函数系统和数据系统组成,支持按需灵活单独或组合使用。 @@ -37,7 +37,7 @@ openYuanrong datasystem 的主要特性包括: ### openYuanrong datasystem 架构 -![](./docs/source_zh_cn/getting-started/image/logical_architecture.png) +![](./docs/source_zh_cn/images/logical_architecture.png) openYuanrong datasystem 由三个部分组成: @@ -49,7 +49,7 @@ openYuanrong datasystem 由三个部分组成: - **集群管理**:依赖 ETCD,实现节点发现/健康检测,支持故障恢复及在线扩缩容。 -![](./docs/source_zh_cn/getting-started/image/deployment.png) +![](./docs/source_zh_cn/images/deployment.png) openYuanrong datasystem 的部署视图如上图所示: @@ -81,7 +81,7 @@ openYuanrong datasystem 的部署视图如上图所示: #### 源码编译方式安装 -使用源码编译方式安装 openYuanrong datasystem 可以参考文档:[源码编译安装 openYuanrong datasystem](./docs/source_zh_cn/getting-started/install.md#源码编译安装) +使用源码编译方式安装 openYuanrong datasystem 可以参考文档:[源码编译安装 openYuanrong datasystem](./docs/source_zh_cn/installation/installation_linux.md#源码编译安装) ### 部署 openYuanrong datasystem diff --git a/README_CN.md b/README_CN.md index 76482bb..9d01ca7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,10 +1,10 @@ -![](./docs/source_zh_cn/getting-started/image/logo-large.png) +![](./docs/source_zh_cn/images/logo-large.png) openYuanrong 是一个 Serverless 分布式计算引擎,致力于以一套统一 Serverless 架构支持 AI、大数据、微服务等各类分布式应用。它提供多语言函数编程接口,以单机编程体验简化分布式应用开发;提供分布式动态调度和数据共享等能力,实现分布式应用的高性能运行和集群的高效资源利用。 ## 简介 -![](./docs/source_zh_cn/getting-started/image/introduction.png) +![](./docs/source_zh_cn/images/introduction.png) openYuanrong 由多语言函数运行时、函数系统和数据系统组成,支持按需灵活单独或组合使用。 @@ -37,7 +37,7 @@ openYuanrong datasystem 的主要特性包括: ### openYuanrong datasystem 架构 -![](./docs/source_zh_cn/getting-started/image/logical_architecture.png) +![](./docs/source_zh_cn/images/logical_architecture.png) openYuanrong datasystem 由三个部分组成: @@ -49,7 +49,7 @@ openYuanrong datasystem 由三个部分组成: - **集群管理**:依赖 ETCD,实现节点发现/健康检测,支持故障恢复及在线扩缩容。 -![](./docs/source_zh_cn/getting-started/image/deployment.png) +![](./docs/source_zh_cn/images/deployment.png) openYuanrong datasystem 的部署视图如上图所示: @@ -81,7 +81,7 @@ openYuanrong datasystem 的部署视图如上图所示: #### 源码编译方式安装 -使用源码编译方式安装 openYuanrong datasystem 可以参考文档:[源码编译安装 openYuanrong datasystem](./docs/source_zh_cn/getting-started/install.md#源码编译安装) +使用源码编译方式安装 openYuanrong datasystem 可以参考文档:[源码编译安装 openYuanrong datasystem](./docs/source_zh_cn/installation/installation_linux.md#源码编译安装) ### 部署 openYuanrong datasystem @@ -244,7 +244,7 @@ openYuanrong datasystem 还提供了基于 Kubernetes 容器化部署方式, ## 文档 -有关 openYuanrong datasystem 安装指南、教程和 API 的更多详细信息,请参阅 [用户文档](docs)。 +有关 openYuanrong datasystem 安装指南、教程和 API 的更多详细信息,请参阅 [用户文档](https://pages.openeuler.openatom.cn/openyuanrong-datasystem/docs/zh-cn/latest/index.html)。 有关 openYuanrong 更多详细信息请参阅 [openYuanrong 文档](https://pages.openeuler.openatom.cn/openyuanrong/docs/zh-cn/latest/index.html),了解如何使用 openYuanrong 开发分布式应用。 diff --git a/docs/.codecheck/check.yml b/docs/.codecheck/check.yml new file mode 100644 index 0000000..d88ba85 --- /dev/null +++ b/docs/.codecheck/check.yml @@ -0,0 +1,9 @@ +version: 2.0 + +steps: + pre_codecheck: + - checkout + +tool_params: + secsolar: + source_dir: ./ \ No newline at end of file diff --git a/docs/README_CN.md b/docs/README_CN.md index 8d95ef9..019569e 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -19,7 +19,7 @@ docs openYuanrong datasystem的教程和API文档均可由[Sphinx](https://www.sphinx-doc.org/en/master/)工具生成,操作前需完成openYuanrong datasystem的安装。 -1. 使用pip安装openYuanrong datasystem模块,API文档需要根据安装后的openYuanrong datasystem模块生成,参考[安装](source_zh_cn/getting-started/install.md)。 +1. 使用pip安装openYuanrong datasystem模块,API文档需要根据安装后的openYuanrong datasystem模块生成,参考[安装](source_zh_cn/installation/installation_linux.md)。 ```bash pip install openyuanrong_datasystem-*.whl diff --git a/docs/_ext/customdocumenter.txt b/docs/_ext/customdocumenter.txt deleted file mode 100644 index 2d37ae4..0000000 --- a/docs/_ext/customdocumenter.txt +++ /dev/null @@ -1,245 +0,0 @@ -import re -import os -from sphinx.ext.autodoc import Documenter - - -class CustomDocumenter(Documenter): - - def document_members(self, all_members: bool = False) -> None: - """Generate reST for member documentation. - - If *all_members* is True, do all members, else those given by - *self.options.members*. - """ - # set current namespace for finding members - self.env.temp_data['autodoc:module'] = self.modname - if self.objpath: - self.env.temp_data['autodoc:class'] = self.objpath[0] - - want_all = all_members or self.options.inherited_members or \ - self.options.members is ALL - # find out which members are documentable - members_check_module, members = self.get_object_members(want_all) - - # **** 排除已写中文接口名 **** - file_path = os.path.join(self.env.app.srcdir, self.env.docname+'.rst') - exclude_re = re.compile(r'(.. py:class::|.. py:function::)\s+(.*?)(\(|\n)') - includerst_re = re.compile(r'.. include::\s+(.*?)\n') - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - excluded_members = exclude_re.findall(content) - if excluded_members: - excluded_members = [i[1].split('.')[-1] for i in excluded_members] - rst_included = includerst_re.findall(content) - if rst_included: - for i in rst_included: - include_path = os.path.join(os.path.dirname(file_path), i) - if os.path.exists(include_path): - with open(include_path, 'r', encoding='utf8') as g: - content_ = g.read() - excluded_member_ = exclude_re.findall(content_) - if excluded_member_: - excluded_member_ = [j[1].split('.')[-1] for j in excluded_member_] - excluded_members.extend(excluded_member_) - - if excluded_members: - if self.options.exclude_members: - self.options.exclude_members |= set(excluded_members) - else: - self.options.exclude_members = excluded_members - - # remove members given by exclude-members - if self.options.exclude_members: - members = [ - (membername, member) for (membername, member) in members - if ( - self.options.exclude_members is ALL or - membername not in self.options.exclude_members - ) - ] - - # document non-skipped members - memberdocumenters = [] # type: List[Tuple[Documenter, bool]] - for (mname, member, isattr) in self.filter_members(members, want_all): - classes = [cls for cls in self.documenters.values() - if cls.can_document_member(member, mname, isattr, self)] - if not classes: - # don't know how to document this member - continue - # prefer the documenter with the highest priority - classes.sort(key=lambda cls: cls.priority) - # give explicitly separated module name, so that members - # of inner classes can be documented - full_mname = self.modname + '::' + \ - '.'.join(self.objpath + [mname]) - documenter = classes[-1](self.directive, full_mname, self.indent) - memberdocumenters.append((documenter, isattr)) - member_order = self.options.member_order or \ - self.env.config.autodoc_member_order - if member_order == 'groupwise': - # sort by group; relies on stable sort to keep items in the - # same group sorted alphabetically - memberdocumenters.sort(key=lambda e: e[0].member_order) - elif member_order == 'bysource' and self.analyzer: - # sort by source order, by virtue of the module analyzer - tagorder = self.analyzer.tagorder - - def keyfunc(entry: Tuple[Documenter, bool]) -> int: - fullname = entry[0].name.split('::')[1] - return tagorder.get(fullname, len(tagorder)) - memberdocumenters.sort(key=keyfunc) - - for documenter, isattr in memberdocumenters: - documenter.generate( - all_members=True, real_modname=self.real_modname, - check_module=members_check_module and not isattr) - - # reset current objects - self.env.temp_data['autodoc:module'] = None - self.env.temp_data['autodoc:class'] = None - - def generate(self, more_content: Any = None, real_modname: str = None, - check_module: bool = False, all_members: bool = False) -> None: - """Generate reST for the object given by *self.name*, and possibly for - its members. - - If *more_content* is given, include that content. If *real_modname* is - given, use that module name to find attribute docs. If *check_module* is - True, only generate if the object is defined in the module name it is - imported from. If *all_members* is True, document all members. - """ - if not self.parse_name(): - # need a module to import - logger.warning( - __('don\'t know which module to import for autodocumenting ' - '%r (try placing a "module" or "currentmodule" directive ' - 'in the document, or giving an explicit module name)') % - self.name, type='autodoc') - return - - # now, import the module and get object to document - if not self.import_object(): - return - - # If there is no real module defined, figure out which to use. - # The real module is used in the module analyzer to look up the module - # where the attribute documentation would actually be found in. - # This is used for situations where you have a module that collects the - # functions and classes of internal submodules. - self.real_modname = real_modname or self.get_real_modname() # type: str - - # try to also get a source code analyzer for attribute docs - try: - self.analyzer = ModuleAnalyzer.for_module(self.real_modname) - # parse right now, to get PycodeErrors on parsing (results will - # be cached anyway) - self.analyzer.find_attr_docs() - except PycodeError as err: - logger.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules - self.analyzer = None - # at least add the module.__file__ as a dependency - if hasattr(self.module, '__file__') and self.module.__file__: - self.directive.filename_set.add(self.module.__file__) - else: - self.directive.filename_set.add(self.analyzer.srcname) - - # check __module__ of object (for members not given explicitly) - if check_module: - if not self.check_module(): - return - - # document members, if possible - self.document_members(all_members) - - -class ModuleDocumenter(CustomDocumenter): - """ - Specialized Documenter subclass for modules. - """ - objtype = 'module' - content_indent = '' - titles_allowed = True - - option_spec = { - 'members': members_option, 'undoc-members': bool_option, - 'noindex': bool_option, 'inherited-members': bool_option, - 'show-inheritance': bool_option, 'synopsis': identity, - 'platform': identity, 'deprecated': bool_option, - 'member-order': identity, 'exclude-members': members_set_option, - 'private-members': bool_option, 'special-members': members_option, - 'imported-members': bool_option, 'ignore-module-all': bool_option - } # type: Dict[str, Callable] - - def __init__(self, *args: Any) -> None: - super().__init__(*args) - merge_members_option(self.options) - - @classmethod - def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any - ) -> bool: - # don't document submodules automatically - return False - - def resolve_name(self, modname: str, parents: Any, path: str, base: Any - ) -> Tuple[str, List[str]]: - if modname is not None: - logger.warning(__('"::" in automodule name doesn\'t make sense'), - type='autodoc') - return (path or '') + base, [] - - def parse_name(self) -> bool: - ret = super().parse_name() - if self.args or self.retann: - logger.warning(__('signature arguments or return annotation ' - 'given for automodule %s') % self.fullname, - type='autodoc') - return ret - - def add_directive_header(self, sig: str) -> None: - Documenter.add_directive_header(self, sig) - - sourcename = self.get_sourcename() - - # add some module-specific options - if self.options.synopsis: - self.add_line(' :synopsis: ' + self.options.synopsis, sourcename) - if self.options.platform: - self.add_line(' :platform: ' + self.options.platform, sourcename) - if self.options.deprecated: - self.add_line(' :deprecated:', sourcename) - - def get_object_members(self, want_all: bool) -> Tuple[bool, List[Tuple[str, object]]]: - if want_all: - if (self.options.ignore_module_all or not - hasattr(self.object, '__all__')): - # for implicit module members, check __module__ to avoid - # documenting imported objects - return True, get_module_members(self.object) - else: - memberlist = self.object.__all__ - # Sometimes __all__ is broken... - if not isinstance(memberlist, (list, tuple)) or not \ - all(isinstance(entry, str) for entry in memberlist): - logger.warning( - __('__all__ should be a list of strings, not %r ' - '(in module %s) -- ignoring __all__') % - (memberlist, self.fullname), - type='autodoc' - ) - # fall back to all members - return True, get_module_members(self.object) - else: - memberlist = self.options.members or [] - ret = [] - for mname in memberlist: - try: - ret.append((mname, safe_getattr(self.object, mname))) - except AttributeError: - logger.warning( - __('missing attribute mentioned in :members: or __all__: ' - 'module %s, attribute %s') % - (safe_getattr(self.object, '__name__', '???'), mname), - type='autodoc' - ) - return False, ret diff --git a/docs/_ext/myautosummary.py b/docs/_ext/myautosummary.py deleted file mode 100644 index c023efe..0000000 --- a/docs/_ext/myautosummary.py +++ /dev/null @@ -1,500 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Customized autosummary directives for sphinx. -""" - -import os -import re -from typing import List, Tuple - -from docutils.nodes import Node -from sphinx.ext.autodoc.directive import DocumenterBridge, Options -from sphinx.ext.autosummary import (Autosummary, Matcher, ModuleAnalyzer, - ModuleType, PycodeError, StringList, - addnodes, autosummary_table, - autosummary_toc, extract_summary, - get_documenter, - get_import_prefixes_from_env, - import_by_name, logger, mangle_signature, - mock, nodes, posixpath, rst, - switch_source_input) -from sphinx.locale import __ - - -MAX_ITEM_CHARS = 50 - - -class DsAutosummary(Autosummary): - """ - Inherited from sphinx's autosummary, add titles and a column for the generated table. - """ - - def init(self): - """ - init method - """ - self.find_doc_name = "" - self.third_title = "" - self.default_doc = "" - - def extract_env_summary(self, doc: List[str]) -> str: - """Extract env summary from docstring.""" - env_sum = self.default_doc - for i, piece in enumerate(doc): - if piece.startswith(self.find_doc_name): - env_sum = doc[i+1][4:] - return env_sum - - def run(self): - """ - run method - """ - self.init() - self.bridge = DocumenterBridge(self.env, self.state.document.reporter, - Options(), self.lineno, self.state) - - names = [x.strip().split()[0] for x in self.content if x.strip() and re.search(r'^[~a-zA-Z_]', x.strip()[0])] - items = self.get_items(names) - teble_nodes = self.get_table(items) - - if 'toctree' not in self.options: - return teble_nodes - - dirname = posixpath.dirname(self.env.docname) - - tree_prefix = self.options['toctree'].strip() - docnames = [] - excluded = Matcher(self.config.exclude_patterns) - for item in items: - docname = posixpath.join(tree_prefix, item[3]) - docname = posixpath.normpath(posixpath.join(dirname, docname)) - if docname not in self.env.found_docs: - location = self.state_machine.get_source_and_line(self.lineno) - if excluded(self.env.doc2path(docname, None)): - msg = __('autosummary references excluded document %r. Ignored.') - else: - msg = __('autosummary: stub file not found %r. ' - 'Check your autosummary_generate setting.') - logger.warning(msg, item[3], location=location) - continue - docnames.append(docname) - - if docnames: - tocnode = addnodes.toctree() - tocnode['includefiles'] = docnames - tocnode['entries'] = [(None, docn) for docn in docnames] - tocnode['maxdepth'] = -1 - tocnode['glob'] = None - teble_nodes.append(autosummary_toc('', '', tocnode)) - return teble_nodes - - def get_items(self, names: List[str]) -> List[Tuple[str, str, str, str, str]]: - """Try to import the given names, and return a list of - ``[(name, signature, summary_string, real_name, env_summary), ...]``. - """ - prefixes = get_import_prefixes_from_env(self.env) - items = [] # type: List[Tuple[str, str, str, str, str]] - - for name in names: - display_name = name - if name.startswith('~'): - name = name[1:] - display_name = name.split('.')[-1] - try: - with mock(self.config.autosummary_mock_imports): - real_name, obj, parent, modname = import_by_name(name, prefixes=prefixes) - except ImportError: - logger.warning(__('failed to import %s'), name) - items.append((name, '', '', name, '')) - continue - - self.bridge.result = StringList() # initialize for each documenter - # give explicitly separated module name, so that members - # of inner classes can be documented - full_name = f"{modname}::{full_name[len(modname) + 1:]}" if not isinstance(obj, ModuleType) else real_name - # NB. using full_name here is important, since Documenters - # handle module prefixes slightly differently - doccls = get_documenter(self.env.app, obj, parent) - documenter = doccls(self.bridge, full_name) - - if not documenter.parse_name(): - logger.warning(__('failed to parse name %s'), real_name) - items.append((display_name, '', '', real_name, '')) - continue - if not documenter.import_object(): - logger.warning(__('failed to import object %s'), real_name) - items.append((display_name, '', '', real_name, '')) - continue - if documenter.options.members and not documenter.check_module(): - continue - - # try to also get a source code analyzer for attribute docs - try: - documenter.analyzer = ModuleAnalyzer.for_module(documenter.get_real_modname()) - # parse right now, to get PycodeErrors on parsing (results will - # be cached anyway) - documenter.analyzer.find_attr_docs() - except PycodeError as err: - logger.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules - documenter.analyzer = None - - # -- Grab the signature - - try: - sig = documenter.format_signature(show_annotation=False) - except TypeError: - # the documenter does not support ``show_annotation`` option - sig = documenter.format_signature() - - if not sig: - sig = '' - else: - sig = mangle_signature(sig, max_chars=max(10, MAX_ITEM_CHARS - len(display_name))) - - # -- Grab the summary - - documenter.add_content(None) - summary = extract_summary(self.bridge.result.data[:], self.state.document) - env_sum = self.extract_env_summary(self.bridge.result.data[:]) - items.append((display_name, sig, summary, real_name, env_sum)) - - return items - - def get_table(self, items: List[Tuple[str, str, str, str, str]]) -> List[Node]: - """Generate a proper list of table nodes for autosummary:: directive. - - *items* is a list produced by :meth:`get_items`. - """ - table_spec = addnodes.tabular_col_spec() - table_spec['spec'] = r'\X{1}{2}\X{1}{2}' - - table = autosummary_table('') - real_table = nodes.table('', classes=['longtable']) - table.append(real_table) - group = nodes.tgroup('', cols=3) - real_table.append(group) - group.append(nodes.colspec('', colwidth=10)) - group.append(nodes.colspec('', colwidth=70)) - group.append(nodes.colspec('', colwidth=30)) - body = nodes.tbody('') - group.append(body) - - def append_row(*column_texts: str) -> None: - row = nodes.row('', color="red") - source, line = self.state_machine.get_source_and_line() - for text in column_texts: - node = nodes.paragraph('') - vl = StringList() - vl.append(text, '%s:%d:' % (source, line)) - with switch_source_input(self.state, vl): - self.state.nested_parse(vl, 0, node) - if node and isinstance(node[0], nodes.paragraph): - node = node[0] - row.append(nodes.entry('', node)) - body.append(row) - - # add table's title - append_row("**API Name**", "**Description**", self.third_title) - for name, sig, summary, real_name, env_sum in items: - qualifier = 'obj' - if 'nosignatures' not in self.options: - col1 = ':%s:`%s <%s>`\\ %s' % (qualifier, name, real_name, rst.escape(sig)) - else: - col1 = ':%s:`%s <%s>`' % (qualifier, name, real_name) - col2 = summary - col3 = env_sum - append_row(col1, col2, col3) - - return [table_spec, table] - - -class DsNoteAutoSummary(DsAutosummary): - """ - Inherited from DsAutosummary. Add a third column about `Note` to the table. - """ - - def init(self): - """ - init method - """ - self.find_doc_name = ".. note::" - self.third_title = "**Note**" - self.default_doc = "None" - - def extract_env_summary(self, doc: List[str]) -> str: - """Extract env summary from docstring.""" - env_sum = self.default_doc - for piece in doc: - if piece.startswith(self.find_doc_name): - env_sum = piece[10:] - return env_sum - - -class DsPlatformAutoSummary(DsAutosummary): - """ - Inherited from DsAutosummary. Add a third column about `Supported Platforms` to the table. - """ - def init(self): - """ - init method - """ - self.find_doc_name = "Supported Platforms:" - self.third_title = "**{}**".format(self.find_doc_name[:-1]) - self.default_doc = "``Ascend`` ``GPU`` ``CPU``" - - -class DsCnAutoSummary(Autosummary): - """Overwrite MsPlatformAutosummary for chinese python api.""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.table_head = () - self.find_doc_name = "" - self.third_title = "" - self.default_doc = "" - self.third_name_en = "" - - def get_third_column_en(self, doc): - """Get the third column for en.""" - third_column = self.default_doc - for i, piece in enumerate(doc): - if piece.startswith(self.third_name_en): - try: - if "eprecated" in doc[i+1][4:]: - third_column = "弃用" - else: - third_column = doc[i+1][4:] - except IndexError: - third_column = '' - return third_column - - def get_summary_re(self, display_name: str): - """Get the summary""" - return re.compile(rf'\.\. \w+:\w+::\s+{display_name}.*?\n\n\s+(.*?)[。\n]') - - def run(self) -> List[Node]: - """Run this program""" - self.bridge = DocumenterBridge(self.env, self.state.document.reporter, - Options(), self.lineno, self.state) - - names = [x.strip().split()[0] for x in self.content if x.strip() and re.search(r'^[~a-zA-Z_]', x.strip()[0])] - items = self.get_items(names) - table_nodes = self.get_table(items) - - dirname = posixpath.dirname(self.env.docname) - - tree_prefix = self.options['toctree'].strip() - docnames = [] - names = [i[0] for i in items] - for name in names: - docname = posixpath.join(tree_prefix, name) - docname = posixpath.normpath(posixpath.join(dirname, docname)) - if docname not in self.env.found_docs: - continue - - docnames.append(docname) - - if docnames: - tocnode = addnodes.toctree() - tocnode['includefiles'] = docnames - tocnode['entries'] = [(None, docn) for docn in docnames] - tocnode['maxdepth'] = -1 - tocnode['glob'] = None - - table_nodes.append(autosummary_toc('', '', tocnode)) - - return table_nodes - - def get_items(self, names: List[str]) -> List[Tuple[str, str, str, str]]: - """Try to import the given names, and return a list of - ``[(name, signature, summary_string, real_name), ...]``. - """ - doc_path = os.path.dirname(self.state.document.current_source) - prefixes = get_import_prefixes_from_env(self.env) - items = [] # type: List[Tuple[str, str, str, str]] - - for name in names: - display_name = name - if name.startswith('~'): - name = name[1:] - display_name = name.split('.')[-1] - - file_path = os.path.normpath(os.path.join(doc_path, self.options['toctree'], display_name+'.rst')) - if self.if_need_solve(doc_path, display_name): - items = self.get_summary_info(file_path, display_name, items) - else: - try: - with mock(self.config.autosummary_mock_imports): - real_name, obj, parent, modname = import_by_name(name, prefixes=prefixes) - except ImportError: - logger.warning(__('failed to import %s'), name) - items.append((name, '', '')) - continue - - self.bridge.result = StringList() # initialize for each documenter - full_name = real_name - if not isinstance(obj, ModuleType): - # give explicitly separated module name, so that members - # of inner classes can be documented - full_name = modname + '::' + full_name[len(modname) + 1:] - # NB. using full_name here is important, since Documenters - # handle module prefixes slightly differently - doccls = get_documenter(self.env.app, obj, parent) - documenter = doccls(self.bridge, full_name) - - if not documenter.parse_name(): - logger.warning(__('failed to parse name %s'), real_name) - items.append((display_name, '', '')) - continue - if not documenter.import_object(): - logger.warning(__('failed to import object %s'), real_name) - items.append((display_name, '', '')) - continue - if documenter.options.members and not documenter.check_module(): - continue - - # try to also get a source code analyzer for attribute docs - try: - documenter.analyzer = ModuleAnalyzer.for_module(documenter.get_real_modname()) - # parse right now, to get PycodeErrors on parsing (results will - # be cached anyway) - documenter.analyzer.find_attr_docs() - except PycodeError as err: - logger.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules - documenter.analyzer = None - - # -- Grab the summary and third_colum - - documenter.add_content(None) - summary = extract_summary(self.bridge.result.data[:], self.state.document) - if self.table_head: - third_colum = self.get_third_column_en(self.bridge.result.data[:]) - items.append((display_name, summary, third_colum)) - else: - items.append((display_name, summary)) - - return items - - def if_need_solve(self, docpath: str, file: str) -> bool: - """If need solve""" - dir_name = self.options['toctree'] - origin_rst_files = self.env.config.rst_files - all_rst_files = self.env.found_docs - - spec_path = os.path.join('api_python', dir_name, file) - file_path = os.path.join(docpath, dir_name, file+'.rst') - generated_files = all_rst_files.difference(origin_rst_files) - return os.path.exists(file_path) and spec_path not in generated_files - - def get_summary_info( - self, file_path: str, file: str, - items: List[Tuple[str, str, str, - str]]) -> List[Tuple[str, str, str, str]]: - """Get the summary info.""" - summary_re_tag = re.compile(rf'\.\. \w+:\w+::\s+{file}.*?\n\s+:.*?:\n\n\s+(.*?)[。\n]') - summary_re_line = re.compile(rf'\.\. \w+:\w+::\s+{file}(?:.|\n|)+?\n\n\s+(.*?)[。\n]') - summary_re = self.get_summary_re(file) - content = '' - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - if content: - summary_str = summary_re.findall(content) - summary_str_tag = summary_re_tag.findall(content) - summary_str_line = summary_re_line.findall(content) - if summary_str: - if re.findall("[::,,。.;;]", summary_str[0][-1]): - logger.warning(f"{file}接口的概述格式需调整") - summary_str = summary_str[0] + '。' - elif summary_str_tag: - if re.findall("[::,,。.;;]", summary_str_tag[0][-1]): - logger.warning(f"{file}接口的概述格式需调整") - summary_str = summary_str_tag[0] + '。' - elif summary_str_line: - if re.findall("[::,,。.;;]", summary_str_line[0][-1]): - logger.warning(f"{file}接口的概述格式需调整") - summary_str = summary_str_line[0] + '。' - else: - summary_str = '' - if not self.table_head: - items.append((file, summary_str)) - else: - third_str = self.get_third_column(file, content) - if third_str: - third_str = third_str[0] - else: - third_str = '' - - items.append((file, summary_str, third_str)) - - return items - - def get_table(self, items: List[Tuple[str, str, str]]) -> List[Node]: - """Generate a proper list of table nodes for autosummary:: directive. - - *items* is a list produced by :meth:`get_items`. - """ - table_spec = addnodes.tabular_col_spec() - table = autosummary_table('') - real_table = nodes.table('', classes=['longtable']) - table.append(real_table) - - if not self.table_head: - table_spec['spec'] = r'\X{1}{2}\X{1}{2}' - group = nodes.tgroup('', cols=2) - real_table.append(group) - group.append(nodes.colspec('', colwidth=10)) - group.append(nodes.colspec('', colwidth=90)) - else: - table_spec['spec'] = r'\X{1}{2}\X{1}{2}\X{1}{2}' - group = nodes.tgroup('', cols=3) - real_table.append(group) - group.append(nodes.colspec('', colwidth=10)) - group.append(nodes.colspec('', colwidth=60)) - group.append(nodes.colspec('', colwidth=30)) - body = nodes.tbody('') - group.append(body) - - def append_row(*column_texts: str) -> None: - row = nodes.row('') - source, line = self.state_machine.get_source_and_line() - for text in column_texts: - node = nodes.paragraph('') - vl = StringList() - vl.append(text, '%s:%d:' % (source, line)) - with switch_source_input(self.state, vl): - self.state.nested_parse(vl, 0, node) - if node and isinstance(node[0], nodes.paragraph): - node = node[0] - row.append(nodes.entry('', node)) - body.append(row) - append_row(*self.table_head) - if not self.table_head: - try: - for name, summary in items: - qualifier = 'obj' - col1 = ':%s:`%s <%s>`' % (qualifier, name, name) - col2 = summary - append_row(col1, col2) - except ValueError: - logger.warning(items) - else: - for name, summary, other in items: - qualifier = 'obj' - col1 = ':%s:`%s <%s>`' % (qualifier, name, name) - col2 = summary - col3 = other - append_row(col1, col2, col3) - return [table_spec, table] diff --git a/docs/_ext/overwriteautosummary_generate.txt b/docs/_ext/overwriteautosummary_generate.txt deleted file mode 100644 index bf895b9..0000000 --- a/docs/_ext/overwriteautosummary_generate.txt +++ /dev/null @@ -1,716 +0,0 @@ -""" - sphinx.ext.autosummary.generate - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Usable as a library or script to generate automatic RST source files for - items referred to in autosummary:: directives. - - Each generated RST file contains a single auto*:: directive which - extracts the docstring of the referred item. - - Example Makefile rule:: - - generate: - sphinx-autogen -o source/generated source/*.rst - - :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import argparse -import importlib -import inspect -import locale -import os -import pkgutil -import pydoc -import re -import sys -import warnings -from gettext import NullTranslations -from os import path -from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, Type, Union - -from jinja2 import TemplateNotFound -from jinja2.sandbox import SandboxedEnvironment - -import sphinx.locale -from sphinx import __display_version__, package_dir -from sphinx.application import Sphinx -from sphinx.builders import Builder -from sphinx.config import Config -try: - from sphinx.deprecation import RemovedInSphinx50Warning -except ImportError: - class RemovedInSphinx50Warning(Warning): # 临时占位 - pass -from sphinx.ext.autodoc import Documenter -from sphinx.ext.autodoc.importer import import_module -try: - from exceptions import ImportExceptionGroup -except ImportError: - class ImportExceptionGroup(Exception): # 简单的回退实现 - pass -from sphinx.ext.autosummary import (get_documenter, import_by_name, - import_ivar_by_name) -from sphinx.locale import __ -from sphinx.pycode import ModuleAnalyzer, PycodeError -from sphinx.registry import SphinxComponentRegistry -from sphinx.util import logging, rst, split_full_qualified_name, get_full_modname -from sphinx.util.inspect import getall, safe_getattr -from sphinx.util.osutil import ensuredir -from sphinx.util.template import SphinxTemplateLoader - -logger = logging.getLogger(__name__) - - -class DummyApplication: - """Dummy Application class for sphinx-autogen command.""" - - def __init__(self, translator: NullTranslations) -> None: - self.config = Config() - self.registry = SphinxComponentRegistry() - self.messagelog: List[str] = [] - self.srcdir = "/" - self.translator = translator - self.verbosity = 0 - self._warncount = 0 - self.warningiserror = False - - self.config.add('autosummary_context', {}, True, None) - self.config.add('autosummary_filename_map', {}, True, None) - self.config.add('autosummary_ignore_module_all', True, 'env', bool) - self.config.add('docs_branch', '', True, None) - self.config.add('branch', '', True, None) - self.config.add('cst_module_name', '', True, None) - self.config.add('copy_repo', '', True, None) - self.config.add('giturl', '', True, None) - self.config.add('repo_whl', '', True, None) - self.config.init_values() - - def emit_firstresult(self, *args: Any) -> None: - pass - - -class AutosummaryEntry(NamedTuple): - name: str - path: str - template: str - recursive: bool - - -def setup_documenters(app: Any) -> None: - from sphinx.ext.autodoc import (AttributeDocumenter, ClassDocumenter, DataDocumenter, - DecoratorDocumenter, ExceptionDocumenter, - FunctionDocumenter, MethodDocumenter, ModuleDocumenter, - NewTypeAttributeDocumenter, NewTypeDataDocumenter, - PropertyDocumenter) - documenters: List[Type[Documenter]] = [ - ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter, - FunctionDocumenter, MethodDocumenter, NewTypeAttributeDocumenter, - NewTypeDataDocumenter, AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter, - ] - for documenter in documenters: - app.registry.add_documenter(documenter.objtype, documenter) - - -def _simple_info(msg: str) -> None: - warnings.warn('_simple_info() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - print(msg) - - -def _simple_warn(msg: str) -> None: - warnings.warn('_simple_warn() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - print('WARNING: ' + msg, file=sys.stderr) - - -def _underline(title: str, line: str = '=') -> str: - if '\n' in title: - raise ValueError('Can only underline single lines') - return title + '\n' + line * len(title) - - -class AutosummaryRenderer: - """A helper class for rendering.""" - - def __init__(self, app: Union[Builder, Sphinx], template_dir: str = None) -> None: - if isinstance(app, Builder): - warnings.warn('The first argument for AutosummaryRenderer has been ' - 'changed to Sphinx object', - RemovedInSphinx50Warning, stacklevel=2) - if template_dir: - warnings.warn('template_dir argument for AutosummaryRenderer is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - - system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')] - loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path, - system_templates_path) - - self.env = SandboxedEnvironment(loader=loader) - self.env.filters['escape'] = rst.escape - self.env.filters['e'] = rst.escape - self.env.filters['underline'] = _underline - - if isinstance(app, (Sphinx, DummyApplication)): - if app.translator: - self.env.add_extension("jinja2.ext.i18n") - self.env.install_gettext_translations(app.translator) - elif isinstance(app, Builder): - if app.app.translator: - self.env.add_extension("jinja2.ext.i18n") - self.env.install_gettext_translations(app.app.translator) - - def exists(self, template_name: str) -> bool: - """Check if template file exists.""" - warnings.warn('AutosummaryRenderer.exists() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - try: - self.env.get_template(template_name) - return True - except TemplateNotFound: - return False - - def render(self, template_name: str, context: Dict) -> str: - """Render a template file.""" - try: - template = self.env.get_template(template_name) - except TemplateNotFound: - try: - # objtype is given as template_name - template = self.env.get_template('autosummary/%s.rst' % template_name) - except TemplateNotFound: - # fallback to base.rst - template = self.env.get_template('autosummary/base.rst') - - return template.render(context) - - -# -- Generating output --------------------------------------------------------- - - -class ModuleScanner: - def __init__(self, app: Any, obj: Any) -> None: - self.app = app - self.object = obj - - def get_object_type(self, name: str, value: Any) -> str: - return get_documenter(self.app, value, self.object).objtype - - def is_skipped(self, name: str, value: Any, objtype: str) -> bool: - try: - return self.app.emit_firstresult('autodoc-skip-member', objtype, - name, value, False, {}) - except Exception as exc: - logger.warning(__('autosummary: failed to determine %r to be documented, ' - 'the following exception was raised:\n%s'), - name, exc, type='autosummary') - return False - - def scan(self, imported_members: bool) -> List[str]: - members = [] - for name in members_of(self.object, self.app.config): - try: - value = safe_getattr(self.object, name) - except AttributeError: - value = None - - objtype = self.get_object_type(name, value) - if self.is_skipped(name, value, objtype): - continue - - try: - if inspect.ismodule(value): - imported = True - elif safe_getattr(value, '__module__') != self.object.__name__: - imported = True - else: - imported = False - except AttributeError: - imported = False - - respect_module_all = not self.app.config.autosummary_ignore_module_all - if imported_members: - # list all members up - members.append(name) - elif imported is False: - # list not-imported members - members.append(name) - elif '__all__' in dir(self.object) and respect_module_all: - # list members that have __all__ set - members.append(name) - - return members - - -def members_of(obj: Any, conf: Config) -> Sequence[str]: - """Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute - - Follows the ``conf.autosummary_ignore_module_all`` setting.""" - - if conf.autosummary_ignore_module_all: - return dir(obj) - else: - return getall(obj) or dir(obj) - - -def generate_autosummary_content(name: str, obj: Any, parent: Any, - template: AutosummaryRenderer, template_name: str, - imported_members: bool, app: Any, - recursive: bool, context: Dict, - modname: str = None, qualname: str = None) -> str: - doc = get_documenter(app, obj, parent) - - def skip_member(obj: Any, name: str, objtype: str) -> bool: - try: - return app.emit_firstresult('autodoc-skip-member', objtype, name, - obj, False, {}) - except Exception as exc: - logger.warning(__('autosummary: failed to determine %r to be documented, ' - 'the following exception was raised:\n%s'), - name, exc, type='autosummary') - return False - - def get_class_members(obj: Any) -> Dict[str, Any]: - members = sphinx.ext.autodoc.get_class_members(obj, [qualname], safe_getattr) - return {name: member.object for name, member in members.items()} - - def get_module_members(obj: Any) -> Dict[str, Any]: - members = {} - for name in members_of(obj, app.config): - try: - members[name] = safe_getattr(obj, name) - except AttributeError: - continue - return members - - def get_all_members(obj: Any) -> Dict[str, Any]: - if doc.objtype == "module": - return get_module_members(obj) - elif doc.objtype == "class": - return get_class_members(obj) - return {} - - def get_members(obj: Any, types: Set[str], include_public: List[str] = [], - imported: bool = True) -> Tuple[List[str], List[str]]: - items: List[str] = [] - public: List[str] = [] - - all_members = get_all_members(obj) - for name, value in all_members.items(): - documenter = get_documenter(app, value, obj) - if documenter.objtype in types: - # skip imported members if expected - if imported or getattr(value, '__module__', None) == obj.__name__: - skipped = skip_member(value, name, documenter.objtype) - if skipped is True: - pass - elif skipped is False: - # show the member forcedly - items.append(name) - public.append(name) - else: - items.append(name) - if name in include_public or not name.startswith('_'): - # considers member as public - public.append(name) - return public, items - - def get_module_attrs(members: Any) -> Tuple[List[str], List[str]]: - """Find module attributes with docstrings.""" - attrs, public = [], [] - try: - analyzer = ModuleAnalyzer.for_module(name) - attr_docs = analyzer.find_attr_docs() - for namespace, attr_name in attr_docs: - if namespace == '' and attr_name in members: - attrs.append(attr_name) - if not attr_name.startswith('_'): - public.append(attr_name) - except PycodeError: - pass # give up if ModuleAnalyzer fails to parse code - return public, attrs - - def get_modules(obj: Any) -> Tuple[List[str], List[str]]: - items: List[str] = [] - for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__): - fullname = name + '.' + modname - try: - module = import_module(fullname) - if module and hasattr(module, '__sphinx_mock__'): - continue - except ImportError: - pass - - items.append(fullname) - public = [x for x in items if not x.split('.')[-1].startswith('_')] - return public, items - - ns: Dict[str, Any] = {} - ns.update(context) - - if doc.objtype == 'module': - scanner = ModuleScanner(app, obj) - ns['members'] = scanner.scan(imported_members) - ns['functions'], ns['all_functions'] = \ - get_members(obj, {'function'}, imported=imported_members) - ns['classes'], ns['all_classes'] = \ - get_members(obj, {'class'}, imported=imported_members) - ns['exceptions'], ns['all_exceptions'] = \ - get_members(obj, {'exception'}, imported=imported_members) - ns['attributes'], ns['all_attributes'] = \ - get_module_attrs(ns['members']) - ispackage = hasattr(obj, '__path__') - if ispackage and recursive: - ns['modules'], ns['all_modules'] = get_modules(obj) - elif doc.objtype == 'class': - ns['members'] = dir(obj) - ns['inherited_members'] = \ - set(dir(obj)) - set(obj.__dict__.keys()) - ns['methods'], ns['all_methods'] = \ - get_members(obj, {'method'}, ['__init__']) - ns['attributes'], ns['all_attributes'] = \ - get_members(obj, {'attribute', 'property'}) - - if modname is None or qualname is None: - modname, qualname = split_full_qualified_name(name) - - if doc.objtype in ('method', 'attribute', 'property'): - ns['class'] = qualname.rsplit(".", 1)[0] - - if doc.objtype in ('class',): - shortname = qualname - else: - shortname = qualname.rsplit(".", 1)[-1] - - ns['fullname'] = name - ns['module'] = modname - ns['objname'] = qualname - ns['name'] = shortname - - ns['objtype'] = doc.objtype - ns['underline'] = len(name) * '=' - - if template_name: - return template.render(template_name, ns) - else: - return template.render(doc.objtype, ns) - - -def generate_autosummary_docs(sources: List[str], output_dir: str = None, - suffix: str = '.rst', base_path: str = None, - builder: Builder = None, template_dir: str = None, - imported_members: bool = False, app: Any = None, - overwrite: bool = True, encoding: str = 'utf-8') -> None: - - if builder: - warnings.warn('builder argument for generate_autosummary_docs() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - - if template_dir: - warnings.warn('template_dir argument for generate_autosummary_docs() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - - showed_sources = list(sorted(sources)) - if len(showed_sources) > 20: - showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] - logger.info(__('[autosummary] generating autosummary for: %s') % - ', '.join(showed_sources)) - - if output_dir: - logger.info(__('[autosummary] writing to %s') % output_dir) - - if base_path is not None: - sources = [os.path.join(base_path, filename) for filename in sources] - - template = AutosummaryRenderer(app) - - # read - items = find_autosummary_in_files(sources) - - # keep track of new files - new_files = [] - - if app: - filename_map = app.config.autosummary_filename_map - else: - filename_map = {} - - # write - for entry in sorted(set(items), key=str): - if entry.path is None: - # The corresponding autosummary:: directive did not have - # a :toctree: option - continue - - path = output_dir or os.path.abspath(entry.path) - ensuredir(path) - - try: - name, obj, parent, modname = import_by_name(entry.name) - qualname = name.replace(modname + ".", "") - except ImportExceptionGroup as exc: - try: - # try to import as an instance attribute - name, obj, parent, modname = import_ivar_by_name(entry.name) - qualname = name.replace(modname + ".", "") - except ImportError as exc2: - if exc2.__cause__: - exceptions: List[BaseException] = exc.exceptions + [exc2.__cause__] - else: - exceptions = exc.exceptions + [exc2] - - errors = list(set("* %s: %s" % (type(e).__name__, e) for e in exceptions)) - logger.warning(__('[autosummary] failed to import %s.\nPossible hints:\n%s'), - entry.name, '\n'.join(errors)) - continue - - context: Dict[str, Any] = {} - if app: - context.update(app.config.autosummary_context) - - content = generate_autosummary_content(name, obj, parent, template, entry.template, - imported_members, app, entry.recursive, context, - modname, qualname) - try: - py_source_rel = get_full_modname(modname, qualname).replace('.', '/') + '.py' - except: - logger.warning(name) - py_source_rel = '' - - re_view = f"\n.. image:: https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/{app.config.docs_branch}/" + \ - f"resource/_static/logo_source_en.svg\n :target: " + app.config.giturl + \ - f"{app.config.copy_repo}/blob/{app.config.branch}/" + app.config.repo_whl + \ - py_source_rel.split(app.config.cst_module_name)[-1] + '\n :alt: View Source On Gitee\n\n' - - if re_view not in content and py_source_rel: - content = re.sub('([=]{5,})\n', r'\1\n' + re_view, content, 1) - filename = os.path.join(path, filename_map.get(name, name) + suffix) - if os.path.isfile(filename): - with open(filename, encoding=encoding) as f: - old_content = f.read() - - if content == old_content: - continue - elif overwrite: # content has changed - with open(filename, 'w', encoding=encoding) as f: - f.write(content) - new_files.append(filename) - else: - with open(filename, 'w', encoding=encoding) as f: - f.write(content) - new_files.append(filename) - - # descend recursively to new files - if new_files: - generate_autosummary_docs(new_files, output_dir=output_dir, - suffix=suffix, base_path=base_path, - builder=builder, template_dir=template_dir, - imported_members=imported_members, app=app, - overwrite=overwrite) - - -# -- Finding documented entries in files --------------------------------------- - -def find_autosummary_in_files(filenames: List[str]) -> List[AutosummaryEntry]: - """Find out what items are documented in source/*.rst. - - See `find_autosummary_in_lines`. - """ - documented: List[AutosummaryEntry] = [] - for filename in filenames: - with open(filename, encoding='utf-8', errors='ignore') as f: - lines = f.read().splitlines() - documented.extend(find_autosummary_in_lines(lines, filename=filename)) - return documented - - -def find_autosummary_in_docstring(name: str, module: str = None, filename: str = None - ) -> List[AutosummaryEntry]: - """Find out what items are documented in the given object's docstring. - - See `find_autosummary_in_lines`. - """ - if module: - warnings.warn('module argument for find_autosummary_in_docstring() is deprecated.', - RemovedInSphinx50Warning, stacklevel=2) - - try: - real_name, obj, parent, modname = import_by_name(name) - lines = pydoc.getdoc(obj).splitlines() - return find_autosummary_in_lines(lines, module=name, filename=filename) - except AttributeError: - pass - except ImportExceptionGroup as exc: - errors = list(set("* %s: %s" % (type(e).__name__, e) for e in exc.exceptions)) - print('Failed to import %s.\nPossible hints:\n%s' % (name, '\n'.join(errors))) - except SystemExit: - print("Failed to import '%s'; the module executes module level " - "statement and it might call sys.exit()." % name) - return [] - - -def find_autosummary_in_lines(lines: List[str], module: str = None, filename: str = None - ) -> List[AutosummaryEntry]: - """Find out what items appear in autosummary:: directives in the - given lines. - - Returns a list of (name, toctree, template) where *name* is a name - of an object and *toctree* the :toctree: path of the corresponding - autosummary directive (relative to the root of the file name), and - *template* the value of the :template: option. *toctree* and - *template* ``None`` if the directive does not have the - corresponding options set. - """ - autosummary_re = re.compile(r'^(\s*)\.\.\s+(ms[a-z]*)?autosummary::\s*') - automodule_re = re.compile( - r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$') - module_re = re.compile( - r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$') - autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?') - recursive_arg_re = re.compile(r'^\s+:recursive:\s*$') - toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$') - template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$') - - documented: List[AutosummaryEntry] = [] - - recursive = False - toctree: str = None - template = None - current_module = module - in_autosummary = False - base_indent = "" - - for line in lines: - if in_autosummary: - m = recursive_arg_re.match(line) - if m: - recursive = True - continue - - m = toctree_arg_re.match(line) - if m: - toctree = m.group(1) - if filename: - toctree = os.path.join(os.path.dirname(filename), - toctree) - continue - - m = template_arg_re.match(line) - if m: - template = m.group(1).strip() - continue - - if line.strip().startswith(':'): - continue # skip options - - m = autosummary_item_re.match(line) - if m: - name = m.group(1).strip() - if name.startswith('~'): - name = name[1:] - if current_module and \ - not name.startswith(current_module + '.'): - name = "%s.%s" % (current_module, name) - documented.append(AutosummaryEntry(name, toctree, template, recursive)) - continue - - if not line.strip() or line.startswith(base_indent + " "): - continue - - in_autosummary = False - - m = autosummary_re.match(line) - if m: - in_autosummary = True - base_indent = m.group(1) - recursive = False - toctree = None - template = None - continue - - m = automodule_re.search(line) - if m: - current_module = m.group(1).strip() - # recurse into the automodule docstring - documented.extend(find_autosummary_in_docstring( - current_module, filename=filename)) - continue - - m = module_re.match(line) - if m: - current_module = m.group(2) - continue - - return documented - - -def get_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - usage='%(prog)s [OPTIONS] ...', - epilog=__('For more information, visit .'), - description=__(""" -Generate ReStructuredText using autosummary directives. - -sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates -the reStructuredText files from the autosummary directives contained in the -given input files. - -The format of the autosummary directive is documented in the -``sphinx.ext.autosummary`` Python module and can be read using:: - - pydoc sphinx.ext.autosummary -""")) - - parser.add_argument('--version', action='version', dest='show_version', - version='%%(prog)s %s' % __display_version__) - - parser.add_argument('source_file', nargs='+', - help=__('source files to generate rST files for')) - - parser.add_argument('-o', '--output-dir', action='store', - dest='output_dir', - help=__('directory to place all output in')) - parser.add_argument('-s', '--suffix', action='store', dest='suffix', - default='rst', - help=__('default suffix for files (default: ' - '%(default)s)')) - parser.add_argument('-t', '--templates', action='store', dest='templates', - default=None, - help=__('custom template directory (default: ' - '%(default)s)')) - parser.add_argument('-i', '--imported-members', action='store_true', - dest='imported_members', default=False, - help=__('document imported members (default: ' - '%(default)s)')) - parser.add_argument('-a', '--respect-module-all', action='store_true', - dest='respect_module_all', default=False, - help=__('document exactly the members in module __all__ attribute. ' - '(default: %(default)s)')) - - return parser - - -def main(argv: List[str] = sys.argv[1:]) -> None: - sphinx.locale.setlocale(locale.LC_ALL, '') - sphinx.locale.init_console(os.path.join(package_dir, 'locale'), 'sphinx') - translator, _ = sphinx.locale.init([], None) - - app = DummyApplication(translator) - logging.setup(app, sys.stdout, sys.stderr) # type: ignore - setup_documenters(app) - args = get_parser().parse_args(argv) - - if args.templates: - app.config.templates_path.append(path.abspath(args.templates)) - app.config.autosummary_ignore_module_all = not args.respect_module_all # type: ignore - - generate_autosummary_docs(args.source_file, args.output_dir, - '.' + args.suffix, - imported_members=args.imported_members, - app=app) - - -if __name__ == '__main__': - main() diff --git a/docs/_ext/overwriteobjectiondirective.txt b/docs/_ext/overwriteobjectiondirective.txt deleted file mode 100644 index b648b03..0000000 --- a/docs/_ext/overwriteobjectiondirective.txt +++ /dev/null @@ -1,440 +0,0 @@ -""" - sphinx.directives - ~~~~~~~~~~~~~~~~~ - - Handlers for additional ReST directives. - - :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import re -import inspect -import importlib -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Tuple, TypeVar, cast - -from docutils import nodes -from docutils.nodes import Node -from docutils.parsers.rst import directives, roles - -from sphinx import addnodes -from sphinx.addnodes import desc_signature -try: - from sphinx.deprecation import RemovedInSphinx50Warning -except ImportError: - class RemovedInSphinx50Warning(Warning): # 临时占位 - pass -from sphinx.util import docutils, logging -from sphinx.util.docfields import DocFieldTransformer, Field, TypedField -from sphinx.util.docutils import SphinxDirective -from sphinx.util.typing import OptionSpec - -if TYPE_CHECKING: - from sphinx.application import Sphinx - - -# RE to strip backslash escapes -nl_escape_re = re.compile(r'\\\n') -strip_backslash_re = re.compile(r'\\(.)') - -T = TypeVar('T') -logger = logging.getLogger(__name__) - -def optional_int(argument: str) -> int: - """ - Check for an integer argument or None value; raise ``ValueError`` if not. - """ - if argument is None: - return None - else: - value = int(argument) - if value < 0: - raise ValueError('negative value; must be positive or zero') - return value - -def get_api(fullname): - try: - module_name, api_name= ".".join(fullname.split('.')[:-1]), fullname.split('.')[-1] - module_import = importlib.import_module(module_name) - except ModuleNotFoundError: - module_name, api_name = ".".join(fullname.split('.')[:-2]), ".".join(fullname.split('.')[-2:]) - module_import = importlib.import_module(module_name) - api = eval(f"module_import.{api_name}") - return api - -def get_example(name: str): - try: - api_doc = inspect.getdoc(get_api(name)) - example_str = re.findall(r'Examples:\n([\w\W]*?)(\n\n|$)', api_doc) - if not example_str: - return [] - if '.. note::' in example_str[0][0]: - api_doc = re.sub(r'Examples:\n \.\. note::(?:.|\n)*? >>>', r'Examples:\n >>>', api_doc) - example_str = re.findall(r'(? Dict[str, Tuple[Field, bool]]: - if self._doc_field_type_map == {}: - self._doc_field_type_map = {} - for field in self.doc_field_types: - for name in field.names: - self._doc_field_type_map[name] = (field, False) - - if field.is_typed: - typed_field = cast(TypedField, field) - for name in typed_field.typenames: - self._doc_field_type_map[name] = (field, True) - - return self._doc_field_type_map - - def get_signatures(self) -> List[str]: - """ - Retrieve the signatures to document from the directive arguments. By - default, signatures are given as arguments, one per line. - - Backslash-escaping of newlines is supported. - """ - lines = nl_escape_re.sub('', self.arguments[0]).split('\n') - if self.config.strip_signature_backslash: - # remove backslashes to support (dummy) escapes; helps Vim highlighting - return [strip_backslash_re.sub(r'\1', line.strip()) for line in lines] - else: - return [line.strip() for line in lines] - - def handle_signature(self, sig: str, signode: desc_signature) -> Any: - """ - Parse the signature *sig* into individual nodes and append them to - *signode*. If ValueError is raised, parsing is aborted and the whole - *sig* is put into a single desc_name node. - - The return value should be a value that identifies the object. It is - passed to :meth:`add_target_and_index()` unchanged, and otherwise only - used to skip duplicates. - """ - raise ValueError - - def add_target_and_index(self, name: Any, sig: str, signode: desc_signature) -> None: - """ - Add cross-reference IDs and entries to self.indexnode, if applicable. - - *name* is whatever :meth:`handle_signature()` returned. - """ - return # do nothing by default - - def before_content(self) -> None: - """ - Called before parsing content. Used to set information about the current - directive context on the build environment. - """ - pass - - def transform_content(self, contentnode: addnodes.desc_content) -> None: - """ - Called after creating the content through nested parsing, - but before the ``object-description-transform`` event is emitted, - and before the info-fields are transformed. - Can be used to manipulate the content. - """ - pass - - def after_content(self) -> None: - """ - Called after parsing content. Used to reset information about the - current directive context on the build environment. - """ - pass - - def check_class_end(self, content): - for i in content: - if not i.startswith('.. include::') and i != "\n" and i != "": - return False - return True - - def extend_items(self, rst_file, start_num, num): - ls = [] - for i in range(1, num+1): - ls.append((rst_file, start_num+i)) - return ls - - def run(self) -> List[Node]: - """ - Main directive entry function, called by docutils upon encountering the - directive. - - This directive is meant to be quite easily subclassable, so it delegates - to several additional methods. What it does: - - * find out if called as a domain-specific directive, set self.domain - * create a `desc` node to fit all description inside - * parse standard options, currently `noindex` - * create an index node if needed as self.indexnode - * parse all given signatures (as returned by self.get_signatures()) - using self.handle_signature(), which should either return a name - or raise ValueError - * add index entries using self.add_target_and_index() - * parse the content and handle doc fields in it - """ - if ':' in self.name: - self.domain, self.objtype = self.name.split(':', 1) - else: - self.domain, self.objtype = '', self.name - self.indexnode = addnodes.index(entries=[]) - - node = addnodes.desc() - node.document = self.state.document - node['domain'] = self.domain - # 'desctype' is a backwards compatible attribute - node['objtype'] = node['desctype'] = self.objtype - node['noindex'] = noindex = ('noindex' in self.options) - if self.domain: - node['classes'].append(self.domain) - node['classes'].append(node['objtype']) - - self.names: List[T] = [] - signatures = self.get_signatures() - for sig in signatures: - # add a signature node for each signature in the current unit - # and add a reference target for it - signode = addnodes.desc_signature(sig, '') - self.set_source_info(signode) - node.append(signode) - try: - # name can also be a tuple, e.g. (classname, objname); - # this is strictly domain-specific (i.e. no assumptions may - # be made in this base class) - name = self.handle_signature(sig, signode) - except ValueError: - # signature parsing failed - signode.clear() - signode += addnodes.desc_name(sig, sig) - continue # we don't want an index entry here - if name not in self.names: - self.names.append(name) - if not noindex: - # only add target and index entry if this is the first - # description of the object with this name in this desc block - self.add_target_and_index(name, sig, signode) - - contentnode = addnodes.desc_content() - node.append(contentnode) - if self.names: - # needed for association of version{added,changed} directives - self.env.temp_data['object'] = self.names[0] - self.before_content() - try: - example = get_example(self.names[0][0]) - platforms = get_platforms(self.names[0][0]) - except Exception as e: - example = '' - platforms = '' - logger.warning(f'Error API names in {self.arguments[0]}.') - logger.warning(f'{e}') - extra = platforms + example - if "**样例:**" not in example and example: - try: - if self.objtype == "method": - index_platforms = 0 - for num, i in enumerate(self.content.data): - if i.startswith('样例:'): - index_platforms = num - break - if index_platforms and platforms: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - count = len(self.content.data) - for i in platforms: - self.content.data.insert(index_platforms-count, i) - else: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - self.content.data.extend(example) - else: - index_num = 0 - index_platforms = 0 - for num, i in enumerate(self.content.data): - if i.startswith('.. py:method::') or self.check_class_end(self.content.data[num:]): - index_num = num - break - if index_num: - for num, j in enumerate(self.content.data[:index_num]): - if j.startswith('样例:'): - index_platforms = num - break - if index_platforms and platforms: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - count = len(self.content.data) - for k in platforms: - self.content.data.insert(index_platforms-count, k) - else: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - count = len(self.content.data) - count_plat = len(platforms) - for i in example: - self.content.data.insert(index_num-count+count_plat, i) - else: - index_platforms = 0 - for num, i in enumerate(self.content.data): - if i.startswith('样例:'): - index_platforms = num - break - if index_platforms and platforms: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - count = len(self.content.data) - for i in platforms: - self.content.data.insert(index_platforms-count, i) - else: - self.content.data[index_platforms] = '**样例:**' - self.content.data.insert(index_platforms+1, '') - self.content.data.extend(example) - except Exception as e: - logger.warning(e) - elif extra: - if self.objtype == "method": - self.content.data.extend(extra) - else: - index_num = 0 - for num, i in enumerate(self.content.data): - if i.startswith('.. py:method::') or self.check_class_end(self.content.data[num:]): - index_num = num - break - if index_num: - count = len(self.content.data) - for i in extra: - self.content.data.insert(index_num-count, i) - else: - self.content.data.extend(extra) - try: - self.content.items.extend(self.extend_items(self.content.items[0][0], self.content.items[-1][1], len(extra))) - except Exception as e: - logger.warning(f'{e}') - self.state.nested_parse(self.content, self.content_offset, contentnode) - self.transform_content(contentnode) - self.env.app.emit('object-description-transform', - self.domain, self.objtype, contentnode) - DocFieldTransformer(self).transform_all(contentnode) - self.env.temp_data['object'] = None - self.after_content() - return [self.indexnode, node] - - -class DefaultRole(SphinxDirective): - """ - Set the default interpreted text role. Overridden from docutils. - """ - - optional_arguments = 1 - final_argument_whitespace = False - - def run(self) -> List[Node]: - if not self.arguments: - docutils.unregister_role('') - return [] - role_name = self.arguments[0] - role, messages = roles.role(role_name, self.state_machine.language, - self.lineno, self.state.reporter) - if role: - docutils.register_role('', role) - self.env.temp_data['default_role'] = role_name - else: - literal_block = nodes.literal_block(self.block_text, self.block_text) - reporter = self.state.reporter - error = reporter.error('Unknown interpreted text role "%s".' % role_name, - literal_block, line=self.lineno) - messages += [error] - - return cast(List[nodes.Node], messages) - - -class DefaultDomain(SphinxDirective): - """ - Directive to (re-)set the default domain for this source file. - """ - - has_content = False - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = False - option_spec = {} # type: Dict - - def run(self) -> List[Node]: - domain_name = self.arguments[0].lower() - # if domain_name not in env.domains: - # # try searching by label - # for domain in env.domains.values(): - # if domain.label.lower() == domain_name: - # domain_name = domain.name - # break - self.env.temp_data['default_domain'] = self.env.domains.get(domain_name) - return [] - -def setup(app: "Sphinx") -> Dict[str, Any]: - app.add_config_value("strip_signature_backslash", False, 'env') - directives.register_directive('default-role', DefaultRole) - directives.register_directive('default-domain', DefaultDomain) - directives.register_directive('describe', ObjectDescription) - # new, more consistent, name - directives.register_directive('object', ObjectDescription) - - app.add_event('object-description-transform') - - return { - 'version': 'builtin', - 'parallel_read_safe': True, - 'parallel_write_safe': True, - } - diff --git a/docs/_ext/overwriteviewcode.txt b/docs/_ext/overwriteviewcode.txt deleted file mode 100644 index a12f161..0000000 --- a/docs/_ext/overwriteviewcode.txt +++ /dev/null @@ -1,382 +0,0 @@ -""" - sphinx.ext.viewcode - ~~~~~~~~~~~~~~~~~~~ - - Add links to module code in Python object descriptions. - - :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -import posixpath -import traceback -import warnings -from os import path -from typing import Any, Dict, Generator, Iterable, Optional, Set, Tuple, cast - -from docutils import nodes -from docutils.nodes import Element, Node - -import sphinx -from sphinx import addnodes -from sphinx.application import Sphinx -from sphinx.builders import Builder -from sphinx.builders.html import StandaloneHTMLBuilder -try: - from sphinx.deprecation import RemovedInSphinx50Warning -except ImportError: - class RemovedInSphinx50Warning(Warning): # 临时占位 - pass -from sphinx.environment import BuildEnvironment -from sphinx.locale import _, __ -from sphinx.pycode import ModuleAnalyzer -from sphinx.transforms.post_transforms import SphinxPostTransform -from sphinx.util import get_full_modname, logging, status_iterator -from sphinx.util.nodes import make_refnode - - -logger = logging.getLogger(__name__) - - -OUTPUT_DIRNAME = '_modules' - - -class viewcode_anchor(Element): - """Node for viewcode anchors. - - This node will be processed in the resolving phase. - For viewcode supported builders, they will be all converted to the anchors. - For not supported builders, they will be removed. - """ - - -def _get_full_modname(app: Sphinx, modname: str, attribute: str) -> Optional[str]: - try: - return get_full_modname(modname, attribute) - except AttributeError: - # sphinx.ext.viewcode can't follow class instance attribute - # then AttributeError logging output only verbose mode. - logger.verbose('Didn\'t find %s in %s', attribute, modname) - return None - except Exception as e: - # sphinx.ext.viewcode follow python domain directives. - # because of that, if there are no real modules exists that specified - # by py:function or other directives, viewcode emits a lot of warnings. - # It should be displayed only verbose mode. - logger.verbose(traceback.format_exc().rstrip()) - logger.verbose('viewcode can\'t import %s, failed with error "%s"', modname, e) - return None - - -def is_supported_builder(builder: Builder) -> bool: - if builder.format != 'html': - return False - elif builder.name == 'singlehtml': - return False - elif builder.name.startswith('epub') and not builder.config.viewcode_enable_epub: - return False - else: - return True - - -def doctree_read(app: Sphinx, doctree: Node) -> None: - env = app.builder.env - if not hasattr(env, '_viewcode_modules'): - env._viewcode_modules = {} # type: ignore - - def has_tag(modname: str, fullname: str, docname: str, refname: str) -> bool: - entry = env._viewcode_modules.get(modname, None) # type: ignore - if entry is False: - return False - - code_tags = app.emit_firstresult('viewcode-find-source', modname) - if code_tags is None: - try: - analyzer = ModuleAnalyzer.for_module(modname) - analyzer.find_tags() - except Exception: - env._viewcode_modules[modname] = False # type: ignore - return False - - code = analyzer.code - tags = analyzer.tags - else: - code, tags = code_tags - - if entry is None or entry[0] != code: - entry = code, tags, {}, refname - env._viewcode_modules[modname] = entry # type: ignore - _, tags, used, _ = entry - if fullname in tags: - used[fullname] = docname - return True - - return False - - for objnode in list(doctree.findall(addnodes.desc)): - if objnode.get('domain') != 'py': - continue - names: Set[str] = set() - for signode in objnode: - if not isinstance(signode, addnodes.desc_signature): - continue - modname = signode.get('module') - fullname = signode.get('fullname') - try: - if fullname and modname==None: - if fullname.split('.')[-1].lower() == fullname.split('.')[-1] and fullname.split('.')[-2].lower() != fullname.split('.')[-2]: - modname = '.'.join(fullname.split('.')[:-2]) - fullname = '.'.join(fullname.split('.')[-2:]) - else: - modname = '.'.join(fullname.split('.')[:-1]) - fullname = fullname.split('.')[-1] - fullname_new = fullname - except Exception: - logger.warning(f'error_modename:{modname}') - logger.warning(f'error_fullname:{fullname}') - refname = modname - if env.config.viewcode_follow_imported_members: - new_modname = app.emit_firstresult( - 'viewcode-follow-imported', modname, fullname, - ) - if not new_modname: - new_modname = _get_full_modname(app, modname, fullname) - modname = new_modname - # logger.warning(f'new_modename:{modname}') - if not modname: - continue - # fullname = signode.get('fullname') - # if fullname and modname==None: - fullname = fullname_new - if not has_tag(modname, fullname, env.docname, refname): - continue - if fullname in names: - # only one link per name, please - continue - names.add(fullname) - pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')) - signode += viewcode_anchor(reftarget=pagename, refid=fullname, refdoc=env.docname) - - -def env_merge_info(app: Sphinx, env: BuildEnvironment, docnames: Iterable[str], - other: BuildEnvironment) -> None: - if not hasattr(other, '_viewcode_modules'): - return - # create a _viewcode_modules dict on the main environment - if not hasattr(env, '_viewcode_modules'): - env._viewcode_modules = {} # type: ignore - # now merge in the information from the subprocess - for modname, entry in other._viewcode_modules.items(): # type: ignore - if modname not in env._viewcode_modules: # type: ignore - env._viewcode_modules[modname] = entry # type: ignore - else: - if env._viewcode_modules[modname]: # type: ignore - used = env._viewcode_modules[modname][2] # type: ignore - for fullname, docname in entry[2].items(): - if fullname not in used: - used[fullname] = docname - - -def env_purge_doc(app: Sphinx, env: BuildEnvironment, docname: str) -> None: - modules = getattr(env, '_viewcode_modules', {}) - - for modname, entry in list(modules.items()): - if entry is False: - continue - - code, tags, used, refname = entry - for fullname in list(used): - if used[fullname] == docname: - used.pop(fullname) - - if len(used) == 0: - modules.pop(modname) - - -class ViewcodeAnchorTransform(SphinxPostTransform): - """Convert or remove viewcode_anchor nodes depends on builder.""" - default_priority = 100 - - def run(self, **kwargs: Any) -> None: - if is_supported_builder(self.app.builder): - self.convert_viewcode_anchors() - else: - self.remove_viewcode_anchors() - - def convert_viewcode_anchors(self) -> None: - for node in self.document.findall(viewcode_anchor): - anchor = nodes.inline('', _('[源代码]'), classes=['viewcode-link']) - refnode = make_refnode(self.app.builder, node['refdoc'], node['reftarget'], - node['refid'], anchor) - node.replace_self(refnode) - - def remove_viewcode_anchors(self) -> None: - for node in list(self.document.findall(viewcode_anchor)): - node.parent.remove(node) - - -def missing_reference(app: Sphinx, env: BuildEnvironment, node: Element, contnode: Node - ) -> Optional[Node]: - # resolve our "viewcode" reference nodes -- they need special treatment - if node['reftype'] == 'viewcode': - warnings.warn('viewcode extension is no longer use pending_xref node. ' - 'Please update your extension.', RemovedInSphinx50Warning) - return make_refnode(app.builder, node['refdoc'], node['reftarget'], - node['refid'], contnode) - - return None - - -def get_module_filename(app: Sphinx, modname: str) -> Optional[str]: - """Get module filename for *modname*.""" - source_info = app.emit_firstresult('viewcode-find-source', modname) - if source_info: - return None - else: - try: - filename, source = ModuleAnalyzer.get_module_source(modname) - return filename - except Exception: - return None - - -def should_generate_module_page(app: Sphinx, modname: str) -> bool: - """Check generation of module page is needed.""" - module_filename = get_module_filename(app, modname) - if module_filename is None: - # Always (re-)generate module page when module filename is not found. - return True - - builder = cast(StandaloneHTMLBuilder, app.builder) - basename = modname.replace('.', '/') + builder.out_suffix - page_filename = path.join(app.outdir, '_modules/', basename) - - try: - if path.getmtime(module_filename) <= path.getmtime(page_filename): - # generation is not needed if the HTML page is newer than module file. - return False - except IOError: - pass - - return True - - -def collect_pages(app: Sphinx) -> Generator[Tuple[str, Dict[str, Any], str], None, None]: - env = app.builder.env - if not hasattr(env, '_viewcode_modules'): - return - if not is_supported_builder(app.builder): - return - highlighter = app.builder.highlighter # type: ignore - urito = app.builder.get_relative_uri - - modnames = set(env._viewcode_modules) # type: ignore - - for modname, entry in status_iterator( - sorted(env._viewcode_modules.items()), # type: ignore - __('highlighting module code... '), "blue", - len(env._viewcode_modules), # type: ignore - app.verbosity, lambda x: x[0]): - if not entry: - continue - if not should_generate_module_page(app, modname): - continue - - code, tags, used, refname = entry - # construct a page name for the highlighted source - pagename = posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/')) - # highlight the source using the builder's highlighter - if env.config.highlight_language in ('python3', 'default', 'none'): - lexer = env.config.highlight_language - else: - lexer = 'python' - highlighted = highlighter.highlight_block(code, lexer, linenos=False) - # split the code into lines - lines = highlighted.splitlines() - # split off wrap markup from the first line of the actual code - before, after = lines[0].split('
')
-        lines[0:1] = [before + '
', after]
-        # nothing to do for the last line; it always starts with 
anyway - # now that we have code lines (starting at index 1), insert anchors for - # the collected tags (HACK: this only works if the tag boundaries are - # properly nested!) - maxindex = len(lines) - 1 - for name, docname in used.items(): - type, start, end = tags[name] - backlink = urito(pagename, docname) + '#' + refname + '.' + name - lines[start] = ( - '
%s' % (name, backlink, _('[文档]')) + - lines[start]) - lines[min(end, maxindex)] += '
' - # try to find parents (for submodules) - parents = [] - parent = modname - while '.' in parent: - parent = parent.rsplit('.', 1)[0] - if parent in modnames: - parents.append({ - 'link': urito(pagename, - posixpath.join(OUTPUT_DIRNAME, parent.replace('.', '/'))), - 'title': parent}) - parents.append({'link': urito(pagename, posixpath.join(OUTPUT_DIRNAME, 'index')), - 'title': _('Module code')}) - parents.reverse() - # putting it all together - context = { - 'parents': parents, - 'title': modname, - 'body': (_('

Source code for %s

') % modname + - '\n'.join(lines)), - } - yield (pagename, context, 'page.html') - - if not modnames: - return - - html = ['\n'] - # the stack logic is needed for using nested lists for submodules - stack = [''] - for modname in sorted(modnames): - if modname.startswith(stack[-1]): - stack.append(modname + '.') - html.append('
    ') - else: - stack.pop() - while not modname.startswith(stack[-1]): - stack.pop() - html.append('
') - stack.append(modname + '.') - html.append('
  • %s
  • \n' % ( - urito(posixpath.join(OUTPUT_DIRNAME, 'index'), - posixpath.join(OUTPUT_DIRNAME, modname.replace('.', '/'))), - modname)) - html.append('' * (len(stack) - 1)) - context = { - 'title': _('Overview: module code'), - 'body': (_('

    All modules for which code is available

    ') + - ''.join(html)), - } - - yield (posixpath.join(OUTPUT_DIRNAME, 'index'), context, 'page.html') - - -def setup(app: Sphinx) -> Dict[str, Any]: - app.add_config_value('viewcode_import', None, False) - app.add_config_value('viewcode_enable_epub', False, False) - app.add_config_value('viewcode_follow_imported_members', True, False) - app.connect('doctree-read', doctree_read) - app.connect('env-merge-info', env_merge_info) - app.connect('env-purge-doc', env_purge_doc) - app.connect('html-collect-pages', collect_pages) - app.connect('missing-reference', missing_reference) - # app.add_config_value('viewcode_include_modules', [], 'env') - # app.add_config_value('viewcode_exclude_modules', [], 'env') - app.add_event('viewcode-find-source') - app.add_event('viewcode-follow-imported') - app.add_post_transform(ViewcodeAnchorTransform) - return { - 'version': sphinx.__display_version__, - 'env_version': 1, - 'parallel_read_safe': True - } diff --git a/docs/_ext/rename_include.py b/docs/_ext/rename_include.py deleted file mode 100644 index bf7dea2..0000000 --- a/docs/_ext/rename_include.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Rename .rst file to .txt file for include directive.""" -import os -import re -import glob -import logging - -logging.basicConfig(level=logging.WARNING, format='%(message)s') -logger = logging.getLogger(__name__) - -origin = "rst" -replace = "txt" - -include_re = re.compile(r'\.\. include::\s+(.*?)(\.rst|\.txt)') -include_re_sub = re.compile(rf'(\.\. include::\s+(.*?))\.{origin}') - -# Specified file_name lists excluded from rename procedure. -whitepaper = ['operations.rst'] - -def repl(matchobj): - """Replace functions for matched.""" - if matchobj.group(2).split('/')[-1] + f'.{origin}' in whitepaper: - return matchobj.group(0) - return rf'{matchobj.group(1)}.{replace}' - -def rename_include(api_dir): - """ - Rename .rst file to .txt file for include directive. - - api_dir - api path relative. - """ - tar = [] - for root, _, files in os.walk(api_dir): - for file in files: - if not file.endswith('.rst'): - continue - try: - with open(os.path.join(root, file), 'r+', encoding='utf-8') as f: - content = f.read() - tar_ = include_re.findall(content) - if tar_: - tar_ = [i[0].split('/')[-1]+f'.{origin}' for i in tar_] - tar.extend(tar_) - sub = include_re_sub.findall(content) - if sub: - content_ = include_re_sub.sub(repl, content) - f.seek(0) - f.truncate() - f.write(content_) - except UnicodeDecodeError: - # pylint: disable=logging-fstring-interpolation - logger.warning(f"UnicodeDecodeError for: {file}") - - all_rst = glob.glob(f'{api_dir}/**/*.{origin}', recursive=True) - - for i in all_rst: - if os.path.dirname(i).endswith("api_python") or os.path.basename(i) in whitepaper: - continue - name = os.path.basename(i) - if name in tar: - os.rename(i, i.replace(f'.{origin}', f'.{replace}')) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 1a0d764..8c6d43c 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,35 +1,16 @@ -/* 移除导航栏固定宽度限制 */ -.wy-nav-side { - width: auto !important; - min-width: 250px; /* 设置最小宽度 */ - max-width: 350px; /* 设置最大宽度防止过宽 */ - } - - /* 让导航菜单根据内容自适应 */ - .wy-menu-vertical { - width: auto !important; - white-space: nowrap; /* 初始不换行 */ - } - - /* 一级标题样式 */ - .wy-menu-vertical li.toctree-l1 > a { - white-space: nowrap; - padding-right: 20px; /* 为可能的滚动条留空间 */ - } - - /* 子标题允许换行 */ - .wy-menu-vertical li.toctree-l2 > a, - .wy-menu-vertical li.toctree-l3 > a { - white-space: normal; - word-wrap: break-word; - } - - /* 添加平滑过渡效果 */ - .wy-menu-vertical { - transition: width 0.3s ease; - } - - /* 鼠标悬停时临时展开 */ - .wy-nav-side:hover { - max-width: 500px; /* 悬停时可更宽 */ - } \ No newline at end of file +html { + --pst-font-family-base: 'Microsoft Yahei', 'Inter', sans-serif; +} + +.bd-page-width { + max-width: 100%; + min-width: 100%; +} + +.bd-content .sd-tab-set>input:checked+label { + border-color: #d3d3d3 #d3d3d3 transparent; +} + +.bd-content .sd-tab-set .sd-tab-content { + border-color: #d3d3d3; +} diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst new file mode 100644 index 0000000..aab2902 --- /dev/null +++ b/docs/_templates/autosummary/class.rst @@ -0,0 +1,44 @@ +{# + It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) + that autosummary will generate warning for inherited instance attributes. + Those warnings will fail our build. + For now, we don't autosummary classes with inherited instance attributes. + To opt out, use `:template: autosummary/class_without_autosummary.rst` +#} + +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :show-inheritance: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :nosignatures: + :toctree: + + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + :toctree: + + {% for item in methods %} + ~{{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 70c89b2..079c1e0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,12 @@ -sphinx>=5.3.0 -ipython -sphinx_rtd_theme -myst-parser -sphinx-design +sphinx>=7.3.7 breathe -exhale \ No newline at end of file +sphinx-book-theme +sphinx-autodoc2 +myst-parser +pyyaml +sphinx_rtd_theme +sphinx-autobuild +sphinx-copybutton +sphinx_design +sphinxcontrib-openapi +sphinx_togglebutton \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_cpp/common.md b/docs/source_zh_cn/api/api_cpp/common.md deleted file mode 100644 index 17e8f02..0000000 --- a/docs/source_zh_cn/api/api_cpp/common.md +++ /dev/null @@ -1,356 +0,0 @@ -# Common - -[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.5.0/resource/_static/logo_source.svg)](../../../../include/datasystem/utils) - -## 接口汇总 - -| 类名 | 说明 | -| --- | --- | -| [Context](#context) | 保存执行中的上下文配置。| -| [ConnectOptions](#connectoptions) | 配置对象客户端的初始化参数类。| -| [Optional](#optional) | 可将任意类型视为 null 的类。| -| [Status](#status) | 返回状态类。| -| [StringView](#stringview) | 免拷贝字符串类。| - -## Context - -\#include <[datasystem/context/context.h](../../../../include/datasystem/context/context.h)> - -Context 类用于保存执行中的上下文配置。 - -### 构造函数和析构函数 - -```cpp -Context() -~Context() = default; -``` - -### 公有成员函数 - -| 函数 | 说明 | -|-------------------------------------------------------------------------------|--------| -| [`static Status SetTraceId(const std::string &traceId)`](#settraceid) | 设置 traceID,可用于请求链路跟踪。 | -| [`static void SetTenantId(const std::string &tenantId)`](#settenantid) | 设置运行时的租户 ID。 | - -#### SetTraceId - -```cpp -static Status SetTraceId(const std::string &traceId) -``` - -设置 traceID,可用于请求链路跟踪。 - -- 参数 - - - `traceId`: 用户描述跟踪链路 ID。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 - -#### SetTenantId - -```cpp -static void SetTenantId(const std::string &tenantId) -``` - -设置运行时的租户 ID。 - -- 参数 - - - `tenantId`: 运行时的租户 ID。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 - -## ConnectOptions - -\#include <[datasystem/utils/connection.h](../../../../include/datasystem/utils/connection.h)> - -ConnectOptions 类用于配置对象客户端的初始化参数。 - -```cpp -struct ConnectOptions { - std::string host; - int32_t port; - int32_t connectTimeoutMs = 60 * 1000; // 60s - std::string clientPublicKey = ""; - SensitiveValue clientPrivateKey = ""; - std::string serverPublicKey = ""; - std::string accessKey = ""; - SensitiveValue secretKey = ""; - std::string tenantId = ""; - bool enableCrossNodeConnection = false; -} -``` - -参数具体说明如下: -|参数| 说明 | -|----|--------------------------------------| -|host | 数据系统 Worker 的主机 IP 地址。 | -|port | 数据系统 Worker 的主机 IP 端口号。 | -|connectTimeoutMs | 客户端连接和请求超时时间,单位为毫秒。默认值:60'000 | -|clientPublicKey | 用于 curve 认证的客户端公钥。默认值:"" | -|clientPrivateKey | 用于 curve 认证的客户端私钥。默认值:"" | -|serverPublicKey | 用于 curve 认证的服务端公钥。默认值:"" | -|accessKey | AK/SK 授权使用的访问密钥。默认值:"" | -|secretKey | AK/SK 授权的密钥。默认值:"" | -|tenantId | 租户 ID。默认值:"" | -|enableCrossNodeConnection | 如果为 `true`,允许客户端在与当前数据系统Worker 连接异常时自动切换到备用节点。默认值:`false` | - -## Optional - -\#include <[datasystem/object_cache/utils/optional.h](../../../../include/datasystem/utils/optional.h)> - -用于表示一个值可能存在,当值不存在时表示为 null。在 [Get](./object_cache.md#get) 接口的传出参数通过 `Optional` 表示对象是否获取成功。 - -### 构造函数和析构函数 - -```cpp -constexpr Optional() -Optional(const Optional &) -Optional &operator=(const Optional &) -Optional(Optional &&other) noexcept -Optional &operator=(Optional &&) noexcept -template explicit Optional(Args &&... args); -~Optional() -``` - -## Status - -\#include <[datasystem/object_cache/utils/status.h](../../../../include/datasystem/utils/status.h)> - -Status 类用于表示请求/方法的执行结果。 - -### 构造函数和析构函数 - -```cpp -Status() noexcept -Status(const Status &other) -Status(Status &&other) noexcept -Status(StatusCode code, std::string msg) -Status(StatusCode code, int lineOfCode, const std::string &fileName, const std::string &extra = "") -~Status() noexcept -``` - -### 公有成员函数 - -| 函数 | 说明 | -|-------------------------------------------------------------------------------|--------| -| [`static Status OK()`](#ok) | 返回状态码为 `K_OK` 的 Status。 | -| [`std::string ToString() const`](#tostring) | 返回 Status 的状态码和报错信息。 | -| [`StatusCode GetCode() const`](#getcode) | 返回 Status 的状态码。 | -| [`std::string GetMsg() const`](#getmsg) | 返回 Status 的报错信息。 | -| [`void AppendMsg(const std::string &appendMsg)`](#appendmsg) | 拼接 Status 的报错信息。 | -| [`explicit operator bool() const`](#operator_bool) | 将 Status 对象自动转换为`bool`值 | -| [`bool operator==(const Status &other) const`](#operator_equal) | 重载 Status `==` 运算符 | -| [`bool operator!=(const Status &other) const`](#operator_not_equal) | 重载 Status `!=` 运算符 | -| [`bool IsOk() const`](#isok) | 判断 Status 的状态码是否为 `K_OK`。 | -| [`bool IsError() const`](#iserror) | 判断 Status 的状态码是否为非 `K_OK` 错误码。 | -| [`static std::string StatusCodeName(StatusCode code)`](#statuscodename) | 获取状态码的字符串表示,多用于打印需求。 | - -#### OK - -```cpp -static Status OK() -``` - -返回状态码为 `K_OK` 的 Status。 - -- 返回值 - - 状态码为 `K_OK` 的 Status。 - -#### ToString - -```cpp -std::string ToString() const -``` - -返回 Status 的状态码和报错信息。 - -- 返回值 - - Status 的状态码和报错信息。 - -#### GetCode - -```cpp -StatusCode GetCode() const -``` - -返回 Status 的状态码。 - -- 返回值 - - Status 的状态码。 - -#### GetMsg - -```cpp -std::string GetMsg() const -``` - -返回 Status 的报错信息。 - -- 返回值 - - Status 的报错信息。 - -#### AppendMsg - -```cpp -void AppendMsg(const std::string &appendMsg) -``` - -拼接 Status 的报错信息。 - -- 参数 - - - `appendMsg`: 需要拼接的报错信息。 - - -#### operator bool - -```cpp -explicit operator bool() const -``` - -将 Status 对象自动转换为`bool`值,在 if、while、for 或逻辑运算(&&、||、!)中,对象可以像 `bool` 一样被检查。 - -#### operator== - - -```cpp -bool operator==(const Status &other) const -``` - -重载 Status `==` 运算符,用于比较两个 Status 对象是否相等。 - -- 参数 - - - `other`: 待比较的 Status 对象。 - -- 返回值 - - `true`表示两个 Status 对象相等。 - -#### operator!= - - -```cpp -bool operator!=(const Status &other) const -``` - -重载 Status `!=` 运算符,用于比较两个 Status 对象是否不相等。 - -- 参数 - - - `other`: 待比较的 Status 对象。 - -- 返回值 - - `true`表示两个 Status 对象不相等。 - -#### IsOK - -```cpp -bool IsOk() const -``` - -判断 Status 的状态码是否为 `K_OK`。 - -- 返回值 - - `true`表示 Status 的状态码为 `K_OK`。 - -#### IsError - -```cpp -bool IsError() const -``` - -判断 Status 的状态码是否为非 `K_OK` 错误码。 - -- 返回值 - - `true`表示 Status 的状态码为非 `K_OK 错误码`。 - -#### StatusCodeName - -```cpp -static std::string StatusCodeName(StatusCode code) -``` - -获取状态码的字符串表示,多用于打印需求。 - -- 参数 - - - `code`: 状态码。 - -- 返回值 - - 状态码的字符串表示。 - - -## StringView - -\#include <[datasystem/object_cache/utils/string_view.h](../../../../include/datasystem/utils/string_view.h)> - -字符串视图类,主要用于高效地传递和访问字符串数据,而无需复制或分配内存。 - -### 构造函数和析构函数 - -```cpp -constexpr StringView() noexcept -constexpr StringView(const StringView &) noexcept -constexpr StringView(const char *str) -constexpr StringView(const char *str, size_t len) -StringView(const std::string &str) -constexpr StringView &operator=(const StringView &) noexcept -~StringView() -``` - -### 公有成员函数 -| 函数 | 说明 | -|-------------------------------------------------------------------------------|--------| -| [`constexpr const char *data() const noexcept`](#data) | 获取 StringView 的数据指针。 | -| [`constexpr size_t size() const noexcept`](#size) | 获取 StringView 的数据大小。 | -| [`constexpr bool empty() const noexcept`](#empty) | 判断 StringView 是否为空。 | - -#### data - -```cpp -constexpr const char *data() const noexcept -``` - -获取 StringView 的数据指针。 - -- 返回值 - - StringView 的数据指针。 - -#### size - -```cpp -constexpr size_t size() const noexcept -``` - -获取 StringView 的数据大小。 - -- 返回值 - - StringView 的数据大小。 - -#### empty - -```cpp -constexpr size_t empty() const noexcept -``` - -判断 StringView 是否为空。 - -- 返回值 - - `true` 表示 StringView 为空。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_cpp/dsclient.md b/docs/source_zh_cn/api/api_cpp/dsclient.md deleted file mode 100644 index 8071069..0000000 --- a/docs/source_zh_cn/api/api_cpp/dsclient.md +++ /dev/null @@ -1,85 +0,0 @@ -# DsClient - -[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.5.0/resource/_static/logo_source.svg)](../../../../include/datasystem/datasystem.h) - -## DsClient - -\#include <[datasystem/datasystem.h](../../../../include/datasystem/datasystem.h)> - -DsClient 为yuanrong-datasystem的客户端总入口,其中包含了异构对象、KV和object多类客户端。 - -### 构造函数和析构函数 - -```cpp -explicit DsClient(const ConnectOptions &connectOptions = {}) -~DsClient() -``` - -### 公有成员函数 -| 函数 | 说明 | -|------------------------------------|--------| -| [`Status Init()`](#init) | 初始化客户端。 | -| [`Status ShutDown()`](#shutdown) | 关闭客户端。 | -| [`std::shared_ptr KV()`](#kv) | 获取 KV 客户端。 | -| [`std::shared_ptr Hetero()`](#hetero) | 获取异构对象客户端。 | -| [`std::shared_ptr Object()`](#object) | 获取 object 客户端。 | - -#### Init - -```cpp -Status Init(); -``` - -初始化客户端,建立与数据系统 Worker 之间的连接。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 - -#### ShutDown - -```cpp -Status ShutDown(); -``` - -关闭客户端,断开与数据系统 Worker 之间的连接。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示关闭成功,否则返回其他错误码。 - -#### KV - -```cpp -std::shared_ptr KV(); -``` - -获取 KV 客户端。 - -- 返回值 - - - std::shared_ptr\,返回 KVClient 客户端指针。 - -#### Hetero - -```cpp -std::shared_ptr Hetero(); -``` - -获取异构对象客户端。 - -- 返回值 - - - std::shared_ptr\,返回异构对象客户端指针。 - -#### Object - -```cpp -std::shared_ptr Object(); -``` - -获取 object 客户端。 - -- 返回值 - - - std::shared_ptr\,返回 object 客户端指针。 diff --git a/docs/source_zh_cn/api/api_cpp/hetero_cache.md b/docs/source_zh_cn/api/api_cpp/hetero_cache.md deleted file mode 100644 index 5d66b62..0000000 --- a/docs/source_zh_cn/api/api_cpp/hetero_cache.md +++ /dev/null @@ -1,402 +0,0 @@ -# HeteroClient - -[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.5.0/resource/_static/logo_source.svg)](../../../../include/datasystem/hetero_cache/hetero_client.h) - -## 接口汇总 -| 类名 | 说明 | -| --- | --- | -| [Blob](#blob) | 描述一段 device 上的内存。| -| [DeviceBlobList](#devicebloblist) | 描述 device 上的一组内存。| -| [HeteroClient](#heteroclient) | 异构对象客户端类。| -| [Future](#future) | 用于接收异步操作结果。| - -## Blob - -\#include <[datasystem/hetero_cache/hetero_client.h](../../../../include/datasystem/hetero_cache/hetero_client.h)> - -Blob 用于描述一段 device 上的内存,当前定义如下: - -```cpp -struct Blob { - void *pointer = 0; - uint64_t size = 0; -}; -``` - -pointer 表示 device 上的指针地址, size 表示指针指向的内存大小。 - -## DeviceBlobList - -\#include <[datasystem/hetero_cache/hetero_client.h](../../../../include/datasystem/hetero_cache/hetero_client.h)> - -DeviceBlobList 用于描述 device 上的一组内存,当前定义如下: - -```cpp -struct DeviceBlobList { - std::vector blobs; - int32_t deviceIdx = -1; -}; -``` -blobs 用于存放多个 [Blob](#blob), deviceIdx 指示 device 内存归属的 NPU 卡的 id。 - -## AsyncResult - -\#include <[datasystem/hetero_cache/hetero_client.h](../../../../include/datasystem/hetero_cache/hetero_client.h)> - -AsyncResult 用于描述异步请求的调用结果,当前定义如下: - -```cpp -struct AsyncResult { - Status status; - std::vector failedList; -}; -``` -status 用于表示异步执行的结果, failedList 用于表示失败的对象列表。 - - -## MetaInfo -\#include <[datasystem/hetero_cache/hetero_client.h](../../../../include/datasystem/hetero_cache/hetero_client.h)> - -用于描述存入key的元信息, 当前的定义如下: -```cpp -struct MetaInfo { - std::vector blobSizeList; -}; -``` -其中 blobSizeList 表示key对应原始存入数据的device内存块大小, 单位(字节) - - -## HeteroClient - -\#include <[datasystem/hetero_cache/hetero_client.h](../../../../include/datasystem/hetero_cache/hetero_client.h)> - -HeteroClient 为异构对象客户端类。 - -### 构造函数和析构函数 - -```cpp -explicit HeteroClient(const ConnectOptions &connectOptions = {}) -~HeteroClient() -``` - -### 公有成员函数 -| 函数 | 说明 | -|--------------------------------------------------------------------------------------------|--------| -| [`Status ShutDown()`](#shutdown) | 关闭异构对象客户端。 | -| [`Status Init()`](#init) | 初始化异构对象客户端。 | -| [`Status MGetH2D(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys, int32_t subTimeoutMs)`](#mgeth2d) | 从 host 中获取数据写入 device。 | -| [`Status MSetD2H(const std::vector &keys, const std::vector &devBlobList, const SetParam &setParam = {});`](#msetd2h) | 将 device 的数据写入到 host。 | -| [`Status Delete(const std::vector &keys, std::vector &failedKeys)`](#delete) | 删除 host中的 key。 | -| [`Status GenerateKey(const std::string &prefix, std::string &key)`](#generatekey) | 生成带数据系统 Worker UUID 的 key。 | -| [`Status DevPublish(const std::vector &keys, const std::vector &devBlobList, std::vector &futureVec)`](#devpublish) | 将 device 上的内存发布为数据系统的异构对象。 | -| [`Status DevSubscribe(const std::vector &keys, const std::vector &devBlobList, std::vector &futureVec)`](#devsubscribe) | 订阅并接收 device 上的数据对象。 | -| [`Status DevMGet(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys, int32_t subTimeoutMs = 0)`](#devmget) | 获取 device 上的对象。 | -| [`Status DevMSet(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys)`](#devmset) | 将 device 对象写入数据系统。 | -| [`std::shared_future AsyncMSetD2H(const std::vector &keys, const std::vector &devBlobList, const SetParam &setParam = {});`](#AsyncMSetD2H) | 异步接口,将 device 对象写入数据系统。 | -| [`std::shared_future AsyncMGetH2D(const std::vector &keys, const std::vector &devBlobList, uint64_t subTimeoutMs)`](#AsyncMGetH2D) | 异步接口,获取 device 上的对象。 | -| [`Status DevLocalDelete(const std::vector &keys, std::vector &failedKeys)`](#devlocaldelete) | 删除本地 device 对象。 | -| [`std::shared_future AsyncDevDelete(const std::vector &keys)`](#asyncdevdelete) | 异步删除 device 对象。 | -| [`Status DevDelete(const std::vector &keys, std::vector &failedKeys)`](#devdelete) | 删除 device 对象。 | -| [`Status GetMetaInfo(const std::vector &keys, const bool isDevKey, std::vector &metaInfos, std::vector &failKeys)`](#getmetainfo) | 获取key对应的元数据信息 | -#### ShutDown - -```cpp -Status ShutDown() -``` - -关闭异构对象客户端,断开与数据系统 Worker 之间的连接。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示关闭成功,否则返回其他错误码。 - -#### Init - -```cpp -Status Init() -``` - -初始化异构对象客户端,建立与数据系统 Worker 之间的连接。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 - -#### MGetH2D - -```cpp -Status MGetH2D(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys, int32_t subTimeoutMs); -``` - -从 host 中获取数据并写入 device 中。 -MGetH2D 和 MSetD2H 需配套使用。 -若 MSetD2H 时将多个内存地址拼接写入了 host,则在 MGetH2D 中自动将 host 的数据拆分成多个内存地址写入 device。 -若 host 的 key 不再使用,可调用 Delete 接口删除。 - -- 参数 - - - `keys`: host 的 key 列表。约束:传入的 key 的数量不能超过 1 万。 - - `devBlobList`: 用于描述 device 上的 HBM 内存地址,用于接收从 host 中读取的数据。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `failedKeys`: 失败的 key 列表。 - - `subTimeoutMs`: 超时时间,当在指定时间内无法获取完成,则返回异常。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示执行成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### MSetD2H - -```cpp -Status MSetD2H(const std::vector &keys, const std::vector &devBlobList, const SetParam &setParam = {}); -``` - -将 device 的数据写入到 host 中。若 device 的 blob 中存在多个内存地址时,会自动将数据拼接后写入 host。 -若 host 的 key 不再使用,可调用 Delete 接口删除。 -MSetD2H 成功的数据是immutable的,不要对已存在的key set新值,否则新值不会生效。 - -- 参数 - - - `keys`: host 的 key 列表。约束:传入的key的数量不能超过1万。 - - `devBlobList`: 用于描述 device 上的 HBM 内存地址。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `setParam`: key的配置参数, 默认值为: - ``` - struct SetParam { - WriteMode writeMode = WriteMode::NONE_L2_CACHE; - uint32_t ttlSecond = 0; - ExistenceOpt existence = ExistenceOpt::NONE; - CacheType cacheType = CacheType::MEMORY; - }; -- 返回值 - - - 返回值状态码为 `K_OK` 时表示执行成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### Delete - -```cpp -Status Delete(const std::vector &keys, std::vector &failedKeys); -``` - -删除 host 中的 key。 -delete 接口与 MGetH2D / MSetD2H 配套使用。 - -- 参数 - - - `keys`: 待删除的 key 列表。约束:传入的 key 的数量不能超过 1 万。 - - `failedKeys`: 删除失败的 key 列表。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示删除成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### GenerateKey - -```cpp -Status GenerateKey(const std::string &prefix, std::string &key); -``` - -生成带数据系统 Worker UUID 的 key。 - -- 参数 - - - `prefix`: key 前缀。 - - `key`: 传出参数,生成成功时会将 key 写入到该参数中。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示 key 生成成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### DevPublish - -```cpp -Status DevPublish(const std::vector &keys, const std::vector &devBlobList, std::vector &futureVec); -``` - -将 device 上的内存发布为数据系统的异构对象。发布后的异构对象可通过 DevSubscribe 获取。 -DevPublish 和 DevSubscribe 需配套使用。 DevPublish 和 DevSubscribe 传入的 Device 内存地址不能归属于同一张 NPU 卡。 -通过 DevSubscribe 获取数据成功后,数据系统会自动删除此异构对象,不再管理此对象对应的 device 内存。 -在key,devBlobList内存地址映射关系均一致的情况下,DevPublish在同进程支持重试。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的key的数量不能超过1万。 - - `devBlobList`: 用于描述 device 上内存结构列表。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `futureVec`: 用于异步获取发布结果。若 future 返回 OK,则表示数据已被对端接收。详见 [Future](#future) 章节。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示数据发布成功。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### DevSubscribe - -```cpp -Status DevSubscribe(const std::vector &keys, const std::vector &devBlobList, std::vector &futureVec); -``` - -订阅发布到数据系统异构对象,并接收数据写入 devBlobList 。 -数据通过 device to device 通道直接传输。 -DevPublish 和 DevSubscribe 需配套使用。 DevPublish 和 DevSubscribe 传入的 Device 内存地址不能归属于同一张 NPU 卡。 -通过 DevSubscribe 获取数据成功后,数据系统会自动删除此异构对象,不再管理此对象对应的 device 内存。 -在执行 DevSubscribe 过程中,执行了 DevPublish 的进程不能退出,否则 DevSubscribe 会失败。 -DevSubscribe单Key的订阅超时时间为20s,多key为60s。 - -- 参数 - - - `keys`: device 的异构对象的 key。 - - `devBlobList`: 用于描述 device 上内存结构列表,用于接收数据。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `futureVec`: 用于异步获取订阅结果。若 future 返回 OK,则表示数据已接收成功。详见 [Future](#future) 章节。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示对象订阅成功。约束:传入的key的数量不能超过1万。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### DevMGet - -```cpp -Status DevMGet(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys, int32_t subTimeoutMs = 0); -``` - -获取 device 中的数据,并写入到 devBlobList 中。数据通过 device to device 通道直接传输。 -DevMSet 和 DevMGet 需配套使用。 DevMSet 和 DevMGet 传入的 Device 内存地址不能归属于同一张 NPU 卡。 -DevMGet 后不会自动删除异构对象,如对象不再使用,可调用 DevLocalDelete 或 DevDelete 删除。 -在执行 DevMGet 过程中,执行了 DevMSet 的进程不能退出,否则 DevMGet 会失败。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的key的数量不能超过1万。 - - `devBlobList`: 用于描述 device 上内存结构列表。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `failedKeys`: 输出参数,用于描述失败的 key 的列表。 - - `subTimeoutMs`: 超时时间,当在指定时间内无法获取完成,则抛出异常。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象获取成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - - 返回值状态码为 `K_NOT_FOUND` 时表示所有对象都无法获取到。 - - 返回值状态码为 `K_RUNTIME_ERROR` 时表示获取对象遇到了错误。 - -#### DevMSet - -```cpp -Status DevMSet(const std::vector &keys, const std::vector &devBlobList, std::vector &failedKeys); -``` - -通过数据系统缓存 Device 上的数据,将 devBlobList 对应的 key 的元数据写入数据系统,可供其他 client 访问。 -DevMSet 和 DevMGet 需配套使用。 DevMSet 和 DevMGet 传入的 Device 内存地址不能归属于同一张 NPU 卡。 -DevMGet 后不会自动删除异构对象,如对象不再使用,可调用 DevLocalDelete 或 DevDelete 删除。 -在key,devBlobList内存地址映射关系均一致的情况下,DevMSet在同进程支持重试。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的key的数量不能超过1万。 - - `devBlobList`: 用于描述 device 上内存结构列表。详见 [DeviceBlobList](#devicebloblist) 章节。 - - `failedKeys`: 输出参数,用于描述失败的key的列表。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象DevMSet成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### DevLocalDelete - -```cpp -Status DevLocalDelete(const std::vector &keys, std::vector &failedKeys); -``` - -从数据系统删除本节点上此 key 的元数据,不再管理此 key 对应的 device 内存。 -DevLocalDelete 与 DevMSet / DevMGet 接口配套使用。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的key的数量不能超过1万。 - - `failedKeys`: 输出参数,用于描述失败的 key 的列表。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象DevLocalDelete成功,否则返回其他错误码。 - -#### DevDelete - -```cpp -Status DevDelete(const std::vector &keys, std::vector &failedKeys); -``` - -从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 -DevDelete 与 DevMSet / DevMGet 接口配套使用。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的 key 的数量不能超过 1 万。 - - `failedKeys`: 输出参数,用于描述失败的 key 的列表。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象DevDelete成功,否则返回其他错误码。 - -#### AsyncDevDelete - -```cpp -std::shared_future AsyncDevDelete(const std::vector &keys); -``` - -异步接口,从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 -AsyncDevDelete 与 DevMSet / DevMGet 接口配套使用。 - -- 参数 - - - `keys`: device 的异构对象的 key。约束:传入的 key 的数量不能超过 1 万。 - -- 返回值 - - 通过返回的future查询异步删除请求的调用结果 - -#### GetMetaInfo -```cpp -Status GetMetaInfo(const std::vector &keys, const bool isDevKey, std::vector &metaInfos, std::vector &failKeys) -``` -获取key对应的元数据信息 - -- 参数 - - - `keys`: 输入参数,device 的异构对象的 key。约束:传入的 key 的数量不能超过 1 万。 - - `isDevKey`: 输入参数,key的属性,true 表示D2D类型,false表示D2H类型。 - - `metaInfos`: 输出参数,用于描述key的元信息。 - - `failKeys`: 输出参数,用于描述失败列表。 -- 返回值 - - 返回值状态码为 `K_OK` 时表示至少有一个对象GetMetaInfo成功,否则返回其他错误码。 - - -## Future - -\#include <[datasystem/hetero_cache/future.h](../../../../include/datasystem/hetero_cache/future.h)> - -Future 用于接收异步操作的结果。 - -### 公有成员函数 - -| 函数 | 说明 | -|-------------------------------------------------------------------------------|--------| -| [`Status Get(uint64_t subTimeoutMs = 60000)`](#get) | 获取异步任务的执行结果。 | - -#### Get - -```cpp -Status Get(uint64_t subTimeoutMs = 60000) -``` - -获取异步任务的执行结果。 - -- 参数 - - - `subTimeoutMs`: 描述等待时长。如果 subTimeoutMs 大于0, 则阻塞直到超时或结果变为可用;如果 subTimeoutMs 等于0,则立即返回结果状态。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示异步操作执行成功,否则返回其他错误码。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_cpp/kv_cache.md b/docs/source_zh_cn/api/api_cpp/kv_cache.md deleted file mode 100644 index ae1413b..0000000 --- a/docs/source_zh_cn/api/api_cpp/kv_cache.md +++ /dev/null @@ -1,391 +0,0 @@ -# KVClient - -[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.5.0/resource/_static/logo_source.svg)](../../../../include/datasystem/kv_cache.h) - -## 接口汇总 - -| 类名 | 说明 | -|--------------------------------|---------------------------------------------| -|[`class KVClient`](#class-kvclient) | KV缓存客户端类| -|[`enum ExistenceOpt`](#enum-existenceopt) |该选项用于配置对象存在时是否允许继续操作 | -|[`class ReadOnlyBuffer`](#class-readonlybuffer) | 只读缓冲区结构类 | -|[`struct MSetParam`](#struct-msetparam) | 同时创建多个Key时设置的属性 | -|[`struct ReadParam`](#struct-readparam) | 偏移读取Key时设置的属性 | -|[`struct SetParam`](#struct-setparam) | 创建Key时设置的属性 | - -## class KVClient - -\#include <[datasystem/kv_cache/kv_client.h](../../../../include/datasystem/kv_cache/kv_client.h)> - -KV缓存对象类 - -### 构造函数和析构函数 -#### explicit KVClient (const ConnectOptions &connectOptions) - -构造[KVClient](#class-kvclient). - -- 参数 - - `connectOptions` 配置与 Worker 的连接选项,包括IP地址和端口,详见[ConnectOptions](common.md#connectoptions)章节 - - -#### ~KVClient () - -析构[KVClient](#class-kvclient). - -### 公有成员函数 - - 函数 | 说明 ---------------------------------|--------------------------------------------- -[`Status Init ()`](#status-init-) | 建立[KVClient](#class-kvclient)与当前数据系统 Worker 之间的连接并完成初始化。 -[`Status ShutDown ()`](#status-shutdown-) | 断开[KVClient](#class-kvclient)与当前数据系统 Worker 之间的连接。 -[`Status Set (const std::string &key, const StringView &val, const SetParam ¶m)`](#set-by-key) | 设置键值对数据。 -[`std::string Set (const StringView &val, const SetParam ¶m)`](#set-by-value) | 设置键值对数据,用户传入值,由接口自动生成键并返回。 -[`Status MSetTx (const std::vector &keys, const std::vector &vals, const MSetParam ¶m)`](#status-msettx) | 事务性创建多键值对接口。 -[`Status MSet (const std::vector &keys, const std::vector &vals, std::vector &outFailedKeys, const MSetParam ¶m)`](#status-mset) | 键值对批量设置接口。 -[`Status Get (const std::string &key, std::string &val, int32_t subTimeoutMs)`](#status-get) | 获取单个数据。 -[`Status Get (const std::string &key, Optional &readOnlyBuffer, int32_t subTimeoutMs)`](#get-readonlybuffer) | 获取单个数据的指针,获取后不可修改。 -[`Status Get (const std::vector &keys, std::vector &vals, int32_t subTimeoutMs)`](#batchget) | 批量获取数据。 -[`Status Get (const std::vector &keys, std::vector &readOnlyBuffers, int32_t subTimeoutMs)`](#batchget-readonlybuffer) | 批量获取多个数据的指针,获取后不可修改。 -[`Status Read (const std::vector &readParams, std::vector> &readOnlyBuffers)`](#status-read) | 偏移读接口,支持获取指定键值的其中一部分。 -[`Status Del (const std::string &key)`](#del) | 删除指定键值对。 -[`Status Del (const std::vector &keys, std::vector &failedKeys)`](#batchdel) | 批量删除指定键值对。 -[`Status GenerateKey (const std::string &prefixKey, std::string &key)`](#generatekey) | 生成具有指定前缀的带workerId的key,以供Set接口使用。 -[`Status HealthCheck ()`](#status-healthcheck-) | 检查连接的 Worker 是否健康。 -[`Status Expire (const std::vector &keys, uint32_t ttlSeconds, std::vector &failedKeys)`](#status-expire) | 为指定数据更新过期生命周期时间。 - -#### Status Init () - -建立[KVClient](#class-kvclient)与当前数据系统 Worker 之间的连接并完成初始化。 - -- 返回值 - 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 - -#### Status ShutDown () - -断开[KVClient](#class-kvclient)与当前数据系统 Worker 之间的连接。 - -- 返回值 - 返回值状态码为 `K_OK` 时表示断链成功,否则返回其他错误码。 - - -#### Status Set (const std::string &key, const StringView &val, const [SetParam](#struct-setparam) ¶m) - -设置键值对数据。 - -- 参数 - - `key` 键 - - `val` 值 - - `param` 设置参数,详见[SetParam](#struct-setparam)章节 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示键值对设置成功。 - - 返回值状态码为 `K_INVALID`时表示参数校验不通过 - - -#### std::string Set (const StringView &val, const [SetParam](#struct-setparam) ¶m) - -设置键值对数据,用户传入值,由接口自动生成键并返回。 - -- 参数 - - `val` 值 - - `param` 设置参数,详见[SetParam](#struct-setparam)章节 - -- 返回值 - - 成功时返回key - - 失败时返回空字符串 - - -#### Status MSetTx (const std::vector\ &keys, const std::vector\ &vals, const [MSetParam](#struct-msetparam) ¶m) - -事务性创建多键值对接口。支持事务性地创建一组键值对,键值对数量限制为1个到8个。如果有一个键已存在,则所有键值对都不会创建。 - - -- 参数 - - `keys` 需要设置的一组key. 约束:传入的key的数量不能超过8。 - - `vals` 需要设置的一组key对应的value. - - `param` 设置参数,详见[MSetParam](#struct-msetparam)章节. - -- 返回值 - 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 - - -#### Status MSet (const std::vector\ &keys, const std::vector\ &vals, std::vector\ &outFailedKeys, const [MSetParam](#struct-msetparam) ¶m) - -键值对批量设置接口。可批量设置键值对并返回失败的键。批量设置个数需小于2000,设置的每个值需小于500KB。 - -- 参数 - - `keys` 需要设置的一组key. 约束:传入的key的数量需要小于2千。 - - `vals` 需要设置的一组key对应的value. - - `outFailedKeys` 传出参数,代表设置失败的key. - - `param` 设置参数,详见[MSetParam](#struct-msetparam)章节. - -- 返回值 - 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 - - -#### Status Get (const std::string &key, std::string &val, int32_t subTimeoutMs) - -获取单个数据。 - -- 参数 - - `key` 键. - - `val` 传出参数,返回键对应的数据. - - `subTimeoutMs` 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待. - -- 返回值 - - 返回`K_OK`表示获取成功 - - 返回`K_INVALID`表示`key`校验不通过 - - 返回`K_NOT_FOUND`表示`key`不存在 - - 返回`K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - -#### Status Get (const std::string &key, Optional\<[ReadOnlyBuffer](#class-readonlybuffer)\> &readOnlyBuffer, int32_t subTimeoutMs) - -获取单个数据的指针,获取后不可修改。该接口相比[`Status Get (const std::string &key, std::string &val, int32_t timeoutMs`)](#get)少一次拷贝,性能更优。 - -- 参数 - - `key` 键. - - `subTimeoutMs` 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待. - - - `readOnlyBuffer` 传出参数,返回的使用[Optional](common.md#optional)封装的只读数据缓冲区. - -- 返回值 - - 返回`K_OK`表示获取成功 - - 返回`K_INVALID`表示`key`校验不通过 - - 返回`K_NOT_FOUND`表示`key`不存在 - - 返回`K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - -#### Status Get (const std::vector\ &keys, std::vector\ &vals, int32_t subTimeoutMs) - -批量获取数据。 - -- 参数 - - `keys` 需要获取的一组key. 约束:传入的key的数量不能超过1万。 - - `subTimeoutMs` 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待. - - `vals` 传出参数,返回一组获取的数据。若有部分数据获取不成功,则对应位置的vector的对象为空。 - -- 返回值 - - 返回`K_OK`表示至少有一个数据获取成功 - - 返回`K_INVALID`表示存在key校验不通过 - - 返回`K_NOT_FOUND`表示所有`keys`不存在 - - 返回`K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - -#### Status Get (const std::vector\ &keys, std::vector\\> &readOnlyBuffers, int32_t subTimeoutMs) - -批量获取多个数据的指针,获取后不可修改。该接口相比[`Status Get (const std::vector &keys, std::vector &vals, int32_t timeoutMs)`](#batchget)少一次拷贝,性能更优。 - -- 参数 - - `keys` 需要获取的一组key. 约束:传入的key的数量不能超过1万。 - - `subTimeoutMs` 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待. - - `readOnlyBuffers` 传出参数,返回的一组使用[Optional](common.md#optional)封装的只读数据缓冲区。若有部分数据获取不成功,则对应位置的vector的对象为空。 - -- 返回值 - - 返回`K_OK`表示至少有一个数据获取成功 - - 返回`K_INVALID`表示存在key校验不通过 - - 返回`K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_NOT_FOUND`表示所有`keys`不存在 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - -#### Status Read (const std::vector\<[ReadParam](#struct-readparam)> &readParams, std::vector\\> &readOnlyBuffers) - -偏移读接口,支持获取指定键值的其中一部分。在某些场景下,可避免读放大带来的性能损失。 - -- 参数 - - `readParams` 指定要查询的一个或多个键,详见[ReadParam](#struct-readparam)章节. 约束:传入的数量不能超过1万。 - - `readOnlyBuffers` 传出参数,返回的一组使用[Optional](common.md#optional)封装的只读数据缓冲区。若有部分数据获取不成功,则对应位置的vector的对象为空。 - -- 返回值 - - 返回`K_OK`表示至少有一个数据获取成功 - - 返回`K_INVALID`表示存在key校验不通过 - - 返回`K_NOT_FOUND`表示所有`keys`不存在 - - 返回`K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - - -#### Status Del (const std::string &key) - -删除指定键值对。key不存在时视为删除成功。 - -- 参数 - - `key` 键. - -- 返回值 - 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 - - -#### Status Del (const std::vector\ &keys, std::vector\ &failedKeys) - -批量删除指定键值对。key不存在时视为删除成功。 - -- 参数 - - `keys` 需要删除的一组key. 约束:传入的key的数量不能超过1万。 - - `failedKeys` 传出参数,返回删除失败的key. - -- 返回值 - - 返回`K_OK`表示至少有一个数据删除成功 - - 返回`K_INVALID`表示存在key校验不通过 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - -#### Status GenerateKey (const std::string &prefixKey, std::string &key) - -生成具有指定前缀的带workerId的key,以供Set接口使用。具有workerId的key具有元数据本节点亲和性,执行Set时性能更优。 - -- 参数 - - `prefixKey` 用户指定key的前缀. - - `key` 传出参数,返回生成的key. - -- 返回值 - 返回值状态码为 `K_OK` 时表示生成成功,否则返回其他错误码。 - -#### Status HealthCheck () - -检查连接的 Worker 是否健康。 - -- 返回值 - 返回值状态码为 `K_OK` 时表示 Worker 健康,否则返回其他错误码。 - - -#### Status KVClient::Exist(const std::vector<std::string> &keys, std::vector &exists) - -批量查询一组键(keys)是否存在,并返回每个键的存在性状态。支持最多10000个键的查询。 - -- 参数 - - `keys` 待查询的键列表,最大支持10000个键 - - `exists` 传出参数,返回每个键的存在性状态 - -- 返回值 - - 返回`K_OK`表示查询成功 - - 返回`K_INVALID`表示提供的键中包含非法字符或为空 - - 返回`K_RPC_UNAVAILABLE` 表示请求遇到了网络错误 - - 返回`K_NOT_READY` 表示服务当前无法处理请求 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - - - -#### Status Expire (const std::vector<std::string> &keys, uint32_t ttlSeconds, std::vector<std::string> &failedKeys) - -批量为一组键(keys)更新过期生命周期(ttlSeconds),并返回更新失败的键(failedKeys)。最多支持 10000 个键的查询。 - -- 参数 - - `keys` 待更新生命周期的键列表 - - `ttlSeconds` 为键设置的新的生命周期,单位为秒 - - `failedKeys` 传出参数,返回操作失败的键 - -- 返回值 - - 返回`K_OK`表示至少有一个键设置生命周期成功 - - 返回`K_INVALID`表示提供的键中包含非法字符或为空 - - 返回`K_NOT_FOUND`表示所有`keys`不存在 - - 返回`K_RPC_UNAVAILABLE` 表示请求遇到了网络错误 - - 返回`K_NOT_READY` 表示服务当前无法处理请求 - - 返回`K_RUNTIME_ERROR`表示 worker 侧存在错误 - -## enum ExistenceOpt - -\#include <[datasystem/kv_cache/kv_client.h](../../../../include/datasystem/kv_cache/kv_client.h)> - -用于配置对象存在时是否允许继续操作的枚举类。 - - 定义 | 说明 ---------------------------------|--------------------------------------------- -NONE | 允许Key存在时执行Set -NX | 不允许Key存在时执行Set - -## class ReadOnlyBuffer - -\#include <[datasystem/kv_cache/read_only_buffer.h](../../../../include/datasystem/kv_cache/read_only_buffer.h)> - -### 构造函数和析构函数 - -#### ReadOnlyBuffer () - -构造只读缓冲区结构类 - -#### ~ReadOnlyBuffer () - -析构只读缓冲区结构类 - -### 公用成员函数 - - 函数 | 说明 ---------------------------------|---------- -[`int64_t GetSize () const`](#int64_t-getsize--const) | 获取数据缓冲区的大小。 -[`const void* ImmutableData ()`](#const-void-immutabledata-) | 获取数据缓冲区的只读指针。 -[`Status RLatch (uint64_t timeout)`](#status-rlatch-uint64_t-timeout) | 对数据缓冲区加读锁。 -[`Status UnRLatch ()`](#status-unrlatch-) | 释放数据缓冲区的读锁。 - - -#### int64_t GetSize () const - -获取数据缓冲区的大小。 - -- 返回值 -数据缓冲区的大小,单位是Byte. - -#### const void* ImmutableData () - -获取数据缓冲区的只读指针。 - -- 返回值 -数据缓冲区的只读指针 - -#### Status RLatch (uint64_t timeout) - -对数据缓冲区加读锁,保护对应的内存不被并发写(允许并发读)。 -- 参数 - - `timeout` 加锁超时时间,单位是:秒,默认超时时间为60秒 - -- 返回值 - 返回值状态码为 `K_OK` 时表示加锁成功,否则返回其他错误码。 - -#### Status UnRLatch () - -释放数据缓冲区的读锁。 - -- 返回值 - 返回值状态码为 `K_OK` 时表示加锁成功,否则返回其他错误码。 - -## struct MSetParam - -\#include <[datasystem/kv_cache/kv_client.h](../../../../include/datasystem/kv_cache/kv_client.h)> - -同时创建多个Key时设置的属性。 - - 函数 | 说明 ---------------------------------|--------------------------------------------- -WriteMode writeMode | 设置数据可靠性级别,默认不写入二级缓存。详见[WriteMode](object_cache.md#writemode)章节 -uint32_t ttlSecond | 设置Key的存活时间,单位:秒。Key达到存活时间后系统自动将其删除。默认为0,表示不设置存活时间,Key会一直存在直到显式调用Del接口。注意:系统重启后存活时间计时将重新开始 -ExistenceOpt existence | 配置Key存在时是否允许继续操作,默认允许。详见[ExistenceOpt](#enum-existenceopt)章节 - -## struct ReadParam - -\#include <[datasystem/kv_cache/kv_client.h](../../../../include/datasystem/kv_cache/kv_client.h)> - -偏移读取Key时设置的属性。 - - 函数 | 说明 ---------------------------------|--------------------------------------------- -std::string key | 要读取的Key -uint64_t offset | 指定从Key对应值的特定偏移量开始读取 -uint64_t size | 从偏移量开始,要读取的大小 - -## struct SetParam - -\#include <[datasystem/kv_cache/kv_client.h](../../../../include/datasystem/kv_cache/kv_client.h)> - -创建Key时设置的属性。 - - 函数 | 说明 ---------------------------------|--------------------------------------------- -WriteMode writeMode | 设置数据可靠性级别,默认不写入二级缓存。详见[WriteMode](object_cache.md#writemode)章节 -uint32_t ttlSecond | 设置Key的存活时间,单位:秒。Key达到存活时间后系统自动将其删除。默认为0,表示不设置存活时间,Key会一直存在直到显式调用Del接口。注意:系统重启后存活时间计时将重新开始 -ExistenceOpt existence | 配置Key存在时是否允许继续操作,默认允许。详见[ExistenceOpt](#enum-existenceopt)章节 -CacheType | key 数据保存位置 详见[CacheType](object_cache.md#CacheType) diff --git a/docs/source_zh_cn/api/api_cpp/object_cache.md b/docs/source_zh_cn/api/api_cpp/object_cache.md deleted file mode 100644 index 04c6a51..0000000 --- a/docs/source_zh_cn/api/api_cpp/object_cache.md +++ /dev/null @@ -1,479 +0,0 @@ -# ObjectClient - -[![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.5.0/resource/_static/logo_source.svg)](../../../../include/datasystem/object_cache.h) - -## 接口汇总 - -| 类名 | 说明 | -| --- | --- | -| [ObjectClient](#objectclient) | 对象缓存客户端类。| -| [Buffer](#buffer) | 对象缓存数据类。| -| [CreateParam](#createparam) | 创建对象时的配置参数类。| -| [WriteMode](#writemode) | 配置对象可靠性的枚举类。| -| [ConsistencyType](#consistencytype) | 配置对象一致性的枚举类。| -| [CacheType](#cachetype) | 配置对象缓存类型。| - -## ObjectClient - -\#include <[datasystem/object_cache/object_client.h](../../../../include/datasystem/object_cache/object_client.h)> - -Buffer 类用于表示对象缓存数据类。 - -### 构造函数和析构函数 - -```cpp -explicit ObjectClient(const ConnectOptions &connectOptions = {}) -~ObjectClient() -``` - -### 公有成员函数 - -| 函数 | 说明 | -|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| -| [`Status Init()`](#init) | 初始化客户端。 | -| [`Status ShutDown()`](#shutdown) | 关闭客户端。 | -| [`Status GIncreaseRef(const std::vector &objectKeys, std::vector &failedObjectKeys)`](#gincreaseref) | 增加对象的全局引用计数。 | -| [`Status GDecreaseRef(const std::vector &objectKeys, std::vector &failedObjectKeys)`](#gdecreaseref) | 减少对象的全局引用计数。 | -| [`int QueryGlobalRefNum(const std::string &objectKey)`](#queryglobalrefnum) | 查询对象的全局引用计数。 | -| [`Status Create(const std::string &objectKey, uint64_t size, const CreateParam ¶m, std::shared_ptr &buffer)`](#create) | 创建对象 Buffer。 | -| [`Status Put(const std::string &objectKey, const uint8_t *data, uint64_t size, const CreateParam ¶m, const std::unordered_set &nestedObjectKeys = {})`](#put) | 创建或更新对象并发布。 | -| [`Status Get(const std::vector &objectKeys, int32_t subTimeoutMs, std::vector> &buffers)`](#get) | 获取对象。 | -| [`Status GenerateObjectKey(const std::string &prefix, std::string &objectKey)`](#generateobjectkey) | 生成带数据系统 Worker UUID 的对象 key。 | -| [`Status HealthCheck()`](#healthcheck) | 检查当前正在连接的数据系统 Worker 健康检查状况。 | - -#### ShutDown - -```cpp -Status ShutDown() -``` - -断开对象缓存客户端与当前数据系统 Worker 之间的连接。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示断链成功,否则返回其他错误码。 - -#### Init - -```cpp -Status Init() -``` - -建立对象缓存客户端与当前数据系统 Worker 之间的连接并完成初始化。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 - -#### GIncreaseRef - -```cpp -Status GIncreaseRef(const std::vector &objectKeys, std::vector &failedObjectKeys) -``` - -增加对象的全局引用计数。 - -- 参数 - - - `objectKeys`: 需要增加全局引用计数的对象 key 数组。约束:传入的object key的数量不能超过1万。 - - `failedObjectKeys`: 传出参数,增加全局引用计数失败的对象 key 会被保存在该字符串数组中。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象增加全局引用计数成功。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### GDecreaseRef - -```cpp -Status GDecreaseRef(const std::vector &objectKeys, std::vector &failedObjectKeys) -``` - -减少对象的全局引用计数。 - -- 参数 - - - `objectKeys`: 需要减少全局引用计数的对象 key 数组。约束:传入的object key的数量不能超过1万。 - - `failedObjectKeys`: 传出参数,减少全局引用计数失败的对象 key 会被保存在该字符串数组中。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象减少全局引用计数成功。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### QueryGlobalRefNum - -```cpp -int QueryGlobalRefNum(const std::string &objectKey) -``` - -查询对象的全局引用计数。 - -- 参数 - - - `objectKey`: 对象 key。 - -- 返回值 - - 返回值为大于等于 0 的数字时表示查询成功,返回值即为对象 key 的全局引用计数;返回 -1 时表示查询失败。 - -#### Create - -```cpp -Status Create(const std::string &objectKey, uint64_t size, const CreateParam ¶m, - std::shared_ptr &buffer) -``` - -创建对象 Buffer。 - -- 参数 - - - `objectKey`: 对象 key。 - - `size`: 对象大小(以字节为单位)。 - - `param`: 详见 [CreateParam](#createparam) 章节。 - - `buffer`: 传出参数,[Buffer](#buffer)。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示创建对象成功。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### Put - -```cpp -Status Put(const std::string &objectKey, const uint8_t *data, uint64_t size, const CreateParam ¶m, - const std::unordered_set &nestedObjectKeys = {}) -``` - -创建或更新对象并发布。 - -- 参数 - - - `objectKey`: 对象 key。 - - `data`: 数据内存地址指针。 - - `size`: 数据大小(以字节为单位)。 - - `param`: 详见 [CreateParam](#createparam) 章节。 - - `nestedObjectKeys`: 对该对象 key 有依赖关系的对象 key 数组。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示对象创建或更新并发布成功。 - - 返回值状态码为 `K_RPC_UNAVAILABLE` 时表示请求遇到了网络错误。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### Get - -```cpp -Status Get(const std::vector &objectKeys, int32_t subTimeoutMs, std::vector> &buffers) -``` - -获取对象。 - -- 参数 - - - `objectKeys`: 需要获取的对象 key 数组。约束:传入的object key的数量不能超过1万。 - - `subTimeoutMs`: 对象订阅时间(以毫秒为单位);对象未就绪时,产生订阅等待对象就绪返回,0 表示不订阅,可配置范围为[0, INT32_MAX]。 - - `buffers`: 对象 [Buffer](#buffer)。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示至少有一个对象获取成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - - 返回值状态码为 `K_NOT_FOUND` 时表示所有对象都无法获取到。 - - 返回值状态码为 `K_RUNTIME_ERROR` 时表示获取对象遇到了错误。 - -#### GenerateObjectKey - -```cpp -Status GenerateObjectKey(const std::string &prefix, std::string &objectKey) -``` - -生成带数据系统 Worker UUID 的对象 key。 - -- 参数 - - - `prefix`: 对象 key 前缀。 - - `objectKey`: 传出参数,生成成功时会将对象 key 写入到该参数中。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示对象 key 生成成功。 - - 返回值状态码为 `K_INVALID` 时表示传入参数未通过参数合法校验。 - -#### HealthCheck - -```cpp -Status HealthCheck() -``` - -检查当前正在连接的数据系统 Worker 健康检查状况。 - -- 返回值 - - - 返回值状态码为 `K_OK` 时表示 Worker 状态健康,否则返回其他错误码。 - -## Buffer - -\#include <[datasystem/object_cache/buffer.h](../../../../include/datasystem/object_cache/buffer.h)> - -Buffer 类用于表示对象缓存数据类。 - -### 构造函数和析构函数 - -```cpp -Buffer() -Buffer(Buffer &&other) noexcept -~Buffer() = default; -``` - -### 公有成员函数 - -| 函数 | 说明 | -|-------------------------------------------------------------------------------|--------| -| [`Status MemoryCopy(const void *data, uint64_t length)`](#memorycopy) | 将数据拷贝到 `Buffer` 的缓存。 | -| [`int64_t GetSize() const`](#getsize) | 获取对象 `Buffer` 的大小。 | -| [`Status Publish(const std::unordered_set &nestedKeys = {})`](#publish) | 将 `Buffer` 的数据发布到数据系统。 | -| [`Status Seal(const std::unordered_set &nestedKeys = {})`](#seal) | 将 `Buffer` 的数据发布到数据系统,并不再允许修改。 | -| [`Status WLatch(uint64_t timeoutSec = 60)`](#wlatch) | 对 `Buffer` 添加写锁。 | -| [`Status RLatch(uint64_t timeoutSec = 60)`](#rlatch) | 对 `Buffer` 添加读锁。 | -| [`Status UnRLatch()`](#unrlatch) | 对 `Buffer` 解除读锁。 | -| [`Status UnWLatch()`](#unwlatch) | 对 `Buffer` 解除写锁。 | -| [`void *MutableData()`](#mutabledata) | 获取 `Buffer` 可读写的缓存数据指针。 | -| [`const void *ImmutableData()`](#immutabledata) | 获取 `Buffer` 只读缓存数据指针。 | -| [`Status InvalidateBuffer()`](#invalidatebuffer) | 使当前主机上的 `Buffer` 数据无效化。 | - -#### MemoryCopy - -```cpp -Status MemoryCopy(const void *data, uint64_t length) -``` - -将数据拷贝到 `Buffer` 的缓存。 - -- 参数 - - - `data`: 需要拷贝的数据内存地址。 - - `length`: 需要拷贝的数据长度。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示数据拷贝成功,否则返回其他错误码。 - -#### GetSize - -```cpp -int64_t GetSize() const -``` - -获取对象 `Buffer` 的大小。 - -- 返回值 - - `Buffer` 大小(以字节为单位)。 - -#### Publish - -```cpp -Status Publish(const std::unordered_set &nestedKeys = {}) -``` - -将 `Buffer` 的数据发布到数据系统。 - -- 参数 - - - `nestedKeys`: 依赖的嵌套对象 key 数组。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示发布成功,否则返回其他错误码。 - -#### Seal - -```cpp -Status Seal(const std::unordered_set &nestedKeys = {}) -``` - -将 `Buffer` 的数据发布到数据系统,并不再允许修改。 - -- 参数 - - - `nestedKeys`: 依赖的嵌套对象 key 数组。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示发布成功,否则返回其他错误码。 - -#### WLatch - -```cpp -Status WLatch(uint64_t timeoutSec = 60) -``` - -对 `Buffer` 添加写锁。 - -- 参数 - - - `timeoutSec`: 添加写锁的超时时间,默认为 60 秒。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示加写锁成功,否则返回其他错误码。 - -#### RLatch - -```cpp -Status RLatch(uint64_t timeoutSec = 60) -``` - -对 `Buffer` 添加读锁。 - -- 参数 - - - `timeoutSec`: 添加读锁的超时时间,默认为 60 秒。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示加读锁成功,否则返回其他错误码。 - -#### UnRLatch - -```cpp -Status UnRLatch() -``` - -对 `Buffer` 解除读锁。 - - -- 返回值 - - 返回值状态码为 `K_OK` 时表示解读锁成功,否则返回其他错误码。 - -#### UnWLatch - -```cpp -Status UnWLatch() -``` - -对 `Buffer` 解除写锁。 - - -- 返回值 - - 返回值状态码为 `K_OK` 时表示解写锁成功,否则返回其他错误码。 - -#### MutableData - -```cpp -void *MutableData() -``` - -获取 `Buffer` 可读写的缓存数据指针。 - -- 返回值 - - 返回值为可读写的缓存数据指针。 - -#### ImmutableData - -```cpp -const void *ImmutableData() -``` - -获取 `Buffer` 只读缓存数据指针。 - -- 返回值 - - 返回值为只读缓存数据指针。 - -#### InvalidateBuffer - -```cpp -Status InvalidateBuffer() -``` - -使当前主机上的 Buffer 数据无效化。 - -- 返回值 - - 返回值状态码为 `K_OK` 时表示无效化成功,否则返回其他错误码。 - -## CreateParam - -\#include <[datasystem/object_cache/object_client.h](../../../../include/datasystem/object_cache/object_client.h)> - -CreateParam 类用于配置对象的相关属性,当前定义如下: - -```cpp -struct CreateParam { - WriteMode writeMode = WriteMode::NONE_L2_CACHE; - ConsistencyType consistencyType = ConsistencyType::PRAM; -}; -``` - -详细参数解释见 [WriteMode](#writemode) 和 [ConsistencyType](#consistencytype) 章节。 - -## WriteMode - -\#include <[datasystem/object_cache/object_enum.h](../../../../include/datasystem/object_cache/object_enum.h)> - -WriteMode 类用于配置对象可靠性,当前定义如下: - -```cpp -enum class WriteMode : int { - NONE_L2_CACHE = 0, - WRITE_THROUGH_L2_CACHE = 1, - WRITE_BACK_L2_CACHE = 2, - NONE_L2_CACHE_EVICT = 3, -}; -``` - -目前,支持以下 `WriteMode`: -|定义| 说明 | -|----|--------------------------------------| -|`WriteMode.NONE_L2_CACHE` | 对象仅写入到缓存中。默认配置| -|`WriteMode.WRITE_THROUGH_L2_CACHE` | 对象同步写入缓存和二级缓存中。| -|`WriteMode.WRITE_BACK_L2_CACHE` | 对象同步写入缓存,异步写入二级缓存中。| -|`WriteMode.NONE_L2_CACHE_EVICT` | 对象是易失性的,如果缓存资源缺乏,对象可能会提前退出生命周期。| - - -## ConsistencyType - -\#include <[datasystem/object_cache/object_enum.h](../../../../include/datasystem/object_cache/object_enum.h)> - -WriteMode 类用于配置对象一致性,当前定义如下: - -```cpp -enum class ConsistencyType : int { - PRAM = 0, - CAUSAL = 1, -}; - -``` - -目前,支持以下 `ConsistencyType`: -|定义| 说明 | -|----|--------------------------------------| -|`ConsistencyType.PRAM` | PRAM 一致性。| -|`ConsistencyType.CAUSAL` | 因果一致性。| - -## enum CacheType -\#include <[datasystem/object_cache/object_enum.h](../../../../include/datasystem/object_cache/object_enum.h)> - -CacheType 类用于配置对象保存位置,当前定义如下: - -```cpp -enum class CacheType : int { - MEMORY = 0, - DISK = 1, -}; - -``` - -目前,支持以下 `CacheType`: -|定义| 说明 | -|----|--------------------------------------| -|`CacheType.MEMORY` | 保存到内存| -|`CacheType.DISK` | 保存到磁盘| diff --git a/docs/source_zh_cn/appendix/index.md b/docs/source_zh_cn/appendix/index.md new file mode 100644 index 0000000..98a4fc9 --- /dev/null +++ b/docs/source_zh_cn/appendix/index.md @@ -0,0 +1,11 @@ +# 附录 + +```{eval-rst} +.. toctree:: + :hidden: + + log_guide.md +``` + +附录包含以下内容: +- [openYuanrong datasystem 日志指南](log_guide.md) \ No newline at end of file diff --git a/docs/source_zh_cn/conf.py b/docs/source_zh_cn/conf.py index ed5675b..e36b946 100644 --- a/docs/source_zh_cn/conf.py +++ b/docs/source_zh_cn/conf.py @@ -12,163 +12,135 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob -import os -import re -import shutil import sys -from sphinx.ext import autodoc as sphinx_autodoc -import sphinx.ext.autosummary.generate as g +import os +import logging +from pathlib import Path +import datetime -sys.path.append(os.path.abspath('../_ext')) -from myautosummary import DsCnAutoSummary +logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO +) -# Fix some dl-label lack class='simple' -from docutils.writers import _html_base +sys.path.append(str(Path("..", "api", "python").resolve())) -with open(_html_base.__file__, "r", encoding="utf-8") as f: - code_str = f.read() - old_str = ''' if self.is_compactable(node): - classes.append('simple')''' - new_str = ''' if classes == []: - classes.append('simple')''' - code_str = code_str.replace(old_str, new_str) - exec(code_str, _html_base.__dict__) +ENV_YR_GIT_COMMIT_ID = os.environ.get("YR_DOC_GIT_COMMIT_ID", "") -# -- Project information ----------------------------------------------------- +build_time = datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta( + hours=8 +) +current_time_str = build_time.strftime("%Y-%m-%d %H:%M:%S") -project = 'openYuanrong datasystem' -copyright = 'openYuanrong datasystem' -author = 'openYuanrong datasystem' +project = "openYuanrong datasystem" +copyright = f"{build_time.year}, openEuler openYuanrong datasystem" +author = "openYuanrong datasystem with CC BY 4.0 LICENSE" -# The full version, including alpha/beta/rc tags -release = 'master' +logging.info( + f"""Doc build configs: +ENV_YR_GIT_COMMIT_ID: {ENV_YR_GIT_COMMIT_ID} -# -- General configuration --------------------------------------------------- +current_date: {current_time_str} +project: {project} +copyright: {copyright} +author: {author} +""" +) -html_static_path = [os.path.abspath('../_static')] +templates_path = ["../_templates"] +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", -html_css_files = [ - 'custom.css', + "README.md", + "sample_code", + "multi_language_function_programming_interface/api/distributed_programming/C++", + "multi_language_function_programming_interface/api/distributed_programming/Java", + "multi_language_function_programming_interface/api/distributed_programming/Python", + + "" ] -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -myst_enable_extensions = ["dollarmath", "amsmath", "colon_fence"] +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -myst_heading_anchors = 5 extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon', - 'sphinx.ext.mathjax', - 'IPython.sphinxext.ipython_console_highlighting', - 'myst_parser', - 'sphinx_design', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_design", + "sphinx_copybutton", + "sphinx_togglebutton", + "myst_parser", + "breathe", + "sphinxcontrib.openapi", # 添加 sphinxcontrib-openapi 扩展 ] -source_suffix = { - '.rst': 'restructuredtext', - '.md': 'markdown', -} - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +autoclass_content = "both" +copybutton_exclude = ".linenos, .gp, .go" -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -mathjax_path = 'https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/mathjax/MathJax-3.2.2/es5/tex-mml-chtml.js' - -mathjax_options = { - 'async':'async' -} - -nbsphinx_requirejs_path = 'https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js' - -nbsphinx_requirejs_options = { - "crossorigin": "anonymous", - "integrity": "sha256-1fEPhSsRKlFKGfK3eO710tEweHh1fwokU5wFGDHO+vg=" -} -smartquotes_action = 'De' - -exclude_patterns = [] - -pygments_style = 'sphinx' - -autodoc_inherit_docstrings = False +# ----------------------------------------------------------------------------- +# FOR PYTHON API GENEREATE +# ----------------------------------------------------------------------------- +autodoc_mock_imports = ["acl", "requests", "fastapi", "numpy"] autosummary_generate = True +autosummary_generate_overwrite = True # 覆盖已生成的文件 +autosummary_ignore_module_all = False # 不忽略 __all__ 的限制 +autosummary_imported_members = True -autosummary_generate_overwrite = False - -html_search_language = 'zh' -# -- Options for HTML output ------------------------------------------------- +# ----------------------------------------------------------------------------- +# HTML templates +# ----------------------------------------------------------------------------- +html_logo = "./images/logo-small.png" +html_theme = "sphinx_book_theme" -# Reconstruction of sphinx auto generated document translation. - -language = 'zh_CN' -locale_dirs = ['../../../../resource/locale/'] -gettext_compact = False - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Modify regex for sphinx.ext.autosummary.generate.find_autosummary_in_lines. -gfile_abs_path = os.path.abspath(g.__file__) -autosummary_re_line_old = r"autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*')" -autosummary_re_line_new = r"autosummary_re = re.compile(r'^(\s*)\.\.\s+(ms[a-z]*)?autosummary::\s*')" -with open(gfile_abs_path, "r+", encoding="utf8") as f: - data = f.read() - data = data.replace(autosummary_re_line_old, autosummary_re_line_new) - exec(data, g.__dict__) - -# Modify default signatures for autodoc. -autodoc_source_path = os.path.abspath(sphinx_autodoc.__file__) -autodoc_source_re = re.compile(r'stringify_signature\(.*?\)') -get_param_func_str = r"""\ -import re -import inspect as inspect_ - -def get_param_func(func): - try: - source_code = inspect_.getsource(func) - if func.__doc__: - source_code = source_code.replace(func.__doc__, '') - all_params_str = re.findall(r"def [\w_\d\-]+\(([\S\s]*?)(\):|\) ->.*?:)", source_code) - if "@classmethod" in source_code: - all_params = re.sub("(self|cls)(,|, )?", '', all_params_str[0][0].replace("\n", "")) - else: - all_params = re.sub("(self)(,|, )?", '', all_params_str[0][0].replace("\n", "")) - return all_params - except: - return '' - -def get_obj(obj): - if isinstance(obj, type): - return obj.__init__ - - return obj -""" - -with open(autodoc_source_path, "r+", encoding="utf8") as f: - code_str = f.read() - code_str = autodoc_source_re.sub('"(" + get_param_func(get_obj(self.object)) + ")"', code_str, count=0) - exec(get_param_func_str, sphinx_autodoc.__dict__) - exec(code_str, sphinx_autodoc.__dict__) +html_static_path = ["../_static"] +html_css_files = [ + "custom.css", +] +html_theme_options = { + "show_navbar_depth": 1, + "max_navbar_depth": 7, + "collapse_navigation": True, + "extra_footer": """ + Built with + Sphinx + using a + theme + provided by + Executable Books Project. + """ +} +# ----------------------------------------------------------------------------- +# Myst extensions +# ----------------------------------------------------------------------------- +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "fieldlist", + "html_admonition", + "html_image", + "colon_fence", + "smartquotes", + "replacements", + "strikethrough", + "substitution", + "tasklist", + "attrs_inline", + "attrs_block", +] -def setup(app): - app.add_directive('dscnautosummary', DsCnAutoSummary) - app.add_config_value('rst_files', set(), False) +myst_heading_anchors = 4 +# ----------------------------------------------------------------------------- +# breathe config +# ----------------------------------------------------------------------------- +breathe_projects = {"openYuanrong_datasystem": "./../.doxygendocs/xml"} +breathe_default_project = "openYuanrong_datasystem" diff --git a/docs/source_zh_cn/contributor_guide/contribution.md b/docs/source_zh_cn/contributor_guide/contribution.md new file mode 100644 index 0000000..e9db19e --- /dev/null +++ b/docs/source_zh_cn/contributor_guide/contribution.md @@ -0,0 +1,20 @@ +# 贡献者指南 + +我们欢迎您对 openYuanrong 做各种形式的贡献,包括但不限于: + +- 代码贡献:提交或检视补丁,补充测试用例等。 +- 非代码贡献:补充文档和示例,参与社区论坛,撰写教程、博客文章等。 + +openYuanrong 是 openEuler 社区中的项目。参与社区贡献前,您需要提前签署 [openEuler 社区贡献者许可协议(CLA)](https://clasign.osinfra.cn/sign/gitee_openeuler-1611298811283968340){target="_blank"}。 + +## 提交或处理 Issue + +我们使用 issue 来记录和追踪开发者的反馈及任务。您可以在项目主页点击工具栏中的“Issues”找到 Issue 列表,上报 Bug 或者提交需求请参考 [Issue 提交指南](https://gitee.com/openeuler/community/blob/master/zh/contributors/issue-submit.md){target="_blank"}。 + +您可以对感兴趣的 Issue 发表自己的意见,参与交流和讨论。如果愿意处理某个 Issue,只需要在评论框内输入 /assign 或 /assign @yourself,机器人就会将问题分配给您,您的名字将显示在负责人列表里。 + +## 如何工作 + +请参考以下内容了解如何贡献代码。 + +- 参考[源码编译 openYuanrong datasystem](../installation/installation_linux.md#源码编译安装) 构建您的代码开发环境。 diff --git a/docs/source_zh_cn/getting-started/deploy.md b/docs/source_zh_cn/deployment/deploy.md similarity index 92% rename from docs/source_zh_cn/getting-started/deploy.md rename to docs/source_zh_cn/deployment/deploy.md index c35b04c..98b3944 100644 --- a/docs/source_zh_cn/getting-started/deploy.md +++ b/docs/source_zh_cn/deployment/deploy.md @@ -1,4 +1,4 @@ -# 部署openYuanrong datasystem +# 部署 openYuanrong datasystem - [openYuanrong datasystem进程部署](#openyuanrong-datasystem进程部署) @@ -29,8 +29,8 @@ openYuanrong datasystem进程部署所需的系统环境依赖如下: |软件名称|版本|作用| |-------|----|----| | openEuler |22.03|运行openYuanrong datasystem的操作系统| -|[CANN](#安装cann)|8.0.0或8.0.rc2|运行异构相关特性的依赖库| -|[Python](#安装python)|3.10-3.11|openYuanrong datasystem dscli的使用依赖Python环境| +|[CANN](#安装cann)|8.2.rc1|运行异构相关特性的依赖库| +|[Python](#安装python)|3.9-3.11|openYuanrong datasystem dscli的使用依赖Python环境| |[dscli](#安装dscli)|-|用于部署openYuanrong datasystem的命令行工具| |[ETCD](#安装并部署etcd)|3.5|openYuanrong datasystem集群管理依赖组件| |[SSH互信配置](#ssh互信配置)|-|仅多机部署需要,配置SSH互信用于机器间互相访问| @@ -38,21 +38,26 @@ openYuanrong datasystem进程部署所需的系统环境依赖如下: 下面给出以上依赖的安装方法。 ### 安装CANN -在[Ascend官网](https://www.hiascend.com/hardware/firmware-drivers/community?product=1&model=30&cann=8.0.0.beta1&driver=Ascend+HDK+24.1.RC3)下载CANN run包,安装 run 包: +CANN的安装依赖Python环境,请确保您在开始安装CANN之前环境中的Python已经就绪。 + +在[Ascend官网](https://www.hiascend.com/developer/download/community/result?module=cann&cann=8.2.RC1)下载CANN run包,安装 run 包: ```bash -./Ascend-cann-toolkit__linux-.run --install +chmod +x ./Ascend-cann-toolkit__linux-.run +./Ascend-cann-toolkit__linux-.run --install --quiet ``` -执行以上命令会打屏华为企业业务最终用户许可协议(EULA)的条款和条件,请输入Y或y同意协议,继续安装流程。 安装完成后,若显示如下信息,则说明软件安装成功: ```bash -xxx install success +Toolkit: Ascend-cann-toolkit__linux- install success ``` -xxx表示安装的实际软件包名。 -如果用户未指定安装路径,则软件会安装到默认路径下,默认安装路径如下。root用户:"/usr/local/Ascend",非root用户:"\${HOME}/Ascend",${HOME}为当前用户目录。 +如果用户未指定安装路径,则软件会安装到默认路径下,默认安装路径如下: +| 用户身份 | 默认路径 | +| ------ | ------------------- | +| root | `/usr/local/Ascend` | +| 非 root | `$HOME/Ascend` | -配置环境变量,以非root用户安装后的默认路径为例,请用户根据set_env.sh的实际路径执行如下命令: +加载环境变量(非 root 示例): ```bash source ${HOME}/Ascend/ascend-toolkit/set_env.sh ``` @@ -87,7 +92,7 @@ python --version ``` ### 安装dscli -dscli命令行工具集成在openYuanrong datasystem的wheel包 `openyuanrong_datasystem--cp311-cp311-manylinux_2_34_.whl`中,安装openYuanrong datasystem请参考[安装openYuanrong datasystem](install.md)。 +dscli命令行工具集成在openYuanrong datasystem的wheel包 `openyuanrong_datasystem--cp311-cp311-manylinux_2_34_.whl`中,安装openYuanrong datasystem请参考[安装openYuanrong datasystem](../installation/installation_linux.md)。 安装完成后,运行如下命令: ```bash @@ -207,7 +212,7 @@ openYuanrong datasystem集群依赖ETCD,部署前需要先部署ETCD,部署E ``` 输出OK说明部署成功。 - 更多快速部署的使用方式请参考:[dscli单机快速部署](../appendix/dscli.md#单机部署) + 更多快速部署的使用方式请参考:[dscli单机快速部署](../deployment/dscli.md#单机部署) - 通过配置项部署 @@ -223,7 +228,7 @@ openYuanrong datasystem集群依赖ETCD,部署前需要先部署ETCD,部署E ``` 输出OK说明部署成功。 - 更多配置项部署的使用方式请参考:[dscli单机配置项部署](../appendix/dscli.md#单机部署) + 更多配置项部署的使用方式请参考:[dscli单机配置项部署](../deployment/dscli.md#单机部署) #### 多机部署 @@ -272,7 +277,7 @@ dscli up -f ./cluster_config.json > - 需要部署的机器上都已安装dscli,dscli安装可参考:[安装dscli](#安装dscli)。 > -更多配置项部署的使用方式请参考:[dscli多机部署](../appendix/dscli.md#多机部署) +更多配置项部署的使用方式请参考:[dscli多机部署](../deployment/dscli.md#多机部署) ### 快速验证 @@ -372,7 +377,7 @@ openYuanrong datasystem Kubernetes部署所需的依赖如下: - 源码编译构建镜像 - 如果需要从源码构建镜像,需要先完成 [源码编译](install.md)。源码编译完成之后执行如下命令: + 如果需要从源码构建镜像,需要先完成 [源码编译](../installation/installation_linux.md)。源码编译完成之后执行如下命令: ```bash cd yuanrong-datasystem/docker @@ -455,7 +460,7 @@ kubectl get pods -o wide 当Pod处于Running状态时说明Pod处于就绪状态,可以正常对外提供服务,部署成功。 -更多部署参数配置请参考:[Kubernetes配置项](../appendix/k8s_configuration.md) +更多部署参数配置请参考:[Kubernetes配置项](../deployment/k8s_configuration.md) ### 快速验证 diff --git a/docs/source_zh_cn/appendix/dscli.md b/docs/source_zh_cn/deployment/dscli.md similarity index 96% rename from docs/source_zh_cn/appendix/dscli.md rename to docs/source_zh_cn/deployment/dscli.md index 38215ec..154bac8 100644 --- a/docs/source_zh_cn/appendix/dscli.md +++ b/docs/source_zh_cn/deployment/dscli.md @@ -20,9 +20,9 @@ - [dscli stop](#dscli-stop) - [dscli up](#dscli-up) - [dscli down](#dscli-down) - - [dscli generate_helm_chart](#dscli-generatehelmchart) - - [dscli generate_cpp_template](#dscli-generatecpptemplate) - - [dscli generate_config](#dscli-generateconfig) + - [dscli generate_helm_chart](#dscli-generate_helm_chart) + - [dscli generate_cpp_template](#dscli-generate_cpp_template) + - [dscli generate_config](#dscli-generate_config) - [dscli collect_log](#dscli-collect_log) - [dscli runscript](#dscli-runscript) - [配置项说明](#配置项说明) @@ -32,7 +32,7 @@ -[![查看源文件](https://Mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/openeuler/yuanrong-datasystem/blob/master/docs/source_zh_cn/appendix/dscli.md) +[![查看源文件](https://Mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/openeuler/yuanrong-datasystem/blob/master/docs/source_zh_cn/deployment/dscli.md) 本文档介绍dscli集群管理工具的使用方法、命令行参数以及配置项说明。 @@ -220,7 +220,7 @@ pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${version}/yr_da ### 源码编译安装 -源码编译安装 openYuanrong datasystem 详细说明请参考:[源码编译方式安装openYuanrong datasystem](../getting-started/install.md#源码编译安装)。 +源码编译安装 openYuanrong datasystem 详细说明请参考:[源码编译方式安装openYuanrong datasystem](../installation/installation_linux.md#源码编译安装)。 ## dscli使用教程 @@ -229,7 +229,7 @@ pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${version}/yr_da ### openYuanrong datasystem集群部署 -openYuanrong datasystem集群管理依赖ETCD,在部署openYuanrong datasystem集群前请确保ETCD集群处于可用状态。ETCD部署教程可参考:[ETCD集群部署](../getting-started/deploy.md#安装并部署ETCD)。 +openYuanrong datasystem集群管理依赖ETCD,在部署openYuanrong datasystem集群前请确保ETCD集群处于可用状态。ETCD部署教程可参考:[ETCD集群部署](../deployment/deploy.md#安装并部署etcd)。 #### 单机部署 @@ -449,7 +449,7 @@ openYuanrong datasystem单机卸载依赖 [dscli stop](#dscli-stop) 命令: ### 生成Helm Chart模板 -通过 [dscli generate_helm_chart](#dscli-generatehelmchart) 命令在当前目录生成 openYuanrong datasystem 的 Helm Chart 包: +通过 [dscli generate_helm_chart](#dscli-generate_helm_chart) 命令在当前目录生成 openYuanrong datasystem 的 Helm Chart 包: ```bash dscli generate_helm_chart @@ -471,13 +471,13 @@ helm install yr_datasystem /home/sn/datasystem-helm-chart/datasystem helm uninstall yr_datasystem ``` -更多 openYuanrong datasystem Kubernetes 部署详细信息请参考:[openYuanrong datasystem Kubernetes部署](../getting-started/deploy.md#openyuanrong-datasystem-kubernetes部署)。 +更多 openYuanrong datasystem Kubernetes 部署详细信息请参考:[openYuanrong datasystem Kubernetes部署](../deployment/deploy.md#openyuanrong-datasystem-kubernetes部署)。 -更多关于 dscli generate_helm_chart 命令的使用请参考:[dscli generate_helm_chart](#dscli-generatehelmchart)。 +更多关于 dscli generate_helm_chart 命令的使用请参考:[dscli generate_helm_chart](#dscli-generate_helm_chart)。 ### 生成Cpp样例代码 -通过 [dscli generate_cpp_template](#dscli-generatecpptemplate) 命令可快速生成 openYuanrong datasystem 的 C++ 样例代码: +通过 [dscli generate_cpp_template](#dscli-generate_cpp_template) 命令可快速生成 openYuanrong datasystem 的 C++ 样例代码: ```bash dscli generate_cpp_template @@ -506,13 +506,13 @@ bash run.sh ### 日志收集 -通过 [dscli collect_log](#dscli-collectlog) 命令可快速收集节点上的日志: +通过 [dscli collect_log](#dscli-collect_log) 命令可快速收集节点上的日志: ```bash dscli collect_log --cluster_config_path ./cluster_config.json ``` -更多关于 dscli generate_helm_chart 命令的使用请参考:[dscli collect_log](#dscli-collectlog)。 +更多关于 dscli generate_helm_chart 命令的使用请参考:[dscli collect_log](#dscli-collect_log)。 ### 多机命令运行 @@ -527,7 +527,7 @@ dscli collect_log --cluster_config_path ./cluster_config.json EOF ``` -2. 设置集群配置信息 `cluster_config.json`,可通过 [dscli generate_config](#dscli-generateconfig) 命令生成并修改,其中 `worker_nodes` 即为要执行脚本的机器: +2. 设置集群配置信息 `cluster_config.json`,可通过 [dscli generate_config](#dscli-generate_config) 命令生成并修改,其中 `worker_nodes` 即为要执行脚本的机器: ```json { diff --git a/docs/source_zh_cn/deployment/index.md b/docs/source_zh_cn/deployment/index.md new file mode 100644 index 0000000..cd55c84 --- /dev/null +++ b/docs/source_zh_cn/deployment/index.md @@ -0,0 +1,15 @@ +# 部署 + +```{eval-rst} +.. toctree:: + :hidden: + + deploy.md + dscli.md + k8s_configuration.md +``` + +本节向您介绍如何部署 openYuanrong datasystem: +- [部署 openYuanrong datasystem](deploy.md) +- [dscli命令行工具详细介绍](dscli.md) +- [Kubernetes详细配置项](k8s_configuration.md) \ No newline at end of file diff --git a/docs/source_zh_cn/appendix/k8s_configuration.md b/docs/source_zh_cn/deployment/k8s_configuration.md similarity index 99% rename from docs/source_zh_cn/appendix/k8s_configuration.md rename to docs/source_zh_cn/deployment/k8s_configuration.md index 3392ad6..97df359 100644 --- a/docs/source_zh_cn/appendix/k8s_configuration.md +++ b/docs/source_zh_cn/deployment/k8s_configuration.md @@ -25,7 +25,7 @@ -[![查看源文件](https://Mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/openeuler/yuanrong-datasystem/blob/master/docs/source_zh_cn/appendix/k8s_configuration.md) +[![查看源文件](https://Mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/openeuler/yuanrong-datasystem/blob/master/docs/source_zh_cn/deployment/k8s_configuration.md) 本文档描述了openYuanrong datasystem Kubernetes快速启动以及详细配置项说明。 @@ -52,7 +52,7 @@ global: etcdAddress: "127.0.0.1:2379" ``` -部署openYuanrong datasystem请参考:[openYuanrong datasystem Kubernetes部署](../getting-started/deploy.md#openyuanrong-datasystem-kubernetes部署)。 +部署openYuanrong datasystem请参考:[openYuanrong datasystem Kubernetes部署](../deployment/deploy.md#openyuanrong-datasystem-kubernetes部署)。 默认情况下,每个 openYuanrong datasystem DaemonSet 最大可使用 2GB 共享内存空间用于缓存数据,如果需要调整该值,可以通过 [global.resources.datasystemWorker.sharedMemory](#资源相关配置) 调整。 diff --git a/docs/source_zh_cn/development-guide/api/cpp/Buffer.rst b/docs/source_zh_cn/development-guide/api/cpp/Buffer.rst new file mode 100644 index 0000000..789332d --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/Buffer.rst @@ -0,0 +1,97 @@ +Buffer +==================== + +.. cpp:class:: Buffer + + :header-file: #include + :namespace: datasystem + + 用于表示共享内存数据的类。 + + **公共函数** + + .. cpp:function:: Buffer() + + 默认构造类,创建一个空的 Buffer。 + + 返回: + 默认 ``Buffer`` 实例。 + + .. cpp:function:: Buffer(Buffer &&other) noexcept + + 移动构造函数。 + + 返回: + 移动构造后的 ``Buffer`` 实例。 + + .. cpp:function:: ~Buffer() = default + + 默认析构函数。 + + .. cpp:function:: void *GetData() + + 获取 ``Buffer`` 可读写的缓存数据指针。 + + 返回: + 可读写的缓存数据指针。 + + .. cpp:function:: int64_t GetSize() const + + 获取键值对 ``Buffer`` 的大小。 + + 返回: + ``Buffer`` 大小(以字节为单位)。 + + .. cpp:function:: Status WLatch(uint64_t timeoutSec = 60) + + 对 ``Buffer`` 添加写锁。 + + .. note:: + 仅在涉及单节点多实例同时访问的场景才需要加锁进行数据保护,否则无需在访问共享内存的数据前对其加锁。 + + 参数: + - **timeoutSec** - 添加写锁的超时时间,默认为 60 秒。 + + 返回: + 返回值状态码为 `K_OK` 时表示加写锁成功,否则返回其他错误码。 + + .. cpp:function:: Status UnWLatch() + + 对 ``Buffer`` 解除写锁。 + + 返回: + 返回值状态码为 `K_OK` 时表示加写锁成功,否则返回其他错误码。 + + .. cpp:function:: Status RLatch(uint64_t timeoutSec = 60) + + 对 ``Buffer`` 添加读锁。 + + .. note:: + 仅在涉及单节点多实例同时访问的场景才需要加锁进行数据保护,否则无需在访问共享内存的数据前对其加锁。 + + 参数: + - **timeoutSec** - 添加写锁的超时时间,默认为 60 秒。 + + 返回: + 返回值状态码为 `K_OK` 时表示加写锁成功,否则返回其他错误码。 + + .. cpp:function:: Status UnRLatch() + + 对 ``Buffer`` 解除读锁。 + + 返回: + 返回值状态码为 `K_OK` 时表示加写锁成功,否则返回其他错误码。 + + .. cpp:function:: Status MemoryCopy(const void *data, uint64_t length) + + 将数据拷贝到 `Buffer` 的缓存。该函数具备多线程并行拷贝的能力,同时能拷贝超过 2GB 的连续内存地址。 + + 参数: + - **data** - 需要拷贝的数据内存地址。 + - **length** - 需要拷贝的数据长度。 + + 返回: + 返回值状态码为 `K_OK` 时表示数据拷贝成功,否则返回其他错误码。 + + + \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/KVClient.rst b/docs/source_zh_cn/development-guide/api/cpp/KVClient.rst new file mode 100644 index 0000000..5545da5 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/KVClient.rst @@ -0,0 +1,247 @@ +KVClient +==================== + +.. cpp:class:: KVClient + + :header-file: #include + :namespace: datasystem + + KV缓存客户端。 + + **公共函数** + + .. cpp:function:: KVClient(const ConnectOptions &connectOptions) + + 构造KV缓存客户端实例。 + + 参数: + - **connectOptions** - 配置连接选项,包括IP地址和端口,详见 cpp:class:`ConnectOptions` 章节 + + 返回: + KV缓存客户端实例。 + + .. cpp:function:: ~KVClient() + + 析构KV缓存客户端实例,析构过程中会自动断开与 Worker 的连接,释放客户端持有的资源。 + + .. cpp:function:: Status Init() + + 建立与数据系统 Worker 之间的连接并完成初始化。 + + 返回: + 返回值状态码为 StatusCode::K_OK 时表示初始化成功,否则返回其他错误码。 + + .. cpp:function:: Status ShutDown() + + 断开与数据系统 Worker 之间的连接。 + + 返回: + 返回值状态码为 StatusCode::K_OK 时表示断链成功,否则返回其他错误码。 + + + .. cpp:function:: Status Create(const std::string &key, uint64_t size, const SetParam ¶m, std::shared_ptr &buffer) + + 创建数据系统共享内存Buffer,可以将数据拷贝到Buffer中,再调用Set接口缓存到数据系统中。该接口应用于避免创建临时内存,减少内存拷贝的场景。 + + 参数: + - **key** - 需要设置的key。key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;``,最大长度为255字节。 + - **size** - 需要创建的共享内存Buffer的大小,以字节为单位。 + - **param** - 设置参数,详见 :cpp:class:`SetParam` 章节。 + - **buffer** - 传出参数,表示创建好的共享内存 :cpp:class:`Buffer` 。 + + 返回: + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status Set(const std::shared_ptr &buffer) + + 将共享内存数据缓存到数据系统。 + + 参数: + - **buffer** - 共享内存 :cpp:class:`Buffer` 。 + + 返回值 + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status Set(const std::string &key, const StringView &val, const SetParam ¶m) + + 设置键值对数据缓存到数据系统。 + + 参数: + - **key** - 键,key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;``,最大长度为255字节. + - **val** - 需要缓存的值. + - **param** - 设置参数,详见 :cpp:class:`SetParam` 章节。 + + 返回: + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status MCreate(const std::vector &keys, const std::vector &sizes, const SetParam ¶m, std::vector> &buffers) + + 创建数据系统共享内存Buffer,可以将数据拷贝到Buffer中,再调用Set接口缓存到数据系统中。该接口应用于避免创建临时内存,减少内存拷贝的场景。 + + 参数: + - **keys** - 需要设置的一组key. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``·-_!@#%^*()+=:;``,单个key最大长度为255字节. key的最大个数为10,000,推荐单次设置的key个数小于等于64个。 + - **sizes** - 设置共享内存Buffer的大小,以字节为单位. 该数组长度需要与 ``keys`` 的长度相等。 + - **param** - 设置参数,详见 :cpp:class:`SetParam` 章节。 + - **buffers** - 传出参数,表示创建好的共享内存 :cpp:class:`Buffer` 数组,该数组的长度与 `keys` 相等,索引位置一一对应,即每个 ``buffers[i]`` 的值与 ``keys[i]`` 相对应。 + + 返回: + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status MSet(const std::vector> &buffers) + + 键值对批量设置接口。与 :cpp:func:`Status MCreate(const std::vector &keys, const std::vector &sizes, const SetParam ¶m, std::vector> &buffers)` 接口相互配合使用,用于批量将共享内存 :cpp:class:`Buffer` 缓存到数据系统中。 + + 参数: + - **buffers** - 需要缓存到数据系统的共享内存 :cpp:class:`Buffer` 数组。 + + 返回: + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status MSet(const std::vector &keys, const std::vector &vals, std::vector &outFailedKeys, const SetParam ¶m) + + 键值对批量设置接口。可批量设置键值对并返回失败的键。 + + 参数: + - **keys** - 需要设置的一组key. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;``,单个key最大长度为255字节. key的最大个数为10,000,推荐单次设置的key个数小于等于64个。 + - **vals** - 需要设置的一组key对应的value. 该数组长度需要与 ``keys`` 的长度相等。 + - **outFailedKeys** - 传出参数,代表设置失败的key。 + - **param** - 设置参数,详见 :cpp:class:`SetParam` 章节。 + + 返回: + 返回值状态码为 `K_OK` 时表示设置成功,否则返回其他错误码。 + + .. cpp:function:: Status Get(const std::string &key, std::string &val, int32_t subTimeoutMs) + + 获取键对应的数据。 + + 参数: + - **key** - 键. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;``,单个key最大长度为255字节。 + - **val** - 传出参数,返回缓存数据。 + - **subTimeoutMs** - 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示获取成功。 + - 返回 ``StatusCode::K_INVALID`` 表示 ``key`` 校验不通过。 + - 返回 ``StatusCode::K_NOT_FOUND`` 表示 ``key`` 不存在。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 时表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + .. cpp:function:: Status Get(const std::string &key, Optional &buffer, int32_t subTimeoutMs) + + 获取键对应的共享内存 :cpp:class:`Buffer` 。该接口相比 :cpp:func:`Status Get(const std::string &key, std::string &val, int32_t subTimeoutMs)` 可减少一次从共享内存到临时内存的拷贝,直接读取缓存在共享内存上的数据,性能更优。 + + 参数: + - **key** - 键. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;`` ,单个key最大长度为255字节。 + - **subTimeoutMs** - 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待。 + - **buffer** - 传出参数,返回的使用 :cpp:class:`Optional` 封装的共享内存 :cpp:class:`Buffer` ,当 Get 返回失败时,``buffer`` 的值为` ``nullptr``。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示获取成功。 + - 返回 ``StatusCode::K_INVALID`` 表示 ``key`` 校验不通过。 + - 返回 ``StatusCode::K_NOT_FOUND`` 表示 ``key`` 不存在。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 时表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + + .. cpp:function:: Status Get(const std::vector &keys, std::vector &vals, int32_t subTimeoutMs) + + 批量获取数据。 + + 参数: + - **keys** - 需要获取的一组key. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;`` ,单个key最大长度为255字节。传入的key的个数不能超过1万,推荐单次获取key个数小于等于64个。 + - **subTimeoutMs** - 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待。 + - **vals** - 传出参数,返回一组获取的数据。若有部分数据获取不成功,则对应位置的vector的对象为空。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示至少有一个数据获取成功。 + - 返回 ``StatusCode::K_INVALID`` 表示存在key校验不通过。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 时表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_NOT_FOUND`` 表示所有`keys`不存在。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + + .. cpp:function:: Status Get(const std::vector &keys, std::vector> &buffers, int32_t subTimeoutMs) + + 批量获取多个键对应的共享内存 :cpp:class:`Buffer` 。该接口相比 :cpp:func:`Status Get(const std::vector &keys, std::vector &vals, int32_t subTimeoutMs)` 少一次从共享内存到临时内存的拷贝,性能更优。 + + 参数: + - **keys** - 需要获取的一组key. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;`` ,单个key最大长度为255字节。传入的key的个数不能超过1万,推荐单次获取key个数小于等于64个。 + - **subTimeoutMs** - 支持订阅不存在的数据,subTimeoutMs表示订阅等待的时长,单位ms。不允许为负数,默认值为0表示不等待。 + - **buffers** - 传出参数,返回的一组使用 :cpp:class:`Optional` 封装的共享内存 :cpp:class:`Buffer` 。若有部分数据获取不成功,则对应位置的vector的对象为空。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示至少有一个数据获取成功。 + - 返回 ``StatusCode::K_INVALID`` 表示存在key校验不通过。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 时表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_NOT_FOUND`` 表示所有`keys`不存在。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + .. cpp:function:: Status Del(const std::string &key) + + 删除指定键值对。key不存在时视为删除成功。 + + 参数: + - **key** - 键. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;`` ,单个key最大长度为255字节。 + + 返回: + 返回值状态码为 `K_OK` 时表示初始化成功,否则返回其他错误码。 + + .. cpp:function:: Status Del(const std::vector &keys, std::vector &failedKeys) + + 批量删除指定键值对。key不存在时视为删除成功。 + + 参数: + - **keys** - 需要获取的一组key. key的合法字符为:英文字母(a-zA-Z)、数字以及 ``-_!@#%^*()+=:;`` ,单个key最大长度为255字节。传入的key的个数不能超过1万,推荐单次获取key个数小于等于64个。 + - **failedKeys** - 传出参数,返回删除失败的key。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示至少有一个数据删除成功。 + - 返回 ``StatusCode::K_INVALID`` 表示存在key校验不通过。 + - 返回值状态码为 ``StatusCode::K_RPC_UNAVAILABLE`` 时表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + .. cpp:function:: std::future DelAll() + + 异步删除集群中所有的键值对。 + + 返回: + ``future`` 代表 ``DelAll`` 操作的未来结果,可以通过 ``std::future::get()`` 函数获取异步结果。 + + .. cpp:function:: Status HealthCheck() + + 检查连接的 Worker 是否健康。 + + 返回: + 返回值状态码为 `K_OK` 时表示 Worker 健康,否则返回其他错误码。 + + .. cpp:function:: Status KVClient::Exist(const std::vector &keys, std::vector &exists) + + 批量查询一组键(keys)是否存在,并返回每个键的存在性状态。支持最多10000个键的查询。 + + 参数: + - **keys** - 待查询的键列表,最大支持10000个键。 + - **exists** - 传出参数,返回每个键的存在性状态。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示查询成功。 + - 返回 ``StatusCode::K_INVALID`` 表示提供的键中包含非法字符或为空。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_NOT_READY`` 表示服务当前无法处理请求。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 + + .. cpp:function:: Status Expire(const std::vector &keys, uint32_t ttlSeconds, std::vector &failedKeys) + + 批量为一组键(keys)更新过期生命周期(ttlSeconds),并返回更新失败的键(failedKeys)。最多支持 10000 个键的查询。 + + 参数: + - **keys** - 待更新生命周期的键列表。 + - **ttlSeconds** - 为键设置的新的生命周期,单位为秒。 + - **failedKeys** - 传出参数,返回操作失败的键。 + + 返回: + - 返回 ``StatusCode::K_OK`` 表示至少有一个键设置生命周期成功。 + - 返回 ``StatusCode::K_INVALID`` 表示提供的键中包含非法字符或为空。 + - 返回 ``StatusCode::K_NOT_FOUND`` 表示所有`keys`不存在。 + - 返回 ``StatusCode::K_RPC_UNAVAILABLE`` 表示请求遇到了网络错误。 + - 返回 ``StatusCode::K_NOT_READY`` 表示服务当前无法处理请求。 + - 返回 ``StatusCode::K_RUNTIME_ERROR`` 表示 worker 侧存在错误。 \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/Optional.rst b/docs/source_zh_cn/development-guide/api/cpp/Optional.rst new file mode 100644 index 0000000..728166a --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/Optional.rst @@ -0,0 +1,60 @@ +Optional +========================== + +.. cpp:class:: Optional + + :header-file: #include + :namespace: datasystem + + 用于表示一个值可能存在,当值不存在时表示为 null。 + + **公共函数** + + .. cpp:function:: constexpr Optional() + + 默认构造函数。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: Optional(const Optional &) + + 拷贝构造函数。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: Optional &operator=(const Optional &) + + 拷贝构造函数。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: Optional(Optional &&other) noexcept + + 移动构造函数。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: Optional &operator=(Optional &&) noexcept + + 移动构造函数。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: template explicit Optional(Args &&... args); + + 构造函数。 + + 参数: + - **args** - 构造入参。 + + 返回: + ``Optional`` 实例。 + + .. cpp:function:: ~Optional() + + 析构函数 \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/SensitiveValue.rst b/docs/source_zh_cn/development-guide/api/cpp/SensitiveValue.rst new file mode 100644 index 0000000..c3ef608 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/SensitiveValue.rst @@ -0,0 +1,85 @@ +SensitiveValue +========================== + +.. cpp:class:: SensitiveValue + + :header-file: #include + :namespace: datasystem + + 用于保存敏感信息的类。 + + **公共函数** + + .. cpp:function:: SensitiveValue() = default + + 默认构造函数。 + + .. cpp:function:: SensitiveValue(const char *str) + + 拷贝传入字符串指针上的敏感数据并构造实例。 + + 参数: + - **str** - 需要读取的字符串指针。 + + .. cpp:function:: SensitiveValue(const std::string &str) + + 拷贝传入字符串的敏感数据并构造实例。 + + 参数: + - **str** - 需要读取的字符串。 + + .. cpp:function:: SensitiveValue(const char *str, size_t size) + + 拷贝传入字符串指针上的敏感数据并构造实例。 + + 参数: + - **str** - 需要读取的字符串指针。 + - **size** - 需要读取的大小。 + + .. cpp:function:: SensitiveValue(std::unique_ptr data, size_t size); + + 拷贝传入数据指针上的敏感数据并构造实例。 + + 参数: + - **data** - 需要读取的数据指针。 + - **size** - 需要读取的大小。 + + .. cpp:function:: ~SensitiveValue() + + 析构函数,析构时会释放保存的敏感数据。 + + .. cpp:function:: bool Empty() const + + 判断是否为空实例,即不包含任何敏感数据。 + + 返回: + 如果返回为 ``true``,表明该实例内不包含敏感数据。 + + .. cpp:function:: const char *GetData() const + + 获取敏感数据的内存地址。 + + 返回: + 敏感数据内存地址。 + + .. cpp:function:: size_t GetSize() const + + 获取敏感数据的大小。 + + 返回: + 敏感数据大小。 + + .. cpp:function:: bool MoveTo(std::unique_ptr &outData, size_t &outSize) + + 将实例内的敏感数据移动到传入的 ``outData`` 中,该过程不涉及敏感数据的拷贝。 + + 参数: + - **outData** - 用于接收敏感数据的智能指针。 + - **outSize** - 敏感数据的大小。 + + 返回: + ``true`` 表示移动成功。 + + .. cpp:function:: void Clear() + + 清理实例内的敏感数据, \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/Status.rst b/docs/source_zh_cn/development-guide/api/cpp/Status.rst new file mode 100644 index 0000000..d10c730 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/Status.rst @@ -0,0 +1,123 @@ +Status +==================== + +.. cpp:class:: Status + + :header-file: #include + :namespace: datasystem + + Status 类用于表示请求/方法的执行结果。 + + **公共函数** + + .. cpp:function:: Status() noexcept + + 默认构造类,创建一个空的 Status 实例,表示 OK。 + + 返回: + 默认 ``Status`` 实例。 + + .. cpp:function:: Status(const Status &other) + + 构造函数。 + + 返回: + 构造后的 ``Status`` 实例。 + + .. cpp:function:: Status(Status &&other) noexcept + + 移动构造函数。 + + 返回: + 移动构造后的 ``Status`` 实例。 + + .. cpp:function:: Status(StatusCode code, std::string msg) + + 构造函数,根据 StatusCode 和 ``msg`` 构造 ``Status`` 实例。 + + 参数: + - **code** - 错误码。 + - **msg** - 错误信息。 + + 返回: + ``Status`` 实例。 + + .. cpp:function:: Status(StatusCode code, int lineOfCode, const std::string &fileName, const std::string &extra = "") + + 构造函数,根据 StatusCode、``lineOfCode``、``filenName`` 以及 ``extra`` 构造 ``Status`` 实例。 + + 参数: + - **code** - 错误码。 + - **lineOfCode** - 文件行号。 + - **filenName** - 文件名。 + - **extra** - 额外的报错信息。 + + 返回: + ``Status`` 实例。 + + .. cpp:function:: ~Status() noexcept + + 默认析构函数。 + + .. cpp:function:: static Status OK() + + 返回状态码为 ``K_OK`` 的 ``Status`` 实例。 + + 返回: + 状态码为 `K_OK` 的 ``Status`` 实例。 + + .. cpp:function:: std::string ToString() const + + 返回 Status 的状态码和报错信息。 + + 返回: + Status 的状态码和报错信息。 + + .. cpp:function:: StatusCode GetCode() const + + 返回 Status 的状态码。 + + 返回: + Status 的状态码。 + + .. cpp:function:: std::string GetMsg() + + 返回 Status 的报错信息。 + + 返回: + Status 的报错信息。 + + .. cpp:function:: void AppendMsg(const std::string &appendMsg) + + 拼接 Status 的报错信息。 + + 参数: + - **appendMsg** - 需要拼接的报错信息。 + + .. cpp:function:: bool IsOk() const + + 判断 Status 的状态码是否为 ``K_OK``。 + + 返回: + ``true`` 表示 Status 的状态码为 ``K_OK``。 + + .. cpp:function:: bool IsError() const + + 判断 Status 的状态码是否为非 ``K_OK`` 错误码。 + + 返回: + ``true`` 表示 Status 的状态码为非 ``K_OK`` 错误码。 + + .. cpp:function:: static std::string StatusCodeName(StatusCode code) + + 获取状态码的字符串表示,多用于打印需求。 + + 参数: + - **code** - 状态码。 + + 返回: + 状态码的字符串表示。 + + + + \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/StringView.rst b/docs/source_zh_cn/development-guide/api/cpp/StringView.rst new file mode 100644 index 0000000..d9f80fc --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/StringView.rst @@ -0,0 +1,71 @@ +StringView +==================== + +.. cpp:class:: StringView + + :header-file: #include + :namespace: datasystem + + 字符串视图类,主要用于高效地传递和访问字符串数据,而无需复制或分配内存。 + + **公共函数** + + .. cpp:function:: constexpr StringView() noexcept + + 默认构造函数,构造一个空字符串视图实例。 + + 返回: + ``StringView`` 实例。 + + .. cpp:function:: constexpr StringView(const char *str) + + 保存字符串指针上的并构造实例,这个过程不会发生拷贝。 + + 参数: + - **str** - 需要保存的字符串指针。 + + 返回: + ``StringView`` 实例。 + + .. cpp:function:: constexpr StringView(const char *str, size_t len) + + 保存字符串指针上的并构造实例,这个过程不会发生拷贝。 + + 参数: + - **str** - 需要保存的字符串指针。 + - **len** - 字符串长度。 + + 返回: + ``StringView`` 实例。 + + .. cpp:function:: StringView(const std::string &str) + + 保存字符串的指针上与字符串大小并构造实例,这个过程不会发生拷贝。 + + 参数: + - **str** - 字符串。 + + .. cpp:function:: ~StringView() + + 默认析构函数。 + + .. cpp:function:: constexpr const char *data() const noexcept + + 获取 StringView 的数据指针。 + + 返回: + ``StringView`` 的数据指针。 + + .. cpp:function:: constexpr size_t size() const noexcept + + 获取 StringView 的数据大小。 + + 返回: + ``StringView`` 的数据大小。 + + .. cpp:function:: constexpr size_t empty() const noexcept + + 判断 StringView 是否为空。 + + 返回: + ``true`` 表示 ``StringView`` 为空。 \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/enum-CacheType.rst b/docs/source_zh_cn/development-guide/api/cpp/enum-CacheType.rst new file mode 100644 index 0000000..e593c48 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/enum-CacheType.rst @@ -0,0 +1,18 @@ +CacheType +======================================== + +.. cpp:class:: CacheType + + :header-file: #include + :namespace: datasystem + + `CacheType` 类定义对象的缓存介质。 + + 目前,支持以下 `CacheType`: + + =================================== ================================================================== + 定义 说明 + =================================== ================================================================== + `CacheType::MEMORY` 内存介质,数据将缓存在共享内存上。 + `CacheType::DISK` 磁盘介质,数据将缓存在磁盘上。 + =================================== ================================================================== diff --git a/docs/source_zh_cn/development-guide/api/cpp/enum-ConsistencyType.rst b/docs/source_zh_cn/development-guide/api/cpp/enum-ConsistencyType.rst new file mode 100644 index 0000000..095712b --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/enum-ConsistencyType.rst @@ -0,0 +1,18 @@ +ConsistencyType +======================================== + +.. cpp:class:: ConsistencyType + + :header-file: #include + :namespace: datasystem + + `ConsistencyType` 类定义对象的一致性配置。 + + 目前,支持以下 `ConsistencyType`: + + =================================== ================================================================== + 定义 说明 + =================================== ================================================================== + `ConsistencyType::PRAM` Pipeline RAM一致性。默认配置 + `ConsistencyType::CAUSAL` 因果一致性。 + =================================== ================================================================== diff --git a/docs/source_zh_cn/development-guide/api/cpp/enum-StatusCode.rst b/docs/source_zh_cn/development-guide/api/cpp/enum-StatusCode.rst new file mode 100644 index 0000000..d4c67fb --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/enum-StatusCode.rst @@ -0,0 +1,27 @@ +StatusCode +======================================== + +.. cpp:class:: StatusCode + + :header-file: #include + :namespace: datasystem + + `StatusCode` 类定义Status的状态码。 + + 目前,支持以下 `StatusCode`: + + =================================== ================================================================== + 定义 说明 + =================================== ================================================================== + `StatusCode::K_OK` 表示成功 + `StatusCode::K_INVALID` 表示入参检验失败 + `StatusCode::K_NOT_FOUND` 表示对象不存在 + `StatusCode::K_RUNTIME_ERROR` 表示运行时错误,多发生于内部错误 + `StatusCode::K_OUT_OF_MEMORY` 表示内存不足 + `StatusCode::K_NOT_AUTHORIZED` 表示认证鉴权失败 + `StatusCode::K_RPC_CANCELLED` 表示请求的连接通道被意外关闭 + `StatusCode::K_RPC_UNAVAILABLE` 表示请求发生超时 + `StatusCode::K_ACL_ERROR` 表示发生了ACL相关的错误 + `StatusCode::K_HCCL_ERROR` 表示发生了HCCL相关的错误 + `StatusCode::K_URMA_ERROR` 表示发生了URMA相关的错误 + =================================== ================================================================== diff --git a/docs/source_zh_cn/development-guide/api/cpp/enum-WriteMode.rst b/docs/source_zh_cn/development-guide/api/cpp/enum-WriteMode.rst new file mode 100644 index 0000000..a452f67 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/enum-WriteMode.rst @@ -0,0 +1,18 @@ +WriteMode +========================== + +.. cpp:class:: WriteMode + + :header-file: #include + :namespace: datasystem + + 配置数据可靠性级别的枚举类。 + + =================================== ================================================================== + 定义 说明 + =================================== ================================================================== + `WriteMode::NONE_L2_CACHE` 对象仅写入到缓存中。默认配置 + `WriteMode::WRITE_THROUGH_L2_CACHE` 对象同步写入缓存和二级缓存中。 + `WriteMode::WRITE_BACK_L2_CACHE` 对象同步写入缓存,异步写入二级缓存中。 + `WriteMode::NONE_L2_CACHE_EVICT` 对象是易失性的,如果缓存资源缺乏,对象可能会提前退出生命周期。 + =================================== ================================================================== diff --git a/docs/source_zh_cn/development-guide/api/cpp/index.rst b/docs/source_zh_cn/development-guide/api/cpp/index.rst new file mode 100644 index 0000000..84b293c --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/index.rst @@ -0,0 +1,56 @@ +C++ +============================== + +.. toctree:: + :glob: + :hidden: + :maxdepth: 1 + + KVClient + Buffer + struct-ConnectOptions + struct-SetParam + enum-WriteMode + enum-ConsistencyType + enum-CacheType + SensitiveValue + StringView + Optional + Status + enum-StatusCode + +KV接口 +----------------------------------- + +.. list-table:: + :widths: 30 70 + :header-rows: 0 + + * - :cpp:func:`KVClient::KVClient` + - 构造KV缓存客户端实例。 + * - :cpp:func:`KVClient::~KVClient` + - 析构KV缓存客户端实例,析构过程中会自动断开与 Worker 的连接,释放客户端持有的资源。 + * - :cpp:func:`KVClient::Init` + - 建立与数据系统 Worker 之间的连接并完成初始化。 + * - :cpp:func:`KVClient::ShutDown` + - 断开与数据系统 Worker 之间的连接。 + * - :cpp:func:`KVClient::Create` + - 创建数据系统共享内存Buffer,可以将数据拷贝到Buffer中,再调用Set接口缓存到数据系统中。 + * - :cpp:func:`KVClient::Set` + - 将共享内存数据缓存到数据系统。 + * - :cpp:func:`KVClient::MCreate` + - 创建数据系统共享内存Buffer,可以将数据拷贝到Buffer中,再调用Set接口缓存到数据系统中。 + * - :cpp:func:`KVClient::MSet` + - 键值对批量设置接口。 + * - :cpp:func:`KVClient::Get` + - 获取键对应的数据。 + * - :cpp:func:`KVClient::Del` + - 删除指定键值对。 + * - :cpp:func:`KVClient::DelAll` + - 异步删除集群中所有的键值对。 + * - :cpp:func:`KVClient::HealthCheck` + - 检查连接的 Worker 是否健康。 + * - :cpp:func:`KVClient::Exist` + - 批量查询一组键(keys)是否存在。 + * - :cpp:func:`KVClient::Expire` + - 批量为一组键(keys)更新过期生命周期。 \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/api/cpp/struct-ConnectOptions.rst b/docs/source_zh_cn/development-guide/api/cpp/struct-ConnectOptions.rst new file mode 100644 index 0000000..1a9b4c3 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/struct-ConnectOptions.rst @@ -0,0 +1,62 @@ +ConnectOptions +========================== + +.. cpp:class:: ConnectOptions + + :header-file: #include + :namespace: datasystem + + 用于配置对象客户端的初始化参数的结构体。 + + **公共成员** + + .. cpp:member:: std::string host + + 数据系统 Worker 的主机 IP 地址。 + + .. cpp:member:: int32_t port + + 数据系统 Worker 的主机 IP 端口号。 + + .. cpp:member:: int32_t connectTimeoutMs = 60 * 1000; + + 客户端连接和请求超时时间,单位为毫秒。默认值:60'000。 + + .. cpp:member:: std::string clientPublicKey = ""; + + 用于 curve 认证的客户端公钥。默认值:""。 + + .. cpp:member:: SensitiveValue clientPrivateKey = ""; + + 用于 curve 认证的客户端私钥。默认值:"" + + .. cpp:member:: std::string serverPublicKey = ""; + + 用于 curve 认证的服务端公钥。默认值:"" + + .. cpp:member:: std::string accessKey = ""; + + AK/SK 授权使用的访问密钥。默认值:"" + + .. cpp:member:: SensitiveValue secretKey = ""; + + AK/SK 授权的密钥。默认值:"" + + .. cpp:member:: std::string tenantId = ""; + + 租户 ID。默认值:"" + + .. cpp:member:: bool enableCrossNodeConnection = false; + + 如果为 true,允许客户端在与当前数据系统Worker 连接异常时自动切换到备用节点。默认值:false + + **公共函数** + + .. cpp:function:: void SetAkSkAuth(const std::string &accessKey, const SensitiveValue &secretKey, const std::string &tenantId) + + 设置 AK/SK 用于后续请求访问。 + + 参数: + - **accessKey** - 设置授权使用的访问密钥。 + - **accessKey** - 设置AK/SK 授权的密钥。 + - **tenantId** - 租户ID。 diff --git a/docs/source_zh_cn/development-guide/api/cpp/struct-SetParam.rst b/docs/source_zh_cn/development-guide/api/cpp/struct-SetParam.rst new file mode 100644 index 0000000..a0f49b7 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/cpp/struct-SetParam.rst @@ -0,0 +1,23 @@ +SetParam +========================== + +.. cpp:class:: SetParam + + :header-file: #include + :namespace: datasystem + + 用于配置对象客户端的初始化参数的结构体。 + + **公共成员** + + .. cpp:member:: WriteMode writeMode = WriteMode::NONE_L2_CACHE + + 设置数据可靠性级别,默认不写入二级缓存。 + + .. cpp:member:: uint32_t ttlSecond = 0 + + 设置Key的存活时间,单位:秒。Key达到存活时间后系统自动将其删除。默认为0,表示不设置存活时间,Key会一直存在直到显式调用Del接口。注意:系统重启后存活时间计时将重新开始。 + + .. cpp:member:: CacheType cacheType = CacheType::MEMORY + + 配置缓存介质,默认为内存缓存介质。 diff --git a/docs/source_zh_cn/development-guide/api/index.md b/docs/source_zh_cn/development-guide/api/index.md new file mode 100644 index 0000000..d8049a8 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/index.md @@ -0,0 +1,16 @@ +# API + +```{eval-rst} +.. toctree:: + :glob: + :maxdepth: 1 + :hidden: + + python/index + cpp/index +``` + +本节向您介绍开发openYuanrong datasystem的相关 API。 + +- [Python](./python/index.rst) +- [C++](./cpp/index.rst) \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.hetero.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.hetero.rst similarity index 33% rename from docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.hetero.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.hetero.rst index 9179ea2..be01015 100644 --- a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.hetero.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.hetero.rst @@ -1,9 +1,9 @@ -datasystem.DsClient.hetero +datasystem.DsClient.hetero ========================== .. py:method:: datasystem.DsClient.hetero - 获取异构对象客户端。 + 获取异构对象客户端实例。 返回: - - **HeteroClient** - 返回异构对象客户端 :class:`datasystem.hetero_client.HeteroClient` 。 \ No newline at end of file + :class:`datasystem.hetero_client.HeteroClient` 异构对象客户端实例。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.init.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.init.rst similarity index 88% rename from docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.init.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.init.rst index 635d078..02c97be 100644 --- a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.init.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.init.rst @@ -1,4 +1,4 @@ -datasystem.DsClient.init +datasystem.DsClient.init ======================== .. py:method:: datasystem.DsClient.init diff --git a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.kv.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.kv.rst similarity index 36% rename from docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.kv.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.kv.rst index 63cd710..452f985 100644 --- a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.kv.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.kv.rst @@ -1,9 +1,9 @@ -datasystem.DsClient.kv +datasystem.DsClient.kv ====================== .. py:method:: datasystem.DsClient.kv - 获取 KV 客户端。 + 获取异构对象客户端实例。 返回: - - **KvClient** - 返回 KV 客户端 :class:`datasystem.kv_client.KVClient` 。 + :class:`datasystem.kv_client.KVClient` KV客户端实例。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.object.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.object.rst similarity index 34% rename from docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.object.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.object.rst index e3d8ab5..cdd9b24 100644 --- a/docs/source_zh_cn/api/api_python/ds_client/DsClient/datasystem.DsClient.object.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.object.rst @@ -1,9 +1,9 @@ -datasystem.DsClient.object +datasystem.DsClient.object ========================== .. py:method:: datasystem.DsClient.object - 获取 object 客户端。 + 获取异构对象客户端实例。 返回: - - **ObjectClient** - 返回 object 客户端 :class:`datasystem.object_client.ObjectClient`。 + :class:`datasystem.object_client.ObjectClient` KV客户端实例。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/ds_client/datasystem.DsClient.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.rst similarity index 73% rename from docs/source_zh_cn/api/api_python/ds_client/datasystem.DsClient.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.rst index 1b4419f..f4f183f 100644 --- a/docs/source_zh_cn/api/api_python/ds_client/datasystem.DsClient.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsClient.rst @@ -20,9 +20,24 @@ datasystem.DsClient 输出: DsClient -.. dscnautosummary:: - :toctree: DsClient - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`init ` + - 初始化数据系统客户端 + * - :doc:`hetero ` + - 获取数据系统异构缓存客户端。 + * - :doc:`kv ` + - 获取数据系统KV缓存客户端。 + * - :doc:`object ` + - 获取数据系统对象缓存客户端。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.DsClient.init datasystem.DsClient.hetero diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_dev_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_dev_delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_dev_delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_dev_delete.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_mget_h2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_mget_h2d.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_mget_h2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_mget_h2d.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_mset_d2h.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_mset_d2h.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.async_mset_d2h.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.async_mset_d2h.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.delete.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_delete.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_local_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_local_delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_local_delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_local_delete.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_mget.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_mget.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_mget.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_mget.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_mset.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_mset.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_mset.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_mset.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_recv.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_recv.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_recv.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_recv.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_send.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_send.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.dev_send.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.dev_send.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst similarity index 96% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst index 39a55c8..d72fa5f 100644 --- a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.get_page_attn_layerwise_d2d.rst @@ -1,5 +1,5 @@ datasystem.DsTensorClient.get_page_attn_layerwise_d2d -=================================================== +===================================================== .. py:method:: datasystem.DsTensorClient.get_page_attn_layerwise_d2d(keys, layer_tensors, block_ids) diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.init.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.init.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.init.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.init.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mget_h2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mget_h2d.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mget_h2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mget_h2d.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst similarity index 96% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst index 36e930e..1327be8 100644 --- a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mget_page_attn_blockwise_h2d.rst @@ -1,5 +1,5 @@ datasystem.DsTensorClient.mget_page_attn_blockwise_h2d -============================================== +====================================================== .. py:method:: datasystem.DsTensorClient.mget_page_attn_blockwise_h2d(keys, layer_tensors, block_ids, sub_timeout_ms) diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mset_d2h.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mset_d2h.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mset_d2h.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mset_d2h.rst diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst similarity index 95% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst index c908688..4defa66 100644 --- a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.mset_page_attn_blockwise_d2h.rst @@ -1,5 +1,5 @@ datasystem.DsTensorClient.mset_page_attn_blockwise_d2h -============================================== +====================================================== .. py:method:: datasystem.DsTensorClient.mset_page_attn_blockwise_d2h(keys, layer_tensors, block_ids) diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst similarity index 96% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst index f70b85a..13be537 100644 --- a/docs/source_zh_cn/api/api_python/ds_tensor_client/DsTensorClient/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.put_page_attn_layerwise_d2d.rst @@ -1,5 +1,5 @@ datasystem.DsTensorClient.put_page_attn_layerwise_d2d -================================================= +===================================================== .. py:method:: datasystem.DsTensorClient.put_page_attn_layerwise_d2d(keys, layer_tensors, block_ids) diff --git a/docs/source_zh_cn/api/api_python/ds_tensor_client/datasystem.DsTensorClient.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.rst similarity index 36% rename from docs/source_zh_cn/api/api_python/ds_tensor_client/datasystem.DsTensorClient.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.rst index e4eebc6..14c607f 100644 --- a/docs/source_zh_cn/api/api_python/ds_tensor_client/datasystem.DsTensorClient.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.DsTensorClient.rst @@ -17,9 +17,50 @@ datasystem.DsTensorClient 输出: DsTensorClient -.. dscnautosummary:: - :toctree: DsTensorClient - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`init ` + - 初始化数据系统客户端。 + * - :doc:`mset_d2h ` + - 将 device 的数据写入到 host 中。 + * - :doc:`mget_h2d ` + - 从 host 中获取数据并写入 device 中。 + * - :doc:`async_mset_d2h ` + - 将 device 的数据写入到 host 中的异步接口。 + * - :doc:`async_mget_h2d ` + - 从 host 中获取数据并写入 device 中的异步接口。 + * - :doc:`delete ` + - 删除 host 中的 key。 + * - :doc:`dev_send ` + - 订阅发布到数据系统的异构对象,并接收数据写入 tensors。 + * - :doc:`dev_recv ` + - 将 device 上的内存发布为数据系统的异构对象,发布后的异构对象可通过 dev_recv 获取。 + * - :doc:`dev_mset ` + - 通过数据系统缓存 Device 上的数据。 + * - :doc:`dev_mget ` + - 获取 device 中的数据,并写入到 Tensor 中。 + * - :doc:`dev_local_delete ` + - 从数据系统删除本节点上此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`dev_delete ` + - 从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`async_dev_delete ` + - 从数据系统删除此 key 的元数据的异步接口,删除成功后不再管理此 key 对应的 device 内存。 + * - :doc:`put_page_attn_layerwise_d2d ` + - 将 PagedAttention 的层级 Tensor 发布为数据系统的异构对象。发布后的异构对象可通过 get_page_attn_layerwise_d2d 获取。 + * - :doc:`get_page_attn_layerwise_d2d ` + - 根据 key 获取缓存在数据系统的 PagedAttention 的层级 Tensor。 + * - :doc:`mset_page_attn_blockwise_d2h ` + - 将 PagedAttention 的层级 Tensor 异步写入 Host 中。 + * - :doc:`mget_page_attn_blockwise_h2d ` + - 从 Host 中获取 PagedAttention 的层级 Tensor 并写入 Device 中。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.DsTensorClient.init datasystem.DsTensorClient.mset_d2h @@ -33,6 +74,7 @@ datasystem.DsTensorClient datasystem.DsTensorClient.dev_mget datasystem.DsTensorClient.dev_local_delete datasystem.DsTensorClient.dev_delete + datasystem.DsTensorClient.async_dev_delete datasystem.DsTensorClient.put_page_attn_layerwise_d2d datasystem.DsTensorClient.get_page_attn_layerwise_d2d datasystem.DsTensorClient.mset_page_attn_blockwise_d2h diff --git a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Blob.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.rst similarity index 54% rename from docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Blob.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.rst index e0b5a3e..aecea1f 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Blob.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.rst @@ -12,9 +12,20 @@ datasystem.hetero_client.Blob 输出: Blob -.. dscnautosummary:: - :toctree: Blob - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`set_dev_ptr ` + - 指定 Device 内存指针地址。 + * - :doc:`set_size ` + - 指定 Device 内存地址空间大小。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.hetero_client.Blob.set_dev_ptr datasystem.hetero_client.Blob.set_size diff --git a/docs/source_zh_cn/api/api_python/hetero_client/Blob/datasystem.hetero_client.Blob.set_dev_ptr.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.set_dev_ptr.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/Blob/datasystem.hetero_client.Blob.set_dev_ptr.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.set_dev_ptr.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/Blob/datasystem.hetero_client.Blob.set_size.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.set_size.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/Blob/datasystem.hetero_client.Blob.set_size.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Blob.set_size.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.append_blob.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.append_blob.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.append_blob.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.append_blob.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.get_blob_list.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.get_blob_list.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.get_blob_list.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.get_blob_list.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.DeviceBlobList.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.rst similarity index 51% rename from docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.DeviceBlobList.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.rst index d9f11e3..2aadd1e 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.DeviceBlobList.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.rst @@ -12,9 +12,22 @@ datasystem.hetero_client.DeviceBlobList 输出: DeviceBlobList -.. dscnautosummary:: - :toctree: DeviceBlobList - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`append_blob ` + - 添加一个 :class:`datasystem.hetero_client.Blob`, 即一段 Device 内存信息。 + * - :doc:`set_dev_idx ` + - 指定 DeviceBlobList 中内存归属的 NPU 卡的 ID。 + * - :doc:`get_blob_list ` + - 获取 :class:`datasystem.hetero_client.Blob` 列表。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.hetero_client.DeviceBlobList.append_blob datasystem.hetero_client.DeviceBlobList.set_dev_idx diff --git a/docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.set_dev_idx.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.set_dev_idx.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/DeviceBlobList/datasystem.hetero_client.DeviceBlobList.set_dev_idx.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.DeviceBlobList.set_dev_idx.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/Future/datasystem.hetero_client.Future.get.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Future.get.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/Future/datasystem.hetero_client.Future.get.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Future.get.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Future.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Future.rst similarity index 49% rename from docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Future.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Future.rst index eac1e53..d5427a6 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.Future.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.Future.rst @@ -8,8 +8,17 @@ datasystem.hetero_client.Future 输出: Future -.. dscnautosummary:: - :toctree: Future - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`get ` + - 获取异步任务的执行结果。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.hetero_client.Future.get diff --git a/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_dev_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_dev_delete.rst new file mode 100644 index 0000000..f96d0fe --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_dev_delete.rst @@ -0,0 +1,17 @@ +datasystem.hetero_client.HeteroClient.async_dev_delete +====================================================== + +.. py:method:: datasystem.hetero_client.HeteroClient.async_dev_delete(keys) + +异步接口,从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。可通过返回的Future查询删除结果。 + + async_dev_delete 与 dev_mset / dev_mget 接口配套使用。 + + 参数: + - **keys** (list) - host 的 key 列表。约束:传入的key的数量不能超过1万。 + + 返回: + - **Future** (Future) - 可通过该Future对象查询异步请求执行结果。当调用Future的get方法时,如果删除key存在部分失败,则返回失败的列表;如果全部失败,则抛出RuntimeError异常。 + + 异常: + - **TypeError** - 输入参数存在非法值。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst similarity index 95% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst index 1c4ca79..9d77c7b 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mget_h2d.rst @@ -1,5 +1,5 @@ datasystem.hetero_client.HeteroClient.async_mget_h2d -============================================== +==================================================== .. py:method:: datasystem.hetero_client.HeteroClient.async_mget_h2d(keys, data_blob_list, sub_timeout_ms) diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst similarity index 95% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst index 9d44c3c..75e6679 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.async_mset_d2h.rst @@ -1,5 +1,5 @@ datasystem.hetero_client.HeteroClient.async_mset_d2h -============================================== +==================================================== .. py:method:: datasystem.hetero_client.HeteroClient.async_mset_d2h(keys, data_blob_list, set_param) diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.delete.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_delete.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_local_delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_local_delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_local_delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_local_delete.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_mget.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_mget.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_mget.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_mget.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_mset.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_mset.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_mset.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_mset.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_publish.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_publish.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_publish.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_publish.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_subscribe.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_subscribe.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.dev_subscribe.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.dev_subscribe.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.generate_key.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.generate_key.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.generate_key.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.generate_key.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.get_meta_info.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.get_meta_info.rst similarity index 93% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.get_meta_info.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.get_meta_info.rst index 8bf2d8c..41deb41 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.get_meta_info.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.get_meta_info.rst @@ -1,5 +1,5 @@ datasystem.hetero_client.HeteroClient.get_meta_info -============================================== +=================================================== .. py:method:: datasystem.hetero_client.HeteroClient.get_meta_info(self, keys, is_dev_key) diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.init.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.init.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.init.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.init.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.mget_h2d.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.mget_h2d.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.mget_h2d.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.mget_h2d.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.mset_d2h.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.mset_d2h.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/hetero_client/HeteroClient/datasystem.hetero_client.HeteroClient.mset_d2h.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.mset_d2h.rst diff --git a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.HeteroClient.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.rst similarity index 45% rename from docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.HeteroClient.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.rst index 4fdf685..d4610c3 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.HeteroClient.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.HeteroClient.rst @@ -20,9 +20,46 @@ datasystem.hetero_client.HeteroClient 输出: HeteroClient -.. dscnautosummary:: - :toctree: HeteroClient - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`init ` + - 初始化异构对象客户端。 + * - :doc:`mget_h2d ` + - 从 host 中获取数据并写入 device 中。 + * - :doc:`mset_d2h ` + - 将 device 的数据写入到 host 中。 + * - :doc:`async_mget_h2d ` + - 从 host 中获取数据并写入 device 中的异步接口。 + * - :doc:`async_mset_d2h ` + - 将 device 的数据写入到 host 中的异步接口。 + * - :doc:`delete ` + - 删除 host 中的 key,与 mget_h2d / mset_d2h 配套使用。 + * - :doc:`dev_publish ` + - 将 device 上的内存发布为数据系统的异构对象。发布后的异构对象可通过 dev_subscribe 获取。 + * - :doc:`dev_subscribe ` + - 订阅发布到数据系统的异构对象,并接收数据写入 data_blob_list。数据通过 device to device 通道直接传输。 + * - :doc:`dev_mset ` + - 通过数据系统缓存 Device 上的数据。 + * - :doc:`dev_mget ` + - 获取 device 中的数据。 + * - :doc:`dev_local_delete ` + - 从数据系统删除本节点上此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`dev_delete ` + - 从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`async_dev_delete ` + - 从数据系统删除此 key 的元数据的异步接口,删除成功后不再管理此 key 对应的 device 内存。 + * - :doc:`generate_key ` + - 生成一个带数据系统 Worker UUID 的 key。 + * - :doc:`get_meta_info ` + - 获取keys 对应的元数据信息。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.hetero_client.HeteroClient.init datasystem.hetero_client.HeteroClient.mget_h2d @@ -36,5 +73,6 @@ datasystem.hetero_client.HeteroClient datasystem.hetero_client.HeteroClient.dev_mget datasystem.hetero_client.HeteroClient.dev_local_delete datasystem.hetero_client.HeteroClient.dev_delete + datasystem.hetero_client.HeteroClient.async_dev_delete datasystem.hetero_client.HeteroClient.generate_key datasystem.hetero_client.HeteroClient.get_meta_info diff --git a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.MetaInfo.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.MetaInfo.rst similarity index 73% rename from docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.MetaInfo.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.MetaInfo.rst index 38e4d4d..b02c89f 100644 --- a/docs/source_zh_cn/api/api_python/hetero_client/datasystem.hetero_client.MetaInfo.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.hetero_client.MetaInfo.rst @@ -1,5 +1,5 @@ datasystem.hetero_client.MetaInfo -============================= +================================= .. py:class:: datasystem.hetero_client.MetaInfo(blob_size_list) @@ -10,7 +10,3 @@ datasystem.hetero_client.MetaInfo 输出: MetaInfo - -.. dscnautosummary:: - :toctree: MetaInfo - :nosignatures: \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.delete.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.delete.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.delete.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.delete.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.exist.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.exist.rst similarity index 92% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.exist.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.exist.rst index 2c89a21..5b0f6e7 100644 --- a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.exist.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.exist.rst @@ -1,5 +1,5 @@ datasystem.kv_client.KVClient.exist -================================= +=================================== .. py:method:: datasystem.kv_client.KVClient.exist(self, keys) diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.expire.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.expire.rst similarity index 94% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.expire.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.expire.rst index c779bbb..9a6626f 100644 --- a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.expire.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.expire.rst @@ -1,5 +1,5 @@ datasystem.kv_client.KVClient.expire -================================= +==================================== .. py:method:: datasystem.kv_client.KVClient.expire(self, keys, ttl_second) diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.generate_key.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.generate_key.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.generate_key.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.generate_key.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.get.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.get.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.get.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.get.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.get_read_only_buffers.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.get_read_only_buffers.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.get_read_only_buffers.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.get_read_only_buffers.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.init.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.init.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.init.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.init.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.mset.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.mset.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.mset.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.mset.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.msettx.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.msettx.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.msettx.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.msettx.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.read.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.read.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.read.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.read.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.KVClient.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.rst similarity index 52% rename from docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.KVClient.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.rst index 36d833b..39b0277 100644 --- a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.KVClient.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.rst @@ -20,18 +20,50 @@ datasystem.kv_client.KVClient 输出: KVClient -.. dscnautosummary:: - :toctree: KVClient - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`init ` + - 初始化KV缓存客户端以连接到 Worker 。 + * - :doc:`set ` + - 设置键的值。 + * - :doc:`set_value ` + - 设置键的值,键由系统生成并返回。 + * - :doc:`mset ` + - 批量设置键值对。 + * - :doc:`msettx ` + - 批量设置键值对(事务操作),它保证所有的键要么都成功设置,要么都失败。 + * - :doc:`get_read_only_buffers ` + - 获取所有给定键的值。 + * - :doc:`get ` + - 获取所有给定键的值。 + * - :doc:`read ` + - 读取指定偏移量的数据。 + * - :doc:`delete ` + - 初始化KV缓存客户端以连接到 Worker 。 + * - :doc:`generate_key ` + - 生成一个带数据系统 Worker UUID 的 key。 + * - :doc:`exist ` + - 查看 key 在数据系统中是否存在。 + * - :doc:`expire ` + - 为一组键设置过期生命周期,返回函数操作状态及设置失败的键列表。 + +.. toctree:: + :maxdepth: 1 + :hidden: - datasystem.kv_client.KVClient.delete - datasystem.kv_client.KVClient.generate_key - datasystem.kv_client.KVClient.get_read_only_buffers - datasystem.kv_client.KVClient.get datasystem.kv_client.KVClient.init + datasystem.kv_client.KVClient.set + datasystem.kv_client.KVClient.set_value datasystem.kv_client.KVClient.mset datasystem.kv_client.KVClient.msettx + datasystem.kv_client.KVClient.get_read_only_buffers + datasystem.kv_client.KVClient.get datasystem.kv_client.KVClient.read - datasystem.kv_client.KVClient.set - datasystem.kv_client.KVClient.set_value + datasystem.kv_client.KVClient.delete + datasystem.kv_client.KVClient.generate_key + datasystem.kv_client.KVClient.exist datasystem.kv_client.KVClient.expire diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.set.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.set.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.set.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.set.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.set_value.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.set_value.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/KVClient/datasystem.kv_client.KVClient.set_value.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.KVClient.set_value.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.immutable_data.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.immutable_data.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.immutable_data.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.immutable_data.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.rlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.rlatch.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.rlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.rlatch.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.ReadOnlyBuffer.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.rst similarity index 40% rename from docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.ReadOnlyBuffer.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.rst index 484e3e1..e3a27b4 100644 --- a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.ReadOnlyBuffer.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.rst @@ -8,9 +8,22 @@ datasystem.kv_client.ReadOnlyBuffer 输出: ReadOnlyBuffer。 -.. dscnautosummary:: - :toctree: ReadOnlyBuffer - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`immutable_data ` + - 获取一个不可变的数据内存视图。 + * - :doc:`rlatch ` + - 获取读锁以保护缓冲区免受并发写操作。 + * - :doc:`unrlatch ` + - 释放读锁。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.kv_client.ReadOnlyBuffer.immutable_data datasystem.kv_client.ReadOnlyBuffer.rlatch diff --git a/docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.unrlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.unrlatch.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/ReadOnlyBuffer/datasystem.kv_client.ReadOnlyBuffer.unrlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadOnlyBuffer.unrlatch.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.ReadParam.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadParam.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.ReadParam.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.ReadParam.rst diff --git a/docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.SetParam.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.SetParam.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/kv_client/datasystem.kv_client.SetParam.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.kv_client.SetParam.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.get_size.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.get_size.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.get_size.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.get_size.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.immutable_data.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.immutable_data.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.immutable_data.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.immutable_data.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.invalidate_buffer.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.invalidate_buffer.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.invalidate_buffer.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.invalidate_buffer.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.memory_copy.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.memory_copy.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.memory_copy.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.memory_copy.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.mutable_data.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.mutable_data.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.mutable_data.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.mutable_data.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.publish.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.publish.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.publish.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.publish.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.rlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rlatch.rst similarity index 91% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.rlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rlatch.rst index 8340457..08e53a0 100644 --- a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.rlatch.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rlatch.rst @@ -3,7 +3,7 @@ datasystem.object_client.Buffer.rlatch .. py:method:: datasystem.object_client.Buffer.rlatch(timeout_sec=60) - 对对象Buffer加读锁。 + 对共享内存Buffer加读锁。 参数: - **timeout_sec** (int) - 尝试加锁的最大等待时间(单位为秒)。默认值: ``60`` 。 diff --git a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.Buffer.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rst similarity index 30% rename from docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.Buffer.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rst index 5694d8a..8907614 100644 --- a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.Buffer.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.rst @@ -8,9 +8,38 @@ datasystem.object_client.Buffer 输出: Buffer。 -.. dscnautosummary:: - :toctree: Buffer - :nosignatures: + **方法**: + + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`wlatch ` + - 对共享内存Buffer加写锁。 + * - :doc:`unwlatch ` + - 对共享内存Buffer解写锁。 + * - :doc:`rlatch ` + - 对共享内存Buffer加读锁。 + * - :doc:`unrlatch ` + - 对共享内存Buffer解读锁。 + * - :doc:`mutable_data ` + - 获取Buffer的可读写 `memoryview`。 + * - :doc:`immutable_data ` + - 获取Buffer的只读 `memoryview`。 + * - :doc:`memory_copy ` + - 将 `value` 中的数据拷贝到Buffer中 + * - :doc:`publish ` + - 将Buffer中的数据发布到数据系统中。 + * - :doc:`seal ` + - 将Buffer中的数据发布到数据系统中,发布成功之后Buffer所对应的对象的值将无法再被修改。 + * - :doc:`invalidate_buffer ` + - 使当前主机上的Buffer数据无效化。 + * - :doc:`get_size ` + - 获取对象buffer的大小。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.object_client.Buffer.wlatch datasystem.object_client.Buffer.unwlatch diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.seal.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.seal.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.seal.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.seal.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unrlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unrlatch.rst similarity index 84% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unrlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unrlatch.rst index 88beb29..a1fc802 100644 --- a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unrlatch.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unrlatch.rst @@ -3,7 +3,7 @@ datasystem.object_client.Buffer.unrlatch .. py:method:: datasystem.object_client.Buffer.unrlatch - 对对象Buffer解读锁。 + 对共享内存Buffer解读锁。 异常: - **RuntimeError** - 解读锁失败。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unwlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unwlatch.rst similarity index 84% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unwlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unwlatch.rst index 3b2d022..0d44fec 100644 --- a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.unwlatch.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.unwlatch.rst @@ -3,7 +3,7 @@ datasystem.object_client.Buffer.unwlatch .. py:method:: datasystem.object_client.Buffer.unwlatch - 对对象Buffer解写锁。 + 对共享内存Buffer解写锁。 异常: - **RuntimeError** - 解写锁失败。 \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.wlatch.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.wlatch.rst similarity index 91% rename from docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.wlatch.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.wlatch.rst index 9266d21..ccc1a2c 100644 --- a/docs/source_zh_cn/api/api_python/object_client/Buffer/datasystem.object_client.Buffer.wlatch.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.Buffer.wlatch.rst @@ -3,7 +3,7 @@ datasystem.object_client.Buffer.wlatch .. py:method:: datasystem.object_client.Buffer.wlatch(timeout_sec=60) - 对对象Buffer加写锁。 + 对共享内存Buffer加写锁。 参数: - **timeout_sec** (int) - 尝试加锁的最大等待时间(单位为秒)。默认值: ``60`` 。 diff --git a/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.CacheType.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.CacheType.rst new file mode 100644 index 0000000..5100a1e --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.CacheType.rst @@ -0,0 +1,15 @@ +datasystem.object_client.CacheType +======================================== + +.. py:class:: datasystem.object_client.CacheType + + `CacheType` 类定义对象的缓存介质。 + + 目前,支持以下 `CacheType`: + + =================================== ================================================================== + 定义 说明 + =================================== ================================================================== + `CacheType::MEMORY` 内存介质,数据将缓存在共享内存上。 + `CacheType::DISK` 磁盘介质,数据将缓存在磁盘上。 + =================================== ================================================================== \ No newline at end of file diff --git a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.ConsistencyType.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ConsistencyType.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.ConsistencyType.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ConsistencyType.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.create.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.create.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.create.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.create.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.g_decrease_ref.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.g_decrease_ref.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.g_decrease_ref.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.g_decrease_ref.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.g_increase_ref.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.g_increase_ref.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.g_increase_ref.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.g_increase_ref.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.generate_object_key.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.generate_object_key.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.generate_object_key.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.generate_object_key.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.get.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.get.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.get.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.get.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.init.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.init.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.init.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.init.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.put.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.put.rst similarity index 96% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.put.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.put.rst index 3b3a34f..0462feb 100644 --- a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.put.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.put.rst @@ -3,7 +3,7 @@ datasystem.object_client.ObjectClient.put .. py:method:: datasystem.object_client.ObjectClient.put(object_key, value, write_mode=WriteMode.NONE_L2_CACHE, consistency_type=ConsistencyType.PRAM, nested_object_keys=None) - 将对象写入到数据系统中。 + 将对象缓存到数据系统中。 参数: - **object_key** (str) - 对象 key。 diff --git a/docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.query_global_ref_num.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.query_global_ref_num.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/ObjectClient/datasystem.object_client.ObjectClient.query_global_ref_num.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.query_global_ref_num.rst diff --git a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.ObjectClient.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.rst similarity index 57% rename from docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.ObjectClient.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.rst index e1446ff..4f1767a 100644 --- a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.ObjectClient.rst +++ b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.ObjectClient.rst @@ -19,9 +19,30 @@ datasystem.object_client.ObjectClient 输出: ObjectClient -.. dscnautosummary:: - :toctree: ObjectClient - :nosignatures: + .. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - :doc:`init ` + - 初始化对象缓存客户端。 + * - :doc:`create ` + - 创建对象buffer。 + * - :doc:`put ` + - 将对象缓存到数据系统中。 + * - :doc:`get ` + - 获取给定列表对象 key 的Buffer。 + * - :doc:`g_increase_ref ` + - 增加给定列表对象 key 的全局引用计数。 + * - :doc:`g_decrease_ref ` + - 减少给定列表对象 key 的全局引用计数。 + * - :doc:`query_global_ref_num ` + - 查询对象全局引用计数。 + * - :doc:`generate_object_key ` + - 生成一个带数据系统Worker UUID的对象 key。 + +.. toctree:: + :maxdepth: 1 + :hidden: datasystem.object_client.ObjectClient.init datasystem.object_client.ObjectClient.create diff --git a/docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.WriteMode.rst b/docs/source_zh_cn/development-guide/api/python/datasystem.object_client.WriteMode.rst similarity index 100% rename from docs/source_zh_cn/api/api_python/object_client/datasystem.object_client.WriteMode.rst rename to docs/source_zh_cn/development-guide/api/python/datasystem.object_client.WriteMode.rst diff --git a/docs/source_zh_cn/development-guide/api/python/index.rst b/docs/source_zh_cn/development-guide/api/python/index.rst new file mode 100644 index 0000000..d132d86 --- /dev/null +++ b/docs/source_zh_cn/development-guide/api/python/index.rst @@ -0,0 +1,181 @@ +Python +============================== + + +.. toctree:: + :glob: + :hidden: + :maxdepth: 1 + + datasystem.DsClient + datasystem.DsTensorClient + datasystem.hetero_client.HeteroClient + datasystem.hetero_client.Blob + datasystem.hetero_client.DeviceBlobList + datasystem.hetero_client.Future + datasystem.hetero_client.MetaInfo + datasystem.kv_client.KVClient + datasystem.kv_client.ReadOnlyBuffer + datasystem.kv_client.SetParam + datasystem.kv_client.ReadParam + datasystem.object_client.ObjectClient + datasystem.object_client.Buffer + datasystem.object_client.ConsistencyType + datasystem.object_client.WriteMode + datasystem.object_client.CacheType + +DsClient:聚合各语义的客户端 +----------------------------------- + +.. list-table:: + :widths: 30 70 + :header-rows: 0 + + * - :doc:`datasystem.DsClient.init ` + - 初始化数据系统客户端 + * - :doc:`datasystem.DsClient.hetero ` + - 获取数据系统异构缓存客户端。 + * - :doc:`datasystem.DsClient.kv ` + - 获取数据系统KV缓存客户端。 + * - :doc:`datasystem.DsClient.object ` + - 获取数据系统对象缓存客户端。 + + +Tensor接口 +----------------- + +.. list-table:: + :header-rows: 0 + :widths: 30 70 + + * - :doc:`datasystem.DsTensorClient.init ` + - 初始化数据系统客户端。 + * - :doc:`datasystem.DsTensorClient.mset_d2h ` + - 将 device 的数据写入到 host 中。 + * - :doc:`datasystem.DsTensorClient.mget_h2d ` + - 从 host 中获取数据并写入 device 中。 + * - :doc:`datasystem.DsTensorClient.async_mset_d2h ` + - 将 device 的数据写入到 host 中的异步接口。 + * - :doc:`datasystem.DsTensorClient.async_mget_h2d ` + - 从 host 中获取数据并写入 device 中的异步接口。 + * - :doc:`datasystem.DsTensorClient.delete ` + - 删除 host 中的 key。 + * - :doc:`datasystem.DsTensorClient.dev_send ` + - 订阅发布到数据系统的异构对象,并接收数据写入 tensors。 + * - :doc:`datasystem.DsTensorClient.dev_recv ` + - 将 device 上的内存发布为数据系统的异构对象,发布后的异构对象可通过 dev_recv 获取。 + * - :doc:`datasystem.DsTensorClient.dev_mset ` + - 通过数据系统缓存 Device 上的数据。 + * - :doc:`datasystem.DsTensorClient.dev_mget ` + - 获取 device 中的数据,并写入到 Tensor 中。 + * - :doc:`datasystem.DsTensorClient.dev_local_delete ` + - 从数据系统删除本节点上此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.DsTensorClient.dev_delete ` + - 从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.DsTensorClient.async_dev_delete ` + - 从数据系统删除此 key 的元数据的异步接口,删除成功后不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.DsTensorClient.put_page_attn_layerwise_d2d ` + - 将 PagedAttention 的层级 Tensor 发布为数据系统的异构对象。发布后的异构对象可通过 get_page_attn_layerwise_d2d 获取。 + * - :doc:`datasystem.DsTensorClient.get_page_attn_layerwise_d2d ` + - 根据 key 获取缓存在数据系统的 PagedAttention 的层级 Tensor。 + * - :doc:`datasystem.DsTensorClient.mset_page_attn_blockwise_d2h ` + - 将 PagedAttention 的层级 Tensor 异步写入 Host 中。 + * - :doc:`datasystem.DsTensorClient.mget_page_attn_blockwise_h2d ` + - 从 Host 中获取 PagedAttention 的层级 Tensor 并写入 Device 中。 + + +异构对象接口 +----------------- + +.. list-table:: + :header-rows: 0 + :widths: 30 70 + + * - :doc:`datasystem.hetero_client.HeteroClient.init ` + - 初始化异构对象客户端。 + * - :doc:`datasystem.hetero_client.HeteroClient.mget_h2d ` + - 从 host 中获取数据并写入 device 中。 + * - :doc:`datasystem.hetero_client.HeteroClient.mset_d2h ` + - 将 device 的数据写入到 host 中。 + * - :doc:`datasystem.hetero_client.HeteroClient.async_mget_h2d ` + - 从 host 中获取数据并写入 device 中的异步接口。 + * - :doc:`datasystem.hetero_client.HeteroClient.async_mset_d2h ` + - 将 device 的数据写入到 host 中的异步接口。 + * - :doc:`datasystem.hetero_client.HeteroClient.delete ` + - 删除 host 中的 key,与 mget_h2d / mset_d2h 配套使用。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_publish ` + - 将 device 上的内存发布为数据系统的异构对象。发布后的异构对象可通过 dev_subscribe 获取。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_subscribe ` + - 订阅发布到数据系统的异构对象,并接收数据写入 data_blob_list。数据通过 device to device 通道直接传输。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_mset ` + - 通过数据系统缓存 Device 上的数据。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_mget ` + - 获取 device 中的数据。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_local_delete ` + - 从数据系统删除本节点上此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.hetero_client.HeteroClient.dev_delete ` + - 从数据系统删除此 key 的元数据,不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.hetero_client.HeteroClient.async_dev_delete ` + - 从数据系统删除此 key 的元数据的异步接口,删除成功后不再管理此 key 对应的 device 内存。 + * - :doc:`datasystem.hetero_client.HeteroClient.generate_key ` + - 生成一个带数据系统 Worker UUID 的 key。 + * - :doc:`datasystem.hetero_client.HeteroClient.get_meta_info ` + - 获取keys 对应的元数据信息。 + + +KV接口 +----------------- + +.. list-table:: + :header-rows: 0 + :widths: 30 70 + + * - :doc:`datasystem.kv_client.KVClient.init ` + - 初始化KV缓存客户端以连接到 Worker 。 + * - :doc:`datasystem.kv_client.KVClient.set ` + - 设置键的值。 + * - :doc:`datasystem.kv_client.KVClient.set_value ` + - 设置键的值,键由系统生成并返回。 + * - :doc:`datasystem.kv_client.KVClient.mset ` + - 批量设置键值对。 + * - :doc:`datasystem.kv_client.KVClient.msettx ` + - 批量设置键值对(事务操作),它保证所有的键要么都成功设置,要么都失败。 + * - :doc:`datasystem.kv_client.KVClient.get_read_only_buffers ` + - 获取所有给定键的值。 + * - :doc:`datasystem.kv_client.KVClient.get ` + - 获取所有给定键的值。 + * - :doc:`datasystem.kv_client.KVClient.read ` + - 读取指定偏移量的数据。 + * - :doc:`datasystem.kv_client.KVClient.delete ` + - 初始化KV缓存客户端以连接到 Worker 。 + * - :doc:`datasystem.kv_client.KVClient.generate_key ` + - 生成一个带数据系统 Worker UUID 的 key。 + * - :doc:`datasystem.kv_client.KVClient.exist ` + - 查看 key 在数据系统中是否存在。 + * - :doc:`datasystem.kv_client.KVClient.expire ` + - 为一组键设置过期生命周期,返回函数操作状态及设置失败的键列表。 + + +对象缓存接口 +----------------- + +.. list-table:: + :header-rows: 0 + :widths: 30 70 + + * - :doc:`datasystem.object_client.ObjectClient.init ` + - 初始化对象缓存客户端。 + * - :doc:`datasystem.object_client.ObjectClient.create ` + - 创建对象buffer。 + * - :doc:`datasystem.object_client.ObjectClient.put ` + - 将对象缓存到数据系统中。 + * - :doc:`datasystem.object_client.ObjectClient.get ` + - 获取给定列表对象 key 的Buffer。 + * - :doc:`datasystem.object_client.ObjectClient.g_increase_ref ` + - 增加给定列表对象 key 的全局引用计数。 + * - :doc:`datasystem.object_client.ObjectClient.g_decrease_ref ` + - 减少给定列表对象 key 的全局引用计数。 + * - :doc:`datasystem.object_client.ObjectClient.query_global_ref_num ` + - 查询对象全局引用计数。 + * - :doc:`datasystem.object_client.ObjectClient.generate_object_key ` + - 生成一个带数据系统Worker UUID的对象 key。 \ No newline at end of file diff --git a/docs/source_zh_cn/development-guide/hetero.md b/docs/source_zh_cn/development-guide/example/hetero.md similarity index 100% rename from docs/source_zh_cn/development-guide/hetero.md rename to docs/source_zh_cn/development-guide/example/hetero.md diff --git a/docs/source_zh_cn/development-guide/example/index.md b/docs/source_zh_cn/development-guide/example/index.md new file mode 100644 index 0000000..b831d1a --- /dev/null +++ b/docs/source_zh_cn/development-guide/example/index.md @@ -0,0 +1,16 @@ +# 编程指南 + +```{eval-rst} +.. toctree:: + :hidden: + + hetero.md + kv.md + object.md +``` + +[异构对象开发指南](hetero.md) + +[KV开发指南](kv.md) + +[对象缓存开发指南](object.md) diff --git a/docs/source_zh_cn/development-guide/kv.md b/docs/source_zh_cn/development-guide/example/kv.md similarity index 100% rename from docs/source_zh_cn/development-guide/kv.md rename to docs/source_zh_cn/development-guide/example/kv.md diff --git a/docs/source_zh_cn/development-guide/object.md b/docs/source_zh_cn/development-guide/example/object.md similarity index 100% rename from docs/source_zh_cn/development-guide/object.md rename to docs/source_zh_cn/development-guide/example/object.md diff --git a/docs/source_zh_cn/development-guide/index.md b/docs/source_zh_cn/development-guide/index.md new file mode 100644 index 0000000..f0d416a --- /dev/null +++ b/docs/source_zh_cn/development-guide/index.md @@ -0,0 +1,20 @@ +# 编程接口 + +```{eval-rst} +.. toctree:: + :hidden: + + example/index.md + api/index.md +``` + +本节向您介绍如何使用 openYuanrong datasystem 开发指南以及 openYuanrong datasystem 相关 API。 + +openYuanrong datasystem 开发指南: +- [异构对象开发指南](example/hetero.md) +- [KV开发指南](example/kv.md) +- [对象缓存开发指南](example/object.md) + +openYuanrong datasystem API: +- [Python](api/python/index.rst) +- [C++](api/cpp/index.rst) diff --git a/docs/source_zh_cn/getting-started/getting_started.md b/docs/source_zh_cn/getting-started/getting_started.md new file mode 100644 index 0000000..e88cac5 --- /dev/null +++ b/docs/source_zh_cn/getting-started/getting_started.md @@ -0,0 +1,358 @@ +(overview-getting-started)= + +# 入门 + +## 安装 openYuanrong datasystem + +### pip 方式安装 + +- 安装 openYuanrong datasystem 完整发行版(包含Python SDK、C++ SDK以及命令行工具): + ```bash + pip install https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/openyuanrong_datasystem-0.5.0-cp39-cp39-manylinux_2_34_x86_64.whl + ``` + +- 仅安装 openYuanrong datasystem Python SDK(不包含C++ SDK以及命令行工具): + ```bash + pip install https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/openyuanrong_datasystem_sdk-0.5.0-cp39-cp39-manylinux_2_34_x86_64.whl + ``` + +### 源码编译方式安装 + +使用源码编译方式安装 openYuanrong datasystem 可以参考文档:[源码编译安装 openYuanrong datasystem](../installation/installation_linux.md#源码编译安装) + +## 部署 openYuanrong datasystem + +### 进程部署 + +- 准备ETCD + + openYuanrong datasystem 的集群管理依赖 ETCD,请先在后台启动单节点 ETCD(示例端口 2379): + ```bash + etcd --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://localhost:2379 & + ``` +- 一键部署 + + 安装 openYuanrong datasystem 完整发行版后,即可通过随包自带的 dscli 命令行工具一键完成集群部署。在当前启动一个监听端口号为 31501 的服务端进程: + ```bash + dscli start -w --worker_address "127.0.0.1:31501" --etcd_address "127.0.0.1:2379" + ``` + +- 一键卸载 + ```bash + dscli stop --worker_address "127.0.0.1:31501" + ``` + +更多进程部署参数与部署方式请参考文档:[openYuanrong datasystem 进程部署](../deployment/deploy.md#openyuanrong-datasystem进程部署) + +### Kubernetes 部署 + +openYuanrong datasystem 还提供了基于 Kubernetes 容器化部署方式,部署前请确保部署环境集群已就绪 Kubernetes、Helm 及可访问的 ETCD 集群。 + +- 获取 openYuanrong datasystem helm chart 包 + + 安装 openYuanrong datasystem 完整发行版后,即可通过随包自带的 dscli 命令行工具在当前路径下快速获取 helm chart 包: + ``` + dscli generate_helm_chart -o ./ + ``` + +- 编辑集群部署配置 + + openYuanrong datasystem 通过 ./datasystem/values.yaml 文件进行集群相关配置,其中必配项如下: + + ```yaml + global: + # 其他配置项... + + # 镜像仓地址 + imageRegistry: "swr.cn-south-1.myhuaweicloud.com/openeuler/" + # 镜像名字和镜像tag + images: + datasystem: "openYuanrong-datasystem:0.5.0" + + etcd: + # ETCD集群地址 + etcdAddress: "127.0.0.1:2379" + ``` + +- 集群部署 + + Helm 会提交 DaemonSet,按节点依次拉起 openYuanrong datasystem 实例: + + ```bash + helm install openyuanrong_datasystem ./datasystem + ``` + +- 集群卸载 + + ```bash + helm uninstall openyuanrong_datasystem + ``` + +更多 openYuanrong datasystem Kubernetes 高级参数配置请参考文档:[openYuanrong datasystem Kubernetes 部署](../deployment/deploy.md#openyuanrong-datasystem-kubernetes部署) + +## 开发指南 + +### 异构对象 + +异构对象实现对 HBM 内存的抽象管理,能够高效实现 D2D/H2D/D2H 的数据传输,加速 AI 训推场景数据读写。 + +主要应用场景 + +- **LLM 长序列推理 KVCache**:基于异构对象提供分布式多级缓存 (HBM/DRAM/SSD) 和高吞吐 D2D/H2D/D2H 访问能力,构建分布式 KV Cache,实现 Prefill 阶段的 KVCache 缓存以及 Prefill/Decode 实例间 KV Cache 快速传递,提升推理吞吐。 +- **模型推理实例 M->N 快速弹性**:利用异构对象的卡间直通及 P2P 数据分发能力实现模型参数快速复制。 +- **训练场景 CheckPoint 快速加载到 HBM**:各节点将待恢复的 Checkpoint 分片加载到异构对象中,利用异构对象的卡间直通传输及 P2P 数据分发能力,快速将 Checkpoint 传递到各节点 HBM。 + +通过异构对象接口,将任意二进制数据以键值对形式写入 HBM: + +```python +import acl +import os +from datasystem import Blob, DsClient, DeviceBlobList + +# hetero_dev_mset and hetero_dev_mget must be executed in different processes +# because they need to be bound to different NPUs. +def hetero_dev_mset(): + client = DsClient("127.0.0.1", 31501) + client.init() + + acl.init() + device_idx = 1 + acl.rt.set_device(device_idx) + + key_list = [ 'key1', 'key2', 'key3' ] + data_size = 1024 * 1024 + test_value = "value" + + in_data_blob_list = [] + for _ in key_list: + tmp_batch_list = [] + for _ in range(4): + dev_ptr, _ = acl.rt.malloc(data_size, 0) + acl.rt.memcpy(dev_ptr, data_size, acl.util.bytes_to_ptr(test_value.encode()), data_size, 1) + blob = Blob(dev_ptr, data_size) + tmp_batch_list.append(blob) + blob_list = DeviceBlobList(device_idx, tmp_batch_list) + in_data_blob_list.append(blob_list) + client.hetero().dev_mset(key_list, in_data_blob_list) + +def hetero_dev_mget(): + client = DsClient("127.0.0.1", 31501) + client.init() + + acl.init() + device_idx = 2 + acl.rt.set_device(device_idx) + + key_list = [ 'key1', 'key2', 'key3' ] + data_size = 1024 * 1024 + out_data_blob_list = [] + for _ in key_list: + tmp_batch_list = [] + for _ in range(4): + dev_ptr, _ = acl.rt.malloc(data_size, 0) + blob = Blob(dev_ptr, data_size) + tmp_batch_list.append(blob) + blob_list = DeviceBlobList(device_idx, tmp_batch_list) + out_data_blob_list.append(blob_list) + client.hetero().dev_mget(key_list, out_data_blob_list, 60000) + client.hetero().dev_delete(key_list) + +pid = os.fork() +if pid == 0: + hetero_dev_mset() + os._exit(0) +else: + hetero_dev_mget() + os.wait() +``` + +更多异构对象使用方式请参考:[异构对象开发指南](../development-guide/example/hetero.md) + +### KV + +基于共享内存实现免拷贝的 KV 数据读写,支持通过对接外部组件提供数据可靠性语义,支持数据在 DRAM / SSD / 二级缓存之间置换,实现大容量高性能缓存。 + +主要应用场景 + +- **训练场景 Checkpoint 快速保存及加载**:基于 KV 接口快速读写 Checkpoint,并支持将数据持久化到二级缓存保证数据可靠性。 + +通过 KV 接口,将任意二进制数据以键值对形式写入 DDR: + +::::{tab-set} + +:::{tab-item} Python + +```python +from datasystem.ds_client import DsClient + +client = DsClient("127.0.0.1", 31501) +client.init() + +key = "key" +expected_val = b"value" +client.kv().set(key, expected_val) + +val = client.kv().get([key]) +assert val[0] == expected_val + +client.kv().delete([key]) +``` + +::: + +:::{tab-item} C++ + +```cpp +#include "datasystem/datasystem.h" + +using namespace datasystem; + +#define ASSERT_TRUE(condition) \ + do { \ + if (!(condition)) { \ + fprintf(stderr, "Assertion failed: %s, file %s, line %d\n", \ + #condition, __FILE__, __LINE__); \ + exit(1); \ + } \ + } while(0) + +int main() +{ + ConnectOptions connectOptions = { .host = "127.0.0.1", .port = 31501 }; + auto client = std::make_shared(connectOptions); + ASSERT_TRUE(client->Init().IsOk()); + + std::string key = "testKey"; + std::string value = "Hello kv client"; + std::string value2 = "Hello modify"; + Status status = client->KV()->Set(key, value); + ASSERT_TRUE(status.IsOk()); + + std::string getValue; + status = client->KV()->Get(key, getValue); + ASSERT_TRUE(status.IsOk()); + ASSERT_TRUE(getValue == value); + + status = client->KV()->Set(key, value2); + ASSERT_TRUE(status.IsOk()); + + status = client->KV()->Get(key, getValue); + ASSERT_TRUE(status.IsOk()); + ASSERT_TRUE(getValue == value2); + + status = client->KV()->Del(key); + ASSERT_TRUE(status.IsOk()); + + status = client->KV()->Get(key, getValue); + ASSERT_TRUE(status.IsError()); + return 0; +} +``` + +::: + +:::: + +更多KV使用方式请参考:[KV开发指南](../development-guide/example/kv.md) + +### Object + +基于共享内存实现 Object 语义读写,提供基于引用计数管理生命周期,将共享内存抽象为 buffer,直接映射共享内存指针,提供更底层灵活的编程接口。 + +::::{tab-set} + +:::{tab-item} Python + +```python +import random +from datasystem.ds_client import DsClient + +def random_str(slen=10): + seed = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@#%^*()_+=-" + sa = [] + for _ in range(slen): + sa.append(random.choice(seed)) + return ''.join(sa) + +def object_test(): + client = DsClient("127.0.0.1", 31501) + client.init() + object_key = "test_key" + value = bytes(random_str(100), encoding='utf8') + buffer = client.object().create(object_key, len(value)) + client.object().g_increase_ref([object_key]) + assert client.object().query_global_ref_num(object_key) == 1 + buffer.wlatch() + buffer.memory_copy(value) + buffer.seal() + buffer.unwlatch() + buffer_list = client.object().get([object_key], 0) + assert buffer_list[0].immutable_data().tobytes() == value + #self.assertEqual(buffer_list[0].immutable_data().tobytes(), value) + client.object().g_decrease_ref([object_key]) + assert client.object().query_global_ref_num(object_key) == 0 + #self.assertEqual(client.object().query_global_ref_num(object_key), 0) + try: + client.object().get([object_key], 0) + except RuntimeError as e: + print("get error:", e) + +object_test() +``` + +::: + +:::{tab-item} C++ + +```cpp +#include "datasystem/datasystem.h" + +using namespace datasystem; + +#define ASSERT_TRUE(condition) \ + do { \ + if (!(condition)) { \ + fprintf(stderr, "Assertion failed: %s, file %s, line %d\n", \ + #condition, __FILE__, __LINE__); \ + exit(1); \ + } \ + } while(0) + +int main() +{ + ConnectOptions connectOptions = { .host = "127.0.0.1", .port = 31501 }; + auto client = std::make_shared(connectOptions); + ASSERT_TRUE(client->Init().IsOk()); + + std::string objectKey = "testKey"; + std::string data = "Hello object client"; + int size = data.size(); + std::shared_ptr buffer; + Status status = client->Object()->Create(objectKey, size, CreateParam{}, buffer); + ASSERT_TRUE(status.IsOk()); + ASSERT_EQ(size, buffer->GetSize()); + std::vector failedobjectKeys; + ASSERT_TRUE(client->Object()->GIncreaseRef({ objectKey }, failedobjectKeys).IsOk()); + buffer->WLatch(); + buffer->MemoryCopy((void *)data.data(), size); + buffer->Seal(); + buffer->UnWLatch(); + + std::vector> buffers; + ASSERT_TRUE(client->Object()->Get({ objectKey }, 0, buffers).IsOk()); + ASSERT_EQ(buffers[0]->GetSize(), size); + buffers[0]->RLatch(); + ASSERT_EQ(memcmp(data.data(), buffers[0]->MutableData(), size), 0); + buffers[0]->UnRLatch(); + ASSERT_TRUE(client->Object()->GDecreaseRef({ objectKey }, failedobjectKeys).IsOk()); + ASSERT_TRUE(client->Object()->Get({ objectKey }, 0, buffers).IsError()); + return 0; +} +``` + +::: + +:::: + +更多对象缓存使用方式请参考:[Object开发指南](../development-guide/example/object.md) \ No newline at end of file diff --git a/docs/source_zh_cn/getting-started/overview.md b/docs/source_zh_cn/getting-started/overview.md deleted file mode 100644 index d836e19..0000000 --- a/docs/source_zh_cn/getting-started/overview.md +++ /dev/null @@ -1,78 +0,0 @@ -# 入门 - -openYuanrong datasystem 是一个分布式缓存系统,利用计算集群的 HBM/DRAM/SSD 资源构建近计算多级缓存,提升模型训练及推理、大数据、微服务等场景数据访问性能。 - -## 整体架构 - -![](./image/logical_architecture.png) - -openYuanrong datasystem 由三个部分组成: - -- **多语言SDK**:提供 Python/C++ 语言接口,封装 heterogeneous object/KV/object 多种语义,支撑业务实现数据快速读写。 - - **heterogeneous object**:基于 NPU 卡的 HBM 内存抽象异构对象接口,实现昇腾 NPU 卡间数据高速直通传输。同时提供 H2D/D2H 高速迁移接口,实现数据快速在 DRAM/HBM 之间传输。 - - **KV**:基于共享内存实现免拷贝的 KV 数据读写,实现高性能数据缓存,支持通过对接外部组件提供数据可靠性语义。 - - **object**:基于 host 侧共享内存抽象数据对象,实现基于引用计数的生命周期管理,将共享内存封装为 buffer,提供指针直接读写。 - -- **worker**:openYuanron datasystem 的核心组件,用于管理 DRAM/SSD 及元数据,提供近计算缓存能力。 - - **共享内存免拷贝**:基于共享内存对外提供数据读写接口,避免 SDK 与 worker 间数据拷贝,提升性能。 - - **NPU 间并发通信**:通过异构对象抽象实现卡间直通通信,自动协调 NPU 间 HCCL 收发顺序,实现 NPU 卡间数据并发传输。 - - **P2P 数据分发**:大规模数据复制时,实现 P2P 传输负载均衡策略,允许由新的数据接收者提供数据,充分利用卡间链路带宽。 - - **元数据管理**:分布式元数据管理,实现系统水平线性扩展,避免单点瓶颈。支持元数据可靠性,提升集群可用性。提供数据订阅发布能力。 - - **多级缓存淘汰置换**:支持基于 LRU 的多级缓存置换淘汰,热数据在内存,暖数据在磁盘,冷数据在二级缓存。 - - **数据管理**:提供引用计数,TTL 等多种数据生命周期管理能力。控制数据读写一致性,提供多种一致性能力。管理各节点数据副本,热点数据多副本缓存,提升读取效率。 - -- **集群管理**:依赖 ETCD,实现节点发现/健康检测,支持故障恢复及在线扩缩容。 - -![](./image/deployment.png) - -openYuanrong datasystem 的部署视图(如上图所示): - -- 需部署 ETCD 用于集群管理。 -- 每个节点需部署 worker 进程并注册到 ETCD。 -- SDK 集成到用户进程中并与同节点的 worker 通信。 - -数据传输协议: - -- SDK 与 worker 之间通过共享内存读写数据。 -- worker 和 worker 之间通过 TCP/RDMA 传输数据(当前版本仅支持 TCP,后续版本支持 RDMA)。 -- 异构对象 HBM 之间通过 HCCS/RoCE 卡间直通传输数据。 - -# 快速开始 - -## 安装部署 - -openYuanrong datasystem 提供了 pip install 及源码安装两种方式,详细请参考[安装指南](install.md)。 -openYuanrong datasystem 提供了两种部署方式: - -- [快速进程部署](deploy.md#openyuanrong-datasystem进程部署) -- [在 Kubernetes上部署](deploy.md#openyuanrong-datasystem-kubernetes部署) - -## 开发指南 - -### 异构对象 - -异构对象实现对 HBM 内存的抽象管理,能够高效实现 D2D/H2D/D2H 的数据传输,加速 AI 训推场景数据读写。 - -主要应用场景 - -- **LLM 长序列推理 KVCache**:基于异构对象提供分布式多级缓存 (HBM/DRAM/SSD) 和高吞吐 D2D/H2D/D2H 访问能力,构建分布式 KV Cache,实现 Prefill 阶段的 KVCache 缓存以及 Prefill/Decode 实例间 KV Cache 快速传递,提升推理吞吐。 -- **模型推理实例 M->N 快速弹性**:利用异构对象的卡间直通及 P2P 数据分发能力实现模型参数快速复制。 -- **训练场景 CheckPoint 快速加载到 HBM**:各节点将待恢复的 Checkpoint 分片加载到异构对象中,利用异构对象的卡间直通传输及 P2P 数据分发能力,快速将 Checkpoint 传递到各节点 HBM。 - -[异构对象开发指南](../development-guide/hetero.md) - -### KV - -基于共享内存实现免拷贝的 KV 数据读写,支持通过对接外部组件提供数据可靠性语义,支持数据在 DRAM / SSD / 二级缓存之间置换,实现大容量高性能缓存。 - -主要应用场景 - -- **训练场景 Checkpoint 快速保存及加载**:基于 KV 接口快速读写 Checkpoint,并支持将数据持久化到二级缓存保证数据可靠性。 - -[KV开发指南](../development-guide/kv.md) - -### Object - -基于共享内存实现 Object 语义读写,提供基于引用计数管理生命周期,将共享内存抽象为 buffer,直接映射共享内存指针,提供更底层灵活的编程接口。 - -[Object开发指南](../development-guide/object.md) \ No newline at end of file diff --git a/docs/source_zh_cn/getting-started/image/deployment.png b/docs/source_zh_cn/images/deployment.png similarity index 100% rename from docs/source_zh_cn/getting-started/image/deployment.png rename to docs/source_zh_cn/images/deployment.png diff --git a/docs/source_zh_cn/getting-started/image/introduction.png b/docs/source_zh_cn/images/introduction.png similarity index 100% rename from docs/source_zh_cn/getting-started/image/introduction.png rename to docs/source_zh_cn/images/introduction.png diff --git a/docs/source_zh_cn/getting-started/image/logical_architecture.png b/docs/source_zh_cn/images/logical_architecture.png similarity index 100% rename from docs/source_zh_cn/getting-started/image/logical_architecture.png rename to docs/source_zh_cn/images/logical_architecture.png diff --git a/docs/source_zh_cn/getting-started/image/logo-large.png b/docs/source_zh_cn/images/logo-large.png similarity index 100% rename from docs/source_zh_cn/getting-started/image/logo-large.png rename to docs/source_zh_cn/images/logo-large.png diff --git a/docs/source_zh_cn/images/logo-small.png b/docs/source_zh_cn/images/logo-small.png new file mode 100644 index 0000000000000000000000000000000000000000..19eefed75012879be364ee3d4385da3de1a3e102 GIT binary patch literal 4466 zcmV-&5smJNP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vG?BLDy{BLR4&KXw2B5eG>`K~#8N?VWpk z71g!Jf9D(^FXR=JhbABalv1j_5N%5kG;nRN7bDsW28)Iw$Qulga%;spMD1jRz2`C8Rp@f`}9L`?-gEeuTeddHYNe~kE_w!kw&&;f~ z=ecLi+H3Ebap5vRL~V&E7Qh*|+kI5m8w>`6BZ#QA-|t_Xlau4i$jG4O%a>p9`FzWW zsI6{mFc=IWA)+QqsY#nQZ93hvXHNo9B|Uodpp6?hexsC{rj&}%%Nh&@LjX#ty9){m z_75IBIKY|cl9G}_`T6+;N~wGGbr=i=1C&x-&zw24Vdl)46crT}Dx3I3=I^eRG;GF;$3NJY2@*E zf}BHb#O-!dYHBJK78dR!qW=1}4F*F4f{5b1Uhk@`tSr^MdGjELP?r)D6GNGqnN(U@ zx?U-jpzn*pV5r9sQM6KO)|xeIya@>jK@Q+@N_>1g<>cg?C!#q-)KuR$gTYXbAR@O? zYG__w-jU?w_%#4F zYTOGiyx?!yvSrYX?NnODwx;XXfbr}!(*fM08wgqp zyL13Cx`Dw^E5dOqU2eLW)~??`N-5n>R55$bJZjXaaaEZXEnCrl&CR0o6&1QERa8{a zv(GJ{7A;!^ZOcx=bSf?YLjWAn*Tb?4KtoFY&J)LW7QnT-0kc5Zy`?J+hFasqskFZK zI$HSrVydhROu~j@@4io{-<^S7(w!2z(CRg7Z5`5yl9M!M-1q=zWG7)bl^+6lUoS7B z6-)d8UIx$*zW34!z-e)8^K}CN3E~|10Cdw042D|c#Hmc5K3(?%L`1Y@>pv)c$~5}o z%cHuAlv1>6^&0AUW2b<1^}92fcJA68U|VwX6g~M=2HgNNV%t^Y!A{ACr*eCny7 zZA)?BRL1CK0Bisdui4!KV7ra&ZJJ$I05`K(xrghxSv3N{PaHa;8vr~AU?PBD1L&a} z2!;SiW3ikyT~Ch82*5Xd92OovK1+1mnm*Jqx8rlt zA3zd-HoAeJu~?t{cvo{S+n1xSP_*F4D?vn>ErgV+jZ1M2fKJbZD_i86gP8`+!~ z&Np_swqO5Xr(s7dfY-&%EM*gb_Id#ThlI^Y0Kc_~tQ0S|V~DWxv14@c_N?W4*?~gm z*K`1osBQmxp=%1@B>-P)HkSM!fIhjfEkt+((^zfmmv zjb?u*fIOT1Bd#l>ZJgz`e6B6KH~@nH6lvwRhPaks2PW&0C6%1}B95d(Ikx(T6Q?ry z@y7!kL@1|XNw37)>F*zH4Jc#r;_?;bV#l&QcV~bued_d}bx8@+sr*4#hTt!vjP6v5 zg)UUWm$Yrj&dl^p=BBHWkxy7&>k(3R}*{Wz8Sv8(2~s zr{d8Zvz=7fB*n)H{Sg2k*p#y*O)CokqlEr50Iz9f?i02T+cb_ zO>;6*KD2R4ztZe0d0$JlvR`ZE>NSo+E~Q9cR4qGqY)9Wak}&weR2(WQ1^|?tJcabB z(=lfBNDvX?JKTs~J%3zP=3H4B_U+pz)>3Q2+i80$#qsV@NYtQD~H^YCq*;o zb9MmT`G(xg%WncORg?i&VvCL(1zgWffuFH60Xwll!q!6ZI<4b>kn>0uPjIwVSNH*} z1TYIgKW>V|@;N%jHcbFLrIoXe1AK0ixOsOE-;7@g{TMNAA+ODieEtsr_#JnzZ)aU+ z0Dl&BJ(rtC5!?iMTdZdsH$^NghhXs#fSnwD{G6NC7Jc}7TC@bPNI04;+zju`*K(e) zmDjxf<0MiUKVecpL~*G|MM`936b*YgjUFC8g7zOcNWBto3tH#KxBp6U@zp135q_i+ zDs0xNE>bxtbnE!3Y!R)6{Og4-lJgdpZkW)4Zy-xopy_0Y{)(mpS58a6R96bt3Oh@7 zg?NzECQ_-6k)lkQ_9TB)=qdpW5IS(v)Y6}|i2@$c_Gz7#o3U7^ZKtEAZ?0XdkA$wN z*7STys{^r`eyGqrCYD#&B+W+){RvH1ud%COK`2uCBFA*@(iOvo4Z|xhEkn1Px?*lt z769P>2Oa<}IyH1_96VuzciTAz0kyD{Cn?C>$+xl88TD$~# zyLVzh|GTP|_wAE}ag(Ovj--oC?A?1l#^GXFf9e%qh-rDl=uh$?{M5EYKG7Og<5X(H zK>%-x`R$_FRhPIq>G}nL{|E3MM-6Xtb4Jowo3D;=oZp2I4#lHICZV2Oa(N^-U7QkX zKf8$O17fZRyH@kUBGh+z z^e9M1cs1o3G3^!84IW2Mg;XeK92(Lb(b>{-RnyiLBqFR{y}D|79UwQ_4?w<6J_G#3 z{0)G8T5>I1I*4f*-pXkg&%a)clnMK*0Y~+Ma~KoE{44->1DL{1t*3?VN}#y6#iF~7 z1A!J*oUS7Wd{#_{6ZzD8995p0L@JV^qGGgb-yZn~4hAgWpMMYs@;?n&R!iOmP$=eG z16aY!t?%9@au2{9t!woo0K3KfrHBIXZ(@F{nBK4DK7PkxwT{cCEBZ?Ct;fH(PqNA2 zu4cT?b=o=171OBZ{iE*^*+bJs>uLsoPFmjOuOX7v;UX)0ADbvi4nAy9IC3g_)DbRT zZyCJjN-<&L#DH~uymxPaT^*3m@BKWfg`5L;%*dNO80%J{JI3P}`f|cB6!PY8!Y}4o z72|?7dbJY3TrvMsP7*%By%ysE{D~dl!@}-L@u|3xT-@vQ01x>42!Q@^EUVs})S7)0 z`TYNw#|gb5bXSV`yq^}Y^QhK`IGXxE^ct1gbbZ$pnf&bpU@_;~x^mr^DfA5rM@}W& zaVk1ir1Q7;-o>`uomI>4AC!X5UAiJVrYR<;Pr-zVlSHW+bDk@7HvqD270u&1tS5$# z+x|BLcvWjU*)a{k9!^9Sal+N&d3I=$dE8xX(Ute>WgfpUipN*{hp@R)oB*&!%qQ`= zdx^)!EEhV6-jQ;x+xzF-q}vjM5AzM%pPM6+_xWi$MQi(3hnthS{rMdiup?S(Gk^^oy`*x_&n9g< z=Q(m&#CZtGdjS5M#V(ZFD)c*arN%B{E5~2hF^VHO>0FDsr)sn|KudO=P6Dt@OKzS8 zkg0X6LQ7uOmaqA`?&sc>>K+Q)xyygHHZ;J`<^RYJq4%|Nwc$KFDZL9D9K~el3mXhh z2ERQrd_=(DHzJ}+pO5A*Ttuy6TL;+#pv6m;R@vt5`IrDu%6)?Z)}fP`nHjVmDGmm| zg`zRP!MhE>U|m@aa`4-G{KS4Az&jj?^wbMnf*8Ib?UIF83vB@0&Jmn^=(st`^Mu&{ z<~+vZ7HtrkRYuP#TuH2M{O=;46~&zeS)9231)skuqHcYu7pm7daVl}~9cbP9w`^Y& zEGaojqsNRRkM;$A=~Q;?+D!oHL+Z9S+Up~ThDwaevJN;sTk@JjrpzipnA-rxQD3W4UDVpe}7uFilt|=o~82g^Sb{C;P{CW8asXhHEz-* zXdO=F3jFuyRqheV<^-F;5H6j$8T1T~OmzmXhr*7X66lSUtAdib=gP_` z^N(|=S@Y&WWt@unL!rS?52>qvDfhbVcKYDMzmxC61zPsva%x|TAGMo*DH{y+93pZn zrH1C^U|SN{u6;@8UfSgTZjMK}0S5e*fGRD^^s*#l;0V z3MbO0O&iM2&i46yzJ*GuSbZA?gW-FKQmX5jGiNr;oH>)CqN0Kvhn+-3M9_>GGpMAb zi-et}Zz;#-9VgTW9AB5I37#cAD1kmWXpLe;yM*si-07*qoM6N<$ Eg6@=U{{R30 literal 0 HcmV?d00001 diff --git a/docs/source_zh_cn/index.rst b/docs/source_zh_cn/index.rst index b30d722..9cb16aa 100644 --- a/docs/source_zh_cn/index.rst +++ b/docs/source_zh_cn/index.rst @@ -1,81 +1,75 @@ -openYuanrong datasystem文档 -========================================= +概述 +===== -.. toctree:: - :glob: - :maxdepth: 1 - :caption: 概述 - :hidden: +**数据系统(openYuanrong datasystem)** 是 openYuanrong 的核心概念抽象,是一个分布式缓存系统,利用计算集群的 HBM/DRAM/SSD 资源构建近计算多级缓存,提升模型训练及推理、大数据、微服务等场景数据访问性能。 - getting-started/overview +openYuanrong datasystem 的主要特性包括: -.. toctree:: - :glob: - :maxdepth: 1 - :caption: 安装教程 - :hidden: +- **高性能分布式多级缓存**:基于 DRAM/SSD 构建分布式多级缓存,应用实例通过共享内存免拷贝读写 DRAM 数据,并提供高性能 H2D(host to device)/D2H(device to host) 接口,实现 HBM 与 DRAM 之间快速 swap。 +- **NPU 间高效数据传输**:将 NPU 的 HBM 抽象为异构对象,自动协调 NPU 间 HCCL 收发顺序,实现简单易用的卡间数据异步并发传输。并支持P2P传输负载均衡策略,充分利用卡间链路带宽。 +- **灵活的生命周期管理**:支持设置 TTL、LRU 缓存淘汰以及 delete 接口等多种生命周期管理策略,数据生命周期既可由数据系统管理,也可交由上层应用管理,提供更高的灵活性。 +- **热点数据多副本**:数据跨节点读取时自动在本地保存副本,支撑热点数据高效访问。本地副本使用 LRU 策略自动淘汰。 +- **多种数据可靠性策略**:支持 write_through、wirte_back 及 none 多种持久化策略,满足不同场景的数据可靠性需求。 +- **数据一致性**:支持 Causal 及 PRAM 两种数据一致性模型,用户可按需选择,实现性能和数据一致性的平衡。 +- **数据发布订阅**:支持数据订阅发布,解耦数据的生产者(发布者)和消费者(订阅者),实现数据的异步传输与共享。 +- **高可靠高可用**:支持分布式元数据管理,实现系统水平线性扩展。支持元数据可靠性,支持动态资源伸缩自动迁移数据,实现系统高可用。 - getting-started/install +openYuanrong datasystem 适用场景 +-------------------------------- -.. toctree:: - :glob: - :maxdepth: 1 - :caption: 部署教程 - :hidden: +- **LLM 长序列推理 KVCache**:基于异构对象提供分布式多级缓存 (HBM/DRAM/SSD) 和高吞吐 D2D/H2D/D2H 访问能力,构建分布式 KV Cache,实现 Prefill 阶段的 KVCache 缓存以及 Prefill/Decode 实例间 KV Cache 快速传递,提升推理吞吐。 +- **模型推理实例 M->N 快速弹性**:利用异构对象的卡间直通及 P2P 数据分发能力实现模型参数快速复制。 +- **强化学习模型参数重排**:利用异构对象的卡间直通传输能力,快速将模型参数从训练侧同步到推理侧。 +- **训练场景 CheckPoint 快速保存及加载**:基于 KV 接口快速写 Checkpoint,并支持将数据持久化到二级缓存保证数据可靠性。Checkpoint恢复时各节点将 Checkpoint 分片快速加载到异构对象中,利用异构对象的卡间直通传输及 P2P 数据分发能力,快速将 Checkpoint 传递到各节点 HBM。 +- **微服务状态数据快速读写**:基于 KV 接口实现内存级读写微服务状态数据,并支持将数据持久化到二级缓存保证数据可靠性。 - getting-started/deploy +openYuanrong datasystem 架构 +---------------------------- -.. toctree:: - :glob: - :maxdepth: 1 - :caption: 开发指南 - :hidden: +.. figure:: ./images/logical_architecture.png + :alt: 逻辑架构图 + :width: 60% + :align: center + + openYuanrong datasystem 逻辑架构 - development-guide/hetero - development-guide/kv - development-guide/object +openYuanrong datasystem 由三个部分组成: -.. toctree:: - :glob: - :maxdepth: 1 - :caption: Python API - :hidden: - - api/api_python/ds_client/datasystem.DsClient - api/api_python/hetero_client/datasystem.hetero_client.HeteroClient - api/api_python/hetero_client/datasystem.hetero_client.Future - api/api_python/hetero_client/datasystem.hetero_client.DeviceBlobList - api/api_python/hetero_client/datasystem.hetero_client.Blob - api/api_python/hetero_client/datasystem.hetero_client.MetaInfo - api/api_python/ds_tensor_client/datasystem.DsTensorClient - api/api_python/kv_client/datasystem.kv_client.KVClient - api/api_python/kv_client/datasystem.kv_client.ReadOnlyBuffer - api/api_python/kv_client/datasystem.kv_client.ReadParam - api/api_python/kv_client/datasystem.kv_client.SetParam - api/api_python/object_client/datasystem.object_client.Buffer - api/api_python/object_client/datasystem.object_client.ObjectClient - api/api_python/object_client/datasystem.object_client.ConsistencyType - api/api_python/object_client/datasystem.object_client.WriteMode - +- **多语言SDK**:提供 Python/C++ 语言接口,封装 heterogeneous object 及 KV 接口,支撑业务实现数据快速读写。提供两种类型接口: -.. toctree:: - :glob: - :maxdepth: 1 - :caption: C++ API - :hidden: + - **heterogeneous object**:基于 NPU 卡的 HBM 内存抽象异构对象接口,实现昇腾 NPU 卡间数据高速直通传输。同时提供 H2D/D2H 高速迁移接口,实现数据快速在 DRAM/HBM 之间传输。 + - **KV**:基于共享内存实现免拷贝的 KV 接口,实现高性能数据缓存,支持通过对接外部组件提供数据可靠性语义。 + +- **worker**:openYuanrong datasystem 的核心组件,用于分配管理 DRAM/SSD 资源以及元数据,提供分布式多级缓存能力。 - api/api_cpp/dsclient - api/api_cpp/hetero_cache - api/api_cpp/kv_cache - api/api_cpp/object_cache - api/api_cpp/common +- **集群管理**:依赖 ETCD,实现节点发现/健康检测,支持故障恢复及在线扩缩容。 + +.. figure:: ./images/deployment.png + :alt: 部署架构图 + :width: 60% + :align: center + + openYuanrong datasystem 部署架构 + +openYuanrong datasystem 的部署视图如上图所示: + +- 需部署 ETCD 用于集群管理。 +- 每个节点需部署 worker 进程并注册到 ETCD。 +- SDK 集成到用户进程中并与同节点的 worker 通信。 + +各组件间的数据传输协议如下: + +- SDK 与 worker 之间通过共享内存读写数据。 +- worker 和 worker 之间通过 TCP/RDMA 传输数据(当前版本仅支持 TCP,RDMA/UB 即将支持)。 +- 异构对象 HBM 之间通过 HCCS/RoCE 卡间直通传输数据。 .. toctree:: - :glob: - :maxdepth: 1 - :caption: 附录 - :hidden: - - appendix/dscli - appendix/k8s_configuration - appendix/log_guide \ No newline at end of file + :hidden: + + self + getting-started/getting_started + installation/installation_linux.md + deployment/index.md + development-guide/index.md + contributor_guide/contribution.md + appendix/index.md \ No newline at end of file diff --git a/docs/source_zh_cn/getting-started/install.md b/docs/source_zh_cn/installation/installation_linux.md similarity index 98% rename from docs/source_zh_cn/getting-started/install.md rename to docs/source_zh_cn/installation/installation_linux.md index 8b24c93..02878a1 100644 --- a/docs/source_zh_cn/getting-started/install.md +++ b/docs/source_zh_cn/installation/installation_linux.md @@ -1,4 +1,6 @@ -# 安装openYuanrong datasystem +(overview-installation)= + +# 安装 @@ -115,7 +117,7 @@ source ${HOME}/Ascend/ascend-toolkit/set_env.sh ### 源码编译额外依赖 -> 如无需源码编译 openYuanrong datasystem,请跳过本章节,直接查看 [一键 pip 安装](#一键-pip-安装) +> 如无需源码编译 openYuanrong datasystem,请跳过本章节。 下表列出了源码编译openYuanrong datasystem所需额外第三方依赖: diff --git a/scripts/build_thirdparty.sh b/scripts/build_thirdparty.sh index 465b747..a870233 100644 --- a/scripts/build_thirdparty.sh +++ b/scripts/build_thirdparty.sh @@ -104,11 +104,11 @@ function on_exit() { local total_duration=$((end_time - overall_start_time)) if [ $exit_status -eq 0 ]; then - echo "$total_duration" > "${TIME_DIR}/${lib_name}.time" - touch "${PROGRESS_DIR}/${lib_name}.complete" - log "$lib_name build success - ${total_duration}s" + echo "$total_duration" > "${TIME_DIR}/${lib_name}.time" + touch "${PROGRESS_DIR}/${lib_name}.complete" + log "$lib_name build success - ${total_duration}s" else - log "$lib_name build failed!!! see ${tmp_compile_dir}/log for details" + log "$lib_name build failed, see ${tmp_compile_dir}/log for details" fi } @@ -127,29 +127,29 @@ function wait_for_dependencies() { local wait_start_time=$(date +%s) for ((i=1; i<=timeout; i++)); do - all_files_created=true - missing_deps=() - for file in "${lib_dependencies[@]}"; do - if [ ! -f "${DEPENDENCY_DIR}/${file}.complete" ]; then - all_files_created=false - missing_deps+=("$file") - fi - done - - if [ "$all_files_created" = true ]; then - local current_time=$(date +%s) - local wait_duration=$((current_time - wait_start_time)) - log "$lib_name all dependencies ready (waited ${wait_duration}s)" - return 0 + all_files_created=true + missing_deps=() + for file in "${lib_dependencies[@]}"; do + if [ ! -f "${DEPENDENCY_DIR}/${file}.complete" ]; then + all_files_created=false + missing_deps+=("$file") fi + done - # 每30秒显示一次等待状态 - if [ $((i % 30)) -eq 0 ]; then - local current_time=$(date +%s) - local elapsed=$((current_time - wait_start_time)) - log "$lib_name still waiting for: ${missing_deps[*]} (${elapsed}s elapsed)" - fi - sleep 1 + if [ "$all_files_created" = true ]; then + local current_time=$(date +%s) + local wait_duration=$((current_time - wait_start_time)) + log "$lib_name all dependencies ready (waited ${wait_duration}s)" + return 0 + fi + + # Display wait status every 30 seconds. + if [ $((i % 30)) -eq 0 ]; then + local current_time=$(date +%s) + local elapsed=$((current_time - wait_start_time)) + log "$lib_name still waiting for: ${missing_deps[*]} (${elapsed}s elapsed)" + fi + sleep 1 done log_failed "$lib_name timeout waiting for dependencies" @@ -170,7 +170,7 @@ function main() { log "Total libraries to compile: $TOTAL_LIBS" - # 编译 L0 级别的库(无依赖) + # Compile L0-level libraries (dependency-free) log "Building independent libraries (L0)" for cmake_file in "${CMAKE_FILES_L0[@]}"; do @@ -200,14 +200,14 @@ EOF touch ${MULTI_BUILD_PATH}/L1_ENV - # 编译 L1 级别的库(有依赖) + # Compile L1-level libraries log "Building dependent libraries (L1)" for cmake_file in "${CMAKE_FILES_L1[@]}"; do tmp_compile_dir="${MULTI_BUILD_PATH}/${cmake_file%%.*}" mkdir -p "${tmp_compile_dir}" - # 查找依赖关系 + # Find dependencies dependencies="" for dep in "${DEPENDENCIES[@]}"; do if [[ "$dep" == "${cmake_file%%.*}:"* ]]; then @@ -219,7 +219,7 @@ EOF ( lib_name="${cmake_file%%.*}" overall_start_time=$(date +%s) - # 等待依赖项完成 + # Wait for dependencies to complete if ! wait_for_dependencies "$lib_name" "$dependencies"; then exit 1 fi @@ -238,10 +238,10 @@ include(${DATASYSTEM_HOME}/cmake/options.cmake) include(${DATASYSTEM_HOME}/cmake/util.cmake) include(${DATASYSTEM_HOME}/cmake/external_libs/${cmake_file}) EOF - mkdir -p build && cd build - cmake "${CMAKE_OPTIONS[@]}" .. >> "${tmp_compile_dir}/log" 2>&1 - cmake --build . -j "${BUILD_THREAD_NUM}" >> "${tmp_compile_dir}/log" 2>&1 - touch ${DEPENDENCY_DIR}/${lib_name}.complete + mkdir -p build && cd build + cmake "${CMAKE_OPTIONS[@]}" .. >> "${tmp_compile_dir}/log" 2>&1 + cmake --build . -j "${BUILD_THREAD_NUM}" >> "${tmp_compile_dir}/log" 2>&1 + touch ${DEPENDENCY_DIR}/${lib_name}.complete ) & done wait -- Gitee