diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 0c0ae56e8163ca5d0792bc53354c85a2ff12032f..5a640122e241be0953a2794d6432e4850d001cea 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -767,3 +767,10 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte); void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, size_t size); #endif /* __ARM64_KVM_PGTABLE_H__ */ + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, + u64 size, kvm_pte_t attr_set, + kvm_pte_t attr_clr, kvm_pte_t *orig_pte, + u32 *level, enum kvm_pgtable_walk_flags flags); +#endif diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 9ed16e848ab9d8c3710c6e890395cae783f8f9e6..c2423578f1b5fb14e139ccc5b588b06f8060b0de 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -34,6 +34,8 @@ #define TMI_ERROR_TTT_DESTROY_AGAIN 14 #define TMI_ERROR_STE_CREATED 15 +#define TMI_IMPORT_INCOMPLETE 39 + #define TMI_RETURN_STATUS(ret) ((ret) & 0xFF) #define TMI_RETURN_INDEX(ret) (((ret) >> 8) & 0xFF) @@ -48,9 +50,11 @@ #define TMI_FEATURE_REGISTER_0_HASH_SHA_256 BIT(28) #define TMI_FEATURE_REGISTER_0_HASH_SHA_512 BIT(29) -#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) +#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) #define TMI_CVM_PARAM_FLAG_SVE BIT(1) #define TMI_CVM_PARAM_FLAG_PMU BIT(2) +#define TMI_CVM_PARAM_FLAG_MIG BIT(3) +#define TMI_CVM_PARAM_FLAG_MIGVM BIT(4) #define TMI_NOT_RUNNABLE 0 #define TMI_RUNNABLE 1 @@ -246,7 +250,7 @@ struct tmi_tec_run { #define TMI_FNUM_TTT_MAP_RANGE U(0x26D) #define TMI_FNUM_TTT_UNMAP_RANGE U(0x26E) #define TMI_FNUM_TTT_DESTROY U(0x26F) -#define TMI_FNUM_INF_TEST U(0x270) +#define TMI_FNUM_INF_TEST U(0x271) #define TMI_FNUM_KAE_INIT U(0x273) #define TMI_FNUM_KAE_ENABLE U(0x274) #define TMI_FNUM_INFO_SHOW U(0x275) @@ -269,6 +273,11 @@ struct tmi_tec_run { #define TMI_FNUM_DEVICE_CREATE U(0x286) #define TMI_FNUM_DEVICE_DESTROY U(0x287) +/* additional TMI call for migration */ +#define TMI_FNUM_MIG_CONTROL U(0x270) +#define TMI_FNUM_MIG_DATA U(0x272) +#define TMI_FNUM_MIG_ATTESTATION U(0x276) + /* TMI SMC64 PIDs handled by the SPMD */ #define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) #define TMI_TMM_DATA_CREATE TMI_FID(SMC_64, TMI_FNUM_DATA_CREATE) @@ -309,6 +318,41 @@ struct tmi_tec_run { #define TMI_TMM_DEV_CREATE TMI_FID(SMC_64, TMI_FNUM_DEVICE_CREATE) #define TMI_TMM_DEV_DESTROY TMI_FID(SMC_64, TMI_FNUM_DEVICE_DESTROY) +/* additional TMI call for migration */ +#define TMI_TMM_MIG_CONTROL TMI_FID(SMC_64, TMI_FNUM_MIG_CONTROL) +#define TMI_TMM_MIG_DATA TMI_FID(SMC_64, TMI_FNUM_MIG_DATA) +#define TMI_TMM_MIG_ATTESTATION TMI_FID(SMC_64, TMI_FNUM_MIG_ATTESTATION) + +enum tmi_tmm_mig_control_fid_e { + TMI_TMM_GET_MIG_CONFIG, + TMI_TMM_MIG_STREAM_CREATE, + TMI_TMM_SET_TMM_MEMSLOT, + TMI_TMM_MIG_UPDATE_CVM_INFO, + TMI_TMM_MIG_MEM_REGION_PROTECT, + TMI_TMM_MIG_IMPORT_COMMIT, + TMI_TMM_DUMP_CHECKSUM, + TMI_TMM_MIG_EXPORT_ABORT, + TMI_TMM_MIG_EXPORT_PAUSE +}; + +enum tmi_tmm_mig_data_fid_e { + TMI_TMM_MIG_EXPORT_IMMUTABLE, + TMI_TMM_MIG_IMPORT_IMMUTABLE, + TMI_TMM_MIG_EXPORT_TRACK, + TMI_TMM_MIG_IMPORT_TRACK, + TMI_TMM_MIG_EXPORT_MEM, + TMI_TMM_MIG_IMPORT_MEM, + TMI_TMM_MIG_EXPORT_TEC, + TMI_TMM_MIG_IMPORT_TEC, + TMI_TMM_MIG_IS_ZERO_PAGE, + TMI_TMM_MIG_IMPORT_ZERO_PAGE +}; + +enum tmi_tmm_mig_attestation_fid_e { + TMI_TMM_MIG_BIND_CLEAN, + TMI_TMM_MIG_BIND_PEEK +}; + #define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) @@ -333,6 +377,8 @@ struct tmi_tec_run { #define KVM_CAP_ARM_TMM_CFG_DBG 3 #define KVM_CAP_ARM_TMM_CFG_PMU 4 #define KVM_CAP_ARM_TMM_CFG_KAE 5 +#define KVM_CAP_ARM_TMM_CFG_MIG 6 +#define KVM_CAP_ARM_TMM_CFG_MIG_CVM 7 #define KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM 11 @@ -373,6 +419,18 @@ struct kvm_cap_arm_tmm_config_item { __u64 sec_addr[KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM]; __u64 hpre_addr[KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM]; }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_MIG */ + struct { + __u32 mig_enable; + __u32 mig_src; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_MIG_CVM */ + struct { + __u32 migration_migvm_cap; + }; + /* Fix the size of the union */ __u8 reserved[256]; }; @@ -454,5 +512,35 @@ int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, unsigned long map_size, enum kvm_pgtable_prot prot, int ret); void virtcca_cvm_set_secure_flag(void *vdev, void *info); bool is_virtcca_available(void); +u64 tmi_mig_stream_create(u64 rd, u64 numa_set); +u64 tmi_get_mig_config(void); +struct arm_smccc_res tmi_export_immutable(uint64_t rd, uint64_t hpa_and_size_pa, + uint64_t page_or_list, uint64_t mig_cmd); +u64 tmi_import_immutable(uint64_t rd, uint64_t hpa_and_size_pa, + uint64_t page_or_list, uint64_t mig_cmd); +struct arm_smccc_res tmi_import_mem(uint64_t rd, uint64_t mig_mem_param); +struct arm_smccc_res tmi_export_mem(uint64_t rd, uint64_t mig_mem_param); +u64 tmi_import_track(uint64_t rd, uint64_t hpa_and_size_pa, uint64_t mig_cmd); +u64 tmi_export_track(uint64_t rd, uint64_t hpa_and_size_pa, uint64_t mig_cmd); +u64 tmi_import_commit(uint64_t rd); +u64 tmi_dump_checksum(uint64_t rd, uint64_t gpa_list_addr, + uint64_t crc_result_addr, uint64_t granularity); +u64 tmi_export_abort(uint64_t rd); +struct arm_smccc_res tmi_is_zero_page(uint64_t rd, uint64_t gpa_list_info_val); +struct arm_smccc_res tmi_import_zero_page(uint64_t rd, uint64_t gpa_list_info_val); +u64 tmi_set_tmm_memslot(uint64_t rd, uint64_t mig_memslot_param); +u64 tmi_import_tec(uint64_t tec_pa, uint64_t mbmd_addr_and_size, + uint64_t page_list_pa, uint64_t stream_info_pa); +struct arm_smccc_res tmi_export_tec(uint64_t tec_pa, uint64_t mbmd_addr_and_size, + uint64_t page_list_pa, uint64_t stream_info_pa); +u64 tmi_update_cvm_info(uint64_t rd, uint64_t cvm_update_info_addr); +void virtcca_set_tmm_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot); +struct arm_smccc_res tmi_export_pause(uint64_t rd); + +/* enable the migcvm ctl */ +int kvm_migcvm_ioctl(struct kvm *kvm, unsigned long arg); +struct arm_smccc_res tmi_mem_region_protect(u64 rd, u64 start, u64 end); +u64 tmi_bind_clean(uint64_t rd); +struct arm_smccc_res tmi_bind_peek(uint64_t rd); #endif #endif diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index 484940589c7cb1bf8973d57d02b4ce972a9888e2..540c1a13810d13e25a068b2f9a886a8bd7f6d3a1 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -14,6 +14,9 @@ enum virtcca_cvm_state { CVM_STATE_DYING }; +#define VIRTCCA_MIG_DST 0 +#define VIRTCCA_MIG_SRC 1 + #define MAX_KAE_VF_NUM 11 /* @@ -38,6 +41,19 @@ struct tmi_cvm_params { u64 kae_vf_num; u64 sec_addr[MAX_KAE_VF_NUM]; u64 hpre_addr[MAX_KAE_VF_NUM]; + u32 mig_enable; /* check the base capability of CVM migration */ + u32 mig_src; /* check the CVM is source or dest*/ + u32 migration_migvm_cap; /* the type of CVM (support migration) */ +}; + +/* the guest cvm and migcvm both use this structure */ +#define KVM_CVM_MIGVM_VERSION 0 +struct mig_cvm { + /* used by guest cvm */ + uint8_t version; /* kvm version of migcvm*/ + uint64_t migvm_cid; /* vsock cid of migvm */ + uint16_t dst_port; /* port of destination cvm */ + char dst_ip[16]; /* ip of destination cvm */ }; struct cvm { @@ -76,6 +92,11 @@ struct virtcca_cvm { struct kvm_numa_info numa_info; struct tmi_cvm_params *params; bool is_mapped; /* Whether the cvm RAM memory is mapped */ + struct virtcca_mig_state *mig_state; + struct mig_cvm *mig_cvm_info; + u64 swiotlb_start; + u64 swiotlb_end; + u64 ipa_start; }; /* @@ -123,6 +144,7 @@ int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target, unsigne void kvm_cvm_unmap_destroy_range(struct kvm *kvm); int kvm_cvm_map_range(struct kvm *kvm); +int kvm_cvm_mig_map_range(struct kvm *kvm); int virtcca_cvm_arm_smmu_domain_set_kvm(void *group); int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map); @@ -157,6 +179,366 @@ static inline unsigned long cvm_ttt_level_mapsize(int level) return (1UL << CVM_TTT_LEVEL_SHIFT(level)); } + +/* virtcca MIG sub-ioctl() commands. */ +enum kvm_cvm_cmd_id { + /* virtcca MIG migcvm commands. */ + KVM_CVM_MIGCVM_SET_CID = 0, + KVM_CVM_MIGCVM_ATTEST, + KVM_CVM_MIGCVM_ATTEST_DST, + KVM_CVM_GET_BIND_STATUS, + KVM_CVM_MIG_EXPORT_ABORT, + /* virtcca MIG stream commands. */ + KVM_CVM_MIG_STREAM_START, + KVM_CVM_MIG_EXPORT_STATE_IMMUTABLE, + KVM_CVM_MIG_IMPORT_STATE_IMMUTABLE, + KVM_CVM_MIG_EXPORT_MEM, + KVM_CVM_MIG_IMPORT_MEM, + KVM_CVM_MIG_EXPORT_TRACK, + KVM_CVM_MIG_IMPORT_TRACK, + KVM_CVM_MIG_EXPORT_PAUSE, + KVM_CVM_MIG_EXPORT_STATE_TEC, + KVM_CVM_MIG_IMPORT_STATE_TEC, + KVM_CVM_MIG_IMPORT_END, + KVM_CVM_MIG_CRC, + KVM_CVM_MIG_GET_MIG_INFO, + KVM_CVM_MIG_IS_ZERO_PAGE, + KVM_CVM_MIG_IMPORT_ZERO_PAGE, + + KVM_CVM_MIG_CMD_NR_MAX, +}; + +struct kvm_virtcca_mig_cmd { + /* enum kvm_tdx_cmd_id */ + __u32 id; + /* flags for sub-commend. If sub-command doesn't use this, set zero. */ + __u32 flags; + /* + * data for each sub-command. An immediate or a pointer to the actual + * data in process virtual address. If sub-command doesn't use it, + * set zero. + */ + __u64 data; + /* + * Auxiliary error code. The sub-command may return TDX SEAMCALL + * status code in addition to -Exxx. + * Defined for consistency with struct kvm_sev_cmd. + */ + __u64 error; +}; + +/* mig virtcca head*/ +#define KVM_DEV_VIRTCCA_MIG_ATTR 0x1 + +struct kvm_dev_virtcca_mig_attr { +#define KVM_DEV_VIRTCCA_MIG_ATTR_VERSION 0 + __u32 version; +/* 4KB buffer can hold 512 entries at most */ +#define VIRTCCA_MIG_BUF_LIST_PAGES_MAX 512 + __u32 buf_list_pages; + __u32 max_migs; +}; + +#define VIRTCCA_MIG_STREAM_MBMD_MAP_OFFSET 0 +#define VIRTCCA_MIG_STREAM_GPA_LIST_MAP_OFFSET 1 +#define VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET 2 +#define VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET 4 + +struct virtcca_bind_info { + int16_t version; + bool premig_done; +}; + +struct virtcca_dst_host_info { + char dst_ip[16]; + uint16_t dst_port; + uint8_t version; +}; + +struct virtcca_mig_mbmd_data { /* both kvm and tmm can access */ + __u16 size; + __u16 mig_version; + __u16 migs_index; /* corresponding stream idx */ + __u8 mb_type; + __u8 rsvd0; /* reserve bit */ + __u32 mb_counter; + __u32 mig_epoch; + __u64 iv_counter; + __u8 type_specific_info[]; +} __packed; + +struct virtcca_mig_mbmd { + struct virtcca_mig_mbmd_data *data; + uint64_t hpa_and_size; /* Host physical address and size of the mbmd */ +}; + +#define VIRTCCA_MIG_EPOCH_START_TOKEN 0xffffffff + +/* + * The buffer list specifies a list of 4KB pages to be used by TDH_EXPORT_MEM + * and TDH_IMPORT_MEM to export and import guest memory pages. Each entry + * is 64-bit and points to a physical address of a 4KB page used as buffer. The + * list itself is a 4KB page, so it can hold up to 512 entries. + */ +union virtcca_mig_buf_list_entry { + uint64_t val; + struct { + uint64_t rsvd0 : 12; + uint64_t pfn : 40; + uint64_t rsvd1 : 11; + uint64_t invalid : 1; + }; +}; + +struct virtcca_mig_buf_list { + union virtcca_mig_buf_list_entry *entries; + // uint64_t *entries; + hpa_t hpa; +}; + +/* + * The page list specifies a list of 4KB pages to be used by the non-memory + * states export and import, i.e. TDH_EXPORT_STATE_* and TDH_IMPORT_STATE_*. + * Each entry is 64-bit and specifies the physical address of a 4KB buffer. + * The list itself is a 4KB page, so it can hold up to 512 entries. + */ +union virtcca_mig_page_list_info { + uint64_t val; + struct { + uint64_t rsvd0 : 12; + uint64_t pfn : 40; + uint64_t rsvd1 : 3; + uint64_t last_entry : 9; + }; +}; + +struct virtcca_mig_page_list { + hpa_t *entries; + union virtcca_mig_page_list_info info; +}; + +union virtcca_mig_gpa_list_entry { + uint64_t val; + struct{ + uint64_t level : 2; /* Bits 1:0: Mapping level */ + uint64_t pending : 1; /* Bit 2: Page is pending */ + uint64_t reserved_0 : 4; /* Bits 6:3 */ + uint64_t l2_map : 3; /* Bits 9:7: L2 mapping flags */ + uint64_t mig_type : 2; /* Bits 11:10: Migration type */ + uint64_t gfn : 40; /* Bits 51:12 */ +#define GPA_LIST_OP_NOP 0 +#define GPA_LIST_OP_EXPORT 1 +#define GPA_LIST_OP_CANCEL 2 + uint64_t operation : 2; /* Bits 53:52 */ + uint64_t reserved_1 : 2; /* Bits 55:54 */ +#define GPA_LIST_S_SUCCESS 0 + uint64_t status : 5; /* Bits 56:52 */ + uint64_t reserved_2 : 3; /* Bits 63:61 */ + }; +}; + +#define TMM_MAX_DIRTY_BITMAP_LEN 8 +/* + * The GPA list specifies a list of GPAs to be used by TDH_EXPORT_MEM and + * TDH_IMPORT_MEM, TDH_EXPORT_BLOCKW, and TDH_EXPORT_RESTORE. The list itself + * is 4KB, so it can hold up to 512 such 64-bit entries. + */ +union virtcca_mig_ipa_list_info { + uint64_t val; + struct { + uint64_t rsvd0 : 3; + uint64_t first_entry: 9; + uint64_t pfn : 40; + uint64_t rsvd1 : 3; + uint64_t last_entry : 9; + }; +}; + +struct virtcca_mig_gpa_list { + union virtcca_mig_gpa_list_entry *entries; + union virtcca_mig_ipa_list_info info; +}; + +/* + * A MAC list specifies a list of MACs over 4KB migrated pages and their GPA + * entries. It is used by TDH_EXPORT_MEM and TDH_IMPORT_MEM. Each entry is + * 128-bit containing a single AES-GMAC-256 of a migrated page. The list itself + * is a 4KB page, so it can hold up to 256 entries. To support the export and + * import of 512 pages, two such MAC lists are needed to be passed to the TDX + * module. + */ +struct virtcca_mig_mac_list { + void *entries; + hpa_t hpa; +}; + +union virtcca_mig_stream_info { + uint64_t val; + struct { + uint64_t index : 16; + uint64_t rsvd : 47; + uint64_t resume : 1; + }; + struct { + uint64_t rsvd1 : 63; + uint64_t in_order : 1; + }; +}; + +struct virtcca_mig_stream { + uint16_t idx; /* stream id */ + uint32_t buf_list_pages; /* ns memory page number of buf_list 5 #include #include +#include #include #include @@ -229,3 +230,250 @@ void virtcca_its_free_shared_pages(void *addr, int order) swiotlb_free(&cvm_alloc_device, (struct page *)addr, (1 << order) * PAGE_SIZE); } + +#define DEVICE_NAME "migvm_queue_mem" + +// IOCTL cmd define +#define QUEUE_IOCTL_MAGIC 'q' +#define MIGVM_CREATE_QUEUE _IOWR(QUEUE_IOCTL_MAGIC, 1, unsigned long) +#define MIGVM_DESTROY_QUEUE _IO(QUEUE_IOCTL_MAGIC, 2) + +#define QUEUE_SIZE (64 * 1024 - 8) // Queue size (64KB - 8B) +#define ALLOC_PAGE (4) + +struct driver_data { + dev_t devno; + struct cdev cdev; + struct class *cls; + struct mig_queue_device *queue_dev; +}; + +struct mig_integrity_share_queue_addr_s { + uint64_t send_buf_ipa; + uint64_t recv_buf_ipa; +}; + +struct mig_queue_device { + struct page *send_page; + struct page *recv_page; + unsigned long rd; + struct mig_integrity_share_queue_addr_s queue_addr; +}; + +static struct driver_data *driver_data_ptr; + +static struct mig_queue_device *migvm_queue_dev_create(unsigned long rd) +{ + struct mig_queue_device *dev = kzalloc(sizeof(*dev), GFP_KERNEL); + + if (!dev) + return NULL; + + /* Allocate send buffer */ + dev->send_page = alloc_pages(GFP_KERNEL, ALLOC_PAGE); + if (!dev->send_page) { + kfree(dev); + return NULL; + } + dev->queue_addr.send_buf_ipa = page_to_phys(dev->send_page); + memset(page_address(dev->send_page), 0, QUEUE_SIZE); + + /* Allocate receive buffer */ + dev->recv_page = alloc_pages(GFP_KERNEL, ALLOC_PAGE); + if (!dev->recv_page) { + __free_page(dev->send_page); + kfree(dev); + return NULL; + } + + dev->queue_addr.recv_buf_ipa = page_to_phys(dev->recv_page); + memset(page_address(dev->recv_page), 0, QUEUE_SIZE); + + if (tsi_mig_integrity_checksum_init(rd, virt_to_phys((void *)&dev->queue_addr))) { + __free_page(dev->send_page); + __free_page(dev->recv_page); + kfree(dev); + return NULL; + } + + dev->rd = rd; + return dev; +} + +static void migvm_queue_dev_destroy(struct mig_queue_device *dev) +{ + if (dev) { + if (dev->recv_page) + __free_page(dev->recv_page); + if (dev->send_page) + __free_page(dev->send_page); + kfree(dev); + } +} + +static int migvm_queue_dev_mmap(struct mig_queue_device *dev, struct vm_area_struct *vma) +{ + if (!dev || !vma) + return -EINVAL; + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long pfn; + + switch (vma->vm_pgoff) { + case 0: + pfn = dev->queue_addr.send_buf_ipa >> PAGE_SHIFT; + break; + case 1: + pfn = dev->queue_addr.recv_buf_ipa >> PAGE_SHIFT; + break; + default: + return -EINVAL; + } + + if (remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot)) + return -EAGAIN; + return 0; +} + +static int migvm_queue_open(struct inode *inode, struct file *filp) +{ + if (!inode) + return -EINVAL; + + struct driver_data *data = container_of(inode->i_cdev, struct driver_data, cdev); + + if (!data || !filp) + return -EINVAL; + data->queue_dev = NULL; + filp->private_data = data; + return 0; +} + +static int migvm_queue_release(struct inode *inode, struct file *filp) +{ + if (!filp) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + if (data->queue_dev) { + migvm_queue_dev_destroy(data->queue_dev); + data->queue_dev = NULL; + } + return 0; +} + +static long migvm_queue_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + if (!filp) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + + switch (cmd) { + case MIGVM_CREATE_QUEUE: + if (!data->queue_dev) { + unsigned long rd; + + if (copy_from_user(&rd, (void __user *)arg, sizeof(unsigned long))) + return -EFAULT; + data->queue_dev = migvm_queue_dev_create(rd); + + if (!data->queue_dev) + return -ENOMEM; + } + break; + case MIGVM_DESTROY_QUEUE: + if (data->queue_dev) { + migvm_queue_dev_destroy(data->queue_dev); + data->queue_dev = NULL; + } + break; + default: + return -ENOTTY; + } + return 0; +} + +static int migvm_queue_mmap(struct file *filp, struct vm_area_struct *vma) +{ + if (!filp || !vma) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + + return migvm_queue_dev_mmap(data->queue_dev, vma); +} + +static const struct file_operations queue_fops = { + .owner = THIS_MODULE, + .open = migvm_queue_open, + .release = migvm_queue_release, + .unlocked_ioctl = migvm_queue_ioctl, + .mmap = migvm_queue_mmap, +}; + +static int __init migvm_queue_driver_init(void) +{ + int ret; + struct driver_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + ret = alloc_chrdev_region(&data->devno, 0, 1, DEVICE_NAME); + if (ret < 0) + goto fail_alloc_dev; + + cdev_init(&data->cdev, &queue_fops); + data->cdev.owner = THIS_MODULE; + + ret = cdev_add(&data->cdev, data->devno, 1); + if (ret < 0) + goto fail_cdev_add; + + data->cls = class_create(DEVICE_NAME); + if (IS_ERR(data->cls)) { + ret = PTR_ERR(data->cls); + goto fail_class_create; + } + + device_create(data->cls, NULL, data->devno, NULL, DEVICE_NAME); + driver_data_ptr = data; + + pr_info("Migvm queue driver loaded. Major=%d\n", MAJOR(data->devno)); + return 0; + +fail_class_create: + cdev_del(&data->cdev); +fail_cdev_add: + unregister_chrdev_region(data->devno, 1); +fail_alloc_dev: + kfree(data); + return ret; +} + +static void __exit migvm_queue_driver_exit(void) +{ + if (driver_data_ptr) { + device_destroy(driver_data_ptr->cls, driver_data_ptr->devno); + class_destroy(driver_data_ptr->cls); + cdev_del(&driver_data_ptr->cdev); + unregister_chrdev_region(driver_data_ptr->devno, 1); + kfree(driver_data_ptr); + driver_data_ptr = NULL; + } + pr_info("Migvm queue driver unloaded\n"); +} + +module_init(migvm_queue_driver_init); +module_exit(migvm_queue_driver_exit); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/kernel/virtcca_cvm_tsi.c b/arch/arm64/kernel/virtcca_cvm_tsi.c index dc3238d5fff74acf3ea883925ab8d7977e86bc5e..fb62f278c0e8535e182258eef40f3d72fe5a214a 100644 --- a/arch/arm64/kernel/virtcca_cvm_tsi.c +++ b/arch/arm64/kernel/virtcca_cvm_tsi.c @@ -22,6 +22,8 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg static int tmm_get_tsi_version(struct virtcca_cvm_tsi_version __user *arg); static int tmm_get_attestation_token(struct virtcca_cvm_attestation_cmd __user *arg); static int tmm_get_device_cert(struct virtcca_device_cert __user *arg); +static int tmm_get_set_migration_info(struct virtcca_migvm_info __user *arg); +static int tmm_migvm_mem_checksum_loop(unsigned long rd); static const struct file_operations tmm_tsi_fops = { .owner = THIS_MODULE, @@ -68,6 +70,8 @@ static void __exit tmm_tsi_exit(void) static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { int ret; + unsigned long rd; + void __user *argp = (void __user *)arg; switch (cmd) { case TMM_GET_TSI_VERSION: @@ -79,6 +83,16 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg case TMM_GET_DEVICE_CERT: ret = tmm_get_device_cert((struct virtcca_device_cert *)arg); break; + case TMM_GET_MIGRATION_INFO: + ret = tmm_get_set_migration_info((struct virtcca_migvm_info *)arg); + break; + case TMM_GET_MIGVM_MEM_CHECKSUM: + if (copy_from_user(&rd, argp, sizeof(unsigned long))) { + pr_err("tmm_tsi: mem checksum copy data from user failed\n"); + return -ENOTTY; + } + ret = tmm_migvm_mem_checksum_loop(rd); + break; default: pr_err("tmm_tsi: unknown ioctl command (0x%x)!\n", cmd); return -ENOTTY; @@ -87,6 +101,15 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg return ret; } +static int tmm_migvm_mem_checksum_loop(unsigned long rd) +{ + unsigned long ret; + + ret = tsi_mig_integrity_checksum_loop(rd); + + return ret; +} + static int tmm_get_tsi_version(struct virtcca_cvm_tsi_version __user *arg) { struct virtcca_cvm_tsi_version ver_measured = {0}; @@ -204,6 +227,136 @@ static int tmm_get_device_cert(struct virtcca_device_cert __user *arg) return 0; } +static int tmm_get_set_migration_info(struct virtcca_migvm_info __user *arg) +{ + unsigned long ret = 0; + struct virtcca_migvm_info migvm_info = {0}; + struct pending_guest_rd_s *rdcontent = NULL; + + if (!access_ok(arg, sizeof(*arg))) { + pr_err("tmm_tsi: invalid user pointer\n"); + ret = -EFAULT; + goto out; + } + + ret = copy_from_user(&migvm_info, arg, sizeof(struct virtcca_migvm_info)); + if (ret) { + pr_err("tmm_tsi: copy challenge from user failed (%lu)!\n", ret); + ret = -EFAULT; + goto out; + } + + if (migvm_info.content) { + if (!access_ok(migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: invalid content address\n"); + ret = -EFAULT; + goto out; + } + } else { + pr_err("tmm_tsi: invalid content pointer\n"); + goto out; + } + + struct migration_info *kcontent = kmalloc(sizeof(struct migration_info), GFP_KERNEL); + + if (!kcontent) { + ret = -ENOMEM; + goto out; + } + if (sizeof(struct migration_info) != migvm_info.size) { + pr_err("tmm_tsi: size mismatch\n"); + ret = -EINVAL; + goto out; + } + + switch (migvm_info.ops) { + case OP_MIGRATE_GET_ATTR: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + ret = tsi_migvm_get_attr(migvm_info.guest_rd, kcontent); + if (!ret) { + if (copy_to_user(migvm_info.content, kcontent, migvm_info.size)) { + pr_err("tmm_tsi: copy to user failed\n"); + ret = -EFAULT; + } + pr_info("tmm_tsi: OP_MIGRATE_GET_ATTRT\n"); + } else { + pr_err("tmm_tsi: get attr failed, ret = 0x%lx\n", ret); + } + + break; + } + case OP_MIGRATE_SET_SLOT: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + ret = tsi_migvm_set_slot(migvm_info.guest_rd, kcontent); + if (ret) { + pr_err("tmm_tsi: set slot failed, ret = %lx\n", ret); + ret = -EINVAL; + } + break; + } + case OP_MIGRATE_PEEK_RDS: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + + if (kcontent->pending_guest_rds) { + if (!access_ok(kcontent->pending_guest_rds, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: invalid content pending guest rds address\n"); + ret = -EFAULT; + goto out; + } + } else { + pr_err("tmm_tsi: invalid content pending guest rds pointer\n"); + goto out; + } + + rdcontent = kmalloc(sizeof(struct pending_guest_rd_s), GFP_KERNEL); + if (!rdcontent) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(rdcontent, kcontent->pending_guest_rds, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + + ret = tsi_peek_binding_list(rdcontent); + if (!ret) { + if (copy_to_user(kcontent->pending_guest_rds, rdcontent, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: copy to user failed\n"); + ret = -EFAULT; + } + } else { + pr_err("tmm_tsi: peek rds failed, ret = 0x%lx\n", ret); + } + break; + } + default: + pr_err("tmm_tsi: invalid operation (%u)!\n", migvm_info.ops); + ret = -EINVAL; + } + +out: + kfree(kcontent); + kfree(rdcontent); + + return ret; +} module_init(tmm_tsi_init); module_exit(tmm_tsi_exit); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e59152ad5c4a26181ebd8a7c9b71167078240bda..dbb07060fcbfefc44d03bad37268cb092f7b4ec0 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2040,6 +2040,10 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) case KVM_LOAD_USER_DATA: { return kvm_load_user_data(kvm, arg); } +/* add the migcvm ioctl*/ + case KVM_CVM_MIG_IOCTL: { + return kvm_migcvm_ioctl(kvm, arg); + } #endif case KVM_CREATE_IRQCHIP: { int ret; diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 874244df723e1bd6dc45aceb04ffc453552d590c..82c09d1f045eae19412874702fa5fe1f2a94daec 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1617,3 +1617,14 @@ void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *p WARN_ON(mm_ops->page_count(pgtable) != 1); mm_ops->put_page(pgtable); } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, + u64 size, kvm_pte_t attr_set, + kvm_pte_t attr_clr, kvm_pte_t *orig_pte, + u32 *level, enum kvm_pgtable_walk_flags flags) +{ + return stage2_update_leaf_attrs(pgt, addr, size, attr_set, + attr_clr, orig_pte, level, flags); +} +#endif diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index cd7589d0506448d8b15d9a7cd6dbf5ce2f5ff361..9db4aaa1083c5668ea5c0773cf133465df8daf9a 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1239,6 +1239,22 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_is_realm(kvm)) { + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + + if (end <= cvm->ipa_start || start >= cvm->ipa_start + cvm->ram_size) + goto handle_ns_mem; + + if (start >= cvm->swiotlb_end || end <= cvm->swiotlb_start) + return; + start = (start < cvm->swiotlb_start) ? cvm->swiotlb_start : start; + end = (end < cvm->swiotlb_end) ? end : cvm->swiotlb_end; + } + +handle_ns_mem: +#endif + stage2_wp_range(&kvm->arch.mmu, start, end); /* diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 6ad1a99b0b9ac79d744bdaca593fc8a7c8a9e977..baf1b14cf3a4658288b0f93339ee0e0c06b0ed71 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -404,3 +404,188 @@ u64 tmi_dev_destroy(u64 dev_num, u64 clean) arm_smccc_1_1_smc(TMI_TMM_DEV_DESTROY, dev_num, clean, &res); return res.a1; } + +/* additional TMI call for migration */ +u64 tmi_get_mig_config(void) +{ + struct arm_smccc_res res; + + /* calculate the max number of these pages(rd,vcpu) */ + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_GET_MIG_CONFIG, &res); + return res.a1; +} + +u64 tmi_mig_stream_create(u64 rd, u64 numa_set) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_STREAM_CREATE, rd, numa_set, &res); + return res.a1; +} + +u64 tmi_set_tmm_memslot(u64 rd, u64 mig_memslot_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_SET_TMM_MEMSLOT, + rd, mig_memslot_param, &res); + return res.a1; +} + +u64 tmi_update_cvm_info(u64 rd, u64 cvm_update_info_addr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_UPDATE_CVM_INFO, rd, + __pa(cvm_update_info_addr), &res); + return res.a1; +} + +struct arm_smccc_res tmi_mem_region_protect(u64 rd, u64 start, u64 end) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_MEM_REGION_PROTECT, + rd, start, end, &res); + return res; +} + +u64 tmi_import_commit(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_IMPORT_COMMIT, rd, &res); + return res.a1; +} + +u64 tmi_dump_checksum(u64 rd, u64 gpa_list_addr, u64 crc_result_addr, u64 granularity) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_DUMP_CHECKSUM, + rd, gpa_list_addr, crc_result_addr, granularity, &res); + return res.a1; +} + +u64 tmi_export_abort(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_EXPORT_ABORT, rd, &res); + return res.a1; +} + +struct arm_smccc_res tmi_export_immutable(u64 rd, u64 hpa_and_size_pa, + u64 page_or_list, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_IMMUTABLE, + rd, hpa_and_size_pa, page_or_list, mig_cmd, &res); + return res; +} + +u64 tmi_import_immutable(u64 rd, u64 hpa_and_size_pa, + u64 page_or_list, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_IMMUTABLE, + rd, hpa_and_size_pa, page_or_list, mig_cmd, &res); + return res.a1; +} + +u64 tmi_export_track(u64 rd, u64 hpa_and_size_pa, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_TRACK, + rd, hpa_and_size_pa, mig_cmd, &res); + return res.a1; +} + +u64 tmi_import_track(u64 rd, u64 hpa_and_size_pa, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_TRACK, + rd, hpa_and_size_pa, mig_cmd, &res); + return res.a1; +} + +struct arm_smccc_res tmi_import_mem(u64 rd, u64 mig_mem_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_MEM, rd, mig_mem_param, &res); + return res; +} + +struct arm_smccc_res tmi_export_mem(u64 rd, u64 mig_mem_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_MEM, + rd, mig_mem_param, &res); + return res; +} + +struct arm_smccc_res tmi_export_tec(u64 tec_pa, u64 mbmd_addr_and_size, + u64 page_list_pa, u64 stream_info_pa) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_TEC, + tec_pa, mbmd_addr_and_size, page_list_pa, stream_info_pa, &res); + return res; +} + +u64 tmi_import_tec(u64 tec_pa, u64 mbmd_addr_and_size, u64 page_list_pa, u64 stream_info_pa) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_TEC, + tec_pa, mbmd_addr_and_size, page_list_pa, stream_info_pa, &res); + return res.a1; +} + +struct arm_smccc_res tmi_is_zero_page(u64 rd, u64 gpa_list_info_val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IS_ZERO_PAGE, + rd, gpa_list_info_val, &res); + return res; +} + +struct arm_smccc_res tmi_import_zero_page(u64 rd, u64 gpa_list_info_val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_ZERO_PAGE, + rd, gpa_list_info_val, &res); + return res; +} + +struct arm_smccc_res tmi_export_pause(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_EXPORT_PAUSE, rd, &res); + return res; +} + +u64 tmi_bind_clean(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_ATTESTATION, TMI_TMM_MIG_BIND_CLEAN, rd, &res); + return res.a1; +} +struct arm_smccc_res tmi_bind_peek(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_ATTESTATION, TMI_TMM_MIG_BIND_PEEK, rd, &res); + return res; +} diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index c270f33ce9338dba4ef52d53439cd28a7b8cd348..4fffcf30b4be42304a5e218774cf136b8409334c 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -7,17 +7,22 @@ #include #include #include +#include +#include #include #include #include #include +#include +#include #include #include -#include +#include #include #include - -#include +#include +#include +#include /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); @@ -29,6 +34,32 @@ static bool virtcca_vtimer_adjust; #define UEFI_DTB_START 0x40000000 #define DTB_MAX_SIZE 0x200000 +#define SEC_CRC_PATH "/tmp/sec_memory_check" +#define NS_CRC_PATH "/tmp/ns_memory_check" +#define CRC_DUMP_CHUNK_SIZE 512 +#define FILE_NAME_LEN 256 + +#define DEFAULT_IPA_START 0x40000000 +#define CRC_POLYNOMIAL 0xEDB88320 +#define CRC_LEN 512 +#define CRC_SHIFT 8 +#define MAX_MAC_PAGES_PER_ARR 256 +#define MAX_BUF_PAGES 512 +/* migvm vsock retry times */ +#define SEND_RETRY_LIMIT 5 +#define RECV_RETRY_LIMIT 5 +#define CONNECT_RETRY_LIMIT 3 +#define TMI_IMPORT_TIMEOUT_MS 600000 +#define TMI_TRACK_TIMEOUT_MS 20 + +static struct virtcca_mig_capabilities g_virtcca_mig_caps; +static struct migcvm_agent_listen_cids g_migcvm_agent_listen_cid; + +static struct crc_config g_crc_configs[2] = { + [0] = { .is_secure = false, .enabled = false }, /* non-secure mem */ + [1] = { .is_secure = true, .enabled = false } /* secure mem */ +}; + bool is_virtcca_available(void) { return static_key_enabled(&virtcca_cvm_is_enable); @@ -175,6 +206,39 @@ int kvm_arm_create_cvm(struct kvm *kvm) goto out; } + if (cvm->params->mig_enable) { + ret = kvm_virtcca_mig_stream_ops_init(); /* init the migration main struct */ + if (ret) { + kvm_err("KVM support migstream ops init failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + + ret = virtcca_mig_capabilities_setup(cvm); + if (ret) { + kvm_err("KVM support migration cap setup failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + + /* this state might along with the protected memory */ + ret = virtcca_mig_state_create(cvm); + if (ret) { + kvm_err("KVM support mig state create failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + } else { + pr_warn("Migration Capability is not set\n"); + } + + if (cvm->params->migration_migvm_cap) { + cvm->mig_cvm_info = kzalloc(sizeof(struct mig_cvm), GFP_KERNEL_ACCOUNT); + if (!cvm->mig_cvm_info) + return -ENOMEM; + pr_info("This CVM is Mig-CVM\n"); + } + WRITE_ONCE(cvm->state, CVM_STATE_NEW); ret = 0; out: @@ -200,6 +264,18 @@ void kvm_destroy_cvm(struct kvm *kvm) if (!cvm) return; + /* disable mig config and clean binding state*/ + if (cvm->mig_state) { + virtcca_mig_state_release(cvm); + kvm_virtcca_mig_stream_ops_exit(); + ret = tmi_bind_clean(cvm->rd); + if (ret) + pr_err("KVM destroy cVM mig tmi_bind_clean failed\n"); + kfree(cvm->mig_state); + } + + kfree(cvm->mig_cvm_info); + #ifdef CONFIG_HISI_VIRTCCA_CODA /* Unmap the cvm with arm smmu domain */ kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); @@ -404,6 +480,9 @@ int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu) struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + if (tec->tec_created) + return 0; + mutex_lock(&vcpu->kvm->lock); tec->run = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!tec->run) { @@ -521,6 +600,32 @@ static int config_cvm_kae(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *c return 0; } +/* Get the qemu's transport migration config */ +static int config_cvm_migration(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + + pr_info("calling %s\n", __func__); + params = cvm->params; + params->mig_enable = cfg->mig_enable; + params->mig_src = cfg->mig_src; + params->flags |= TMI_CVM_PARAM_FLAG_MIG; + return 0; +} + +static int config_cvm_migvm(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + + params = cvm->params; + + params->migration_migvm_cap = cfg->migration_migvm_cap; + params->flags |= TMI_CVM_PARAM_FLAG_MIGVM; + return 0; +} + static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) { struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; @@ -546,6 +651,12 @@ static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) case KVM_CAP_ARM_TMM_CFG_KAE: r = config_cvm_kae(kvm, &cfg); break; + case KVM_CAP_ARM_TMM_CFG_MIG: /* enable the mig config of cvm */ + r = config_cvm_migration(kvm, &cfg); + break; + case KVM_CAP_ARM_TMM_CFG_MIG_CVM: + r = config_cvm_migvm(kvm, &cfg); + break; default: r = -EINVAL; @@ -590,6 +701,54 @@ int kvm_cvm_map_range(struct kvm *kvm) return ret; } + +int kvm_cvm_mig_map_range(struct kvm *kvm) +{ + int ret = 0; + u64 curr_numa_set; + int idx; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info = &cvm->numa_info; + struct kvm_numa_node *numa_node; + gpa_t gpa; + + curr_numa_set = kvm_get_first_binded_numa_set(kvm); + /* uefi boot */ + if (cvm->ipa_start == UEFI_LOADER_START) { + gpa = cvm->ipa_start; + numa_node = &numa_info->numa_nodes[0]; + ret = tmi_ttt_map_range(cvm->rd, gpa, + UEFI_SIZE, + curr_numa_set, numa_node->host_numa_nodes[0]); + if (ret) { + kvm_err("tmi_ttt_map_range failed: %d.\n", ret); + return ret; + } + } + + for (idx = 0; idx < numa_info->numa_cnt; idx++) { + numa_node = &numa_info->numa_nodes[idx]; + gpa = numa_node->ipa_start; + if (gpa >= numa_node->ipa_start && + gpa < numa_node->ipa_start + numa_node->ipa_size) { + ret = tmi_ttt_map_range(cvm->rd, gpa, + numa_node->ipa_size, + curr_numa_set, numa_node->host_numa_nodes[0]); + if (ret) { + kvm_err("tmi_ttt_map_range failed: %d.\n", ret); + return ret; + } + } + } + /* Vfio driver will pin memory in advance, + * if the ram already mapped, activate cvm + * does not need to map twice + */ + cvm->is_mapped = true; + return ret; +} + + static int kvm_activate_cvm(struct kvm *kvm) { #ifdef CONFIG_HISI_VIRTCCA_CODA @@ -599,6 +758,19 @@ static int kvm_activate_cvm(struct kvm *kvm) #endif struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + if (cvm->mig_state) { + kvm_info("%s: vm->mig_state->mig_src = %d", __func__, cvm->mig_state->mig_src); + if (cvm->mig_state->mig_src == VIRTCCA_MIG_DST) { + kvm_info("%s: vm->mig_state->mig_src == VIRTCCA_MIG_DST", __func__); + return 0; + } + } + + if (virtcca_cvm_state(kvm) == CVM_STATE_ACTIVE) { + kvm_info("cVM%d is already activated!\n", cvm->cvm_vmid); + return 0; + } + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; @@ -656,6 +828,18 @@ static int kvm_populate_ipa_cvm_range(struct kvm *kvm, u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); phys_addr_t ipa_base1, ipa_end2; + /* + * if cvm comes from live migraion, mem is already maped. + * Set mig_state to VIRTCCA_MIG_SRC, init live migration structs. + */ + if (cvm->mig_state) { + if (cvm->mig_state->mig_src == VIRTCCA_MIG_DST) { + cvm->mig_state->mig_src = VIRTCCA_MIG_SRC; + kvm_info("the ipa range is populated before migraion\n"); + return 0; + } + } + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; if (!IS_ALIGNED(args->populate_ipa_base1, PAGE_SIZE) || @@ -673,6 +857,8 @@ static int kvm_populate_ipa_cvm_range(struct kvm *kvm, ipa_base1 = round_down(args->populate_ipa_base1, l2_granule); ipa_end2 = round_up(args->populate_ipa_base2 + args->populate_ipa_size2, l2_granule); + cvm->ipa_start = ipa_base1; + /* uefi boot, uefi image and uefi ram from 0 to 128M */ if (ipa_base1 == UEFI_LOADER_START) { phys_addr_t ipa_base2 = round_down(args->populate_ipa_base2, l2_granule); @@ -898,6 +1084,40 @@ static inline bool is_dtb_info_has_extend_data(u64 dtb_info) return dtb_info & 0x1; } +int kvm_migcvm_ioctl(struct kvm *kvm, unsigned long arg) +{ + struct kvm_virtcca_mig_cmd cvm_cmd; + int ret = 0; + void __user *argp = (void __user *)arg; + + if (copy_from_user(&cvm_cmd, argp, sizeof(struct kvm_virtcca_mig_cmd))) + return -EINVAL; + + if (cvm_cmd.id < KVM_CVM_MIGCVM_SET_CID || cvm_cmd.id >= KVM_CVM_MIG_STREAM_START) + return -EINVAL; + + switch (cvm_cmd.id) { + case KVM_CVM_MIGCVM_SET_CID: + ret = virtcca_save_migvm_cid(kvm, &cvm_cmd); + break; + case KVM_CVM_MIGCVM_ATTEST: + ret = virtcca_migvm_agent_ratstls(kvm, &cvm_cmd); + break; + case KVM_CVM_MIGCVM_ATTEST_DST: + ret = virtcca_migvm_agent_ratstls_dst(kvm, &cvm_cmd); + break; + case KVM_CVM_GET_BIND_STATUS: + ret = virtcca_get_bind_info(kvm, &cvm_cmd); + break; + case KVM_CVM_MIG_EXPORT_ABORT: + ret = virtcca_mig_export_abort(kvm); + break; + default: + return -EINVAL; + } + return ret; +} + int kvm_load_user_data(struct kvm *kvm, unsigned long arg) { struct kvm_user_data user_data; @@ -1317,6 +1537,14 @@ int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, if (!is_virtcca_cvm_enable() || !kvm_is_realm(kvm)) return ret; + if (kvm->arch.virtcca_cvm->mig_state && + kvm->arch.virtcca_cvm->mig_state->mig_src == VIRTCCA_MIG_SRC) { + if (ipa >= kvm->arch.virtcca_cvm->swiotlb_start && + ipa < kvm->arch.virtcca_cvm->swiotlb_end) { + return ret; + } + } + struct page *dst_page = pfn_to_page(pfn); phys_addr_t dst_phys = page_to_phys(dst_page); @@ -1379,4 +1607,1959 @@ int virtcca_cvm_arm_smmu_domain_set_kvm(void *group) (void *)NULL, cvm_arm_smmu_domain_set_kvm); return ret; } + +/* now bypass the migCVM, config staightly 1 is source, 2 is dest*/ +bool virtcca_is_migration_source(struct virtcca_cvm *cvm) +{ + if (!cvm || !cvm->mig_state) { + pr_err("Error: cvm or cvm->params is NULL\n"); + return false; + } + + if (cvm->mig_state->mig_src == VIRTCCA_MIG_SRC) + return true; + + return false; +} + +/* read the max-migs , max of rd/tec pages support */ +int virtcca_mig_capabilities_setup(struct virtcca_cvm *cvm) +{ + uint64_t res; + uint16_t immutable_state_pages, rd_state_pages, tec_state_pages; + + res = tmi_get_mig_config(); + crc32_init(); + + g_virtcca_mig_caps.max_migs = (uint32_t)(res >> 48) & 0xFFFF; + + immutable_state_pages = (uint32_t)(res >> 32) & 0xFFFF; + + rd_state_pages = (uint32_t)(res >> 16) & 0xFFFF; + + tec_state_pages = (uint32_t)res & 0xFFFF; + /* + * The minimal number of pages required. It hould be large enough to + * store all the non-memory states. + */ + g_virtcca_mig_caps.nonmem_state_pages = max3(immutable_state_pages, + rd_state_pages, tec_state_pages); + + return 0; +} + +static void virtcca_mig_stream_get_virtcca_mig_attr(struct virtcca_mig_stream *stream, + struct kvm_dev_virtcca_mig_attr *attr) +{ + attr->version = KVM_DEV_VIRTCCA_MIG_ATTR_VERSION; + attr->max_migs = g_virtcca_mig_caps.max_migs; + attr->buf_list_pages = stream->buf_list_pages; +} + +static int virtcca_mig_stream_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + struct virtcca_mig_stream *stream = dev->private; + u64 __user *uaddr = (u64 __user *)(long)attr->addr; + + switch (attr->group) { + case KVM_DEV_VIRTCCA_MIG_ATTR: { + struct kvm_dev_virtcca_mig_attr virtcca_mig_attr; + + if (attr->attr != sizeof(struct kvm_dev_virtcca_mig_attr)) { + pr_err("Incompatible kvm_dev_get_tdx_mig_attr\n"); + return -EINVAL; + } + + virtcca_mig_stream_get_virtcca_mig_attr(stream, &virtcca_mig_attr); + if (copy_to_user(uaddr, &virtcca_mig_attr, sizeof(virtcca_mig_attr))) + return -EFAULT; + break; + } + default: + return -EINVAL; + } + + return 0; +} +/* this func is to check and cut the max page num of a stream */ +static int virtcca_mig_stream_set_virtcca_mig_attr(struct virtcca_mig_stream *stream, + struct kvm_dev_virtcca_mig_attr *attr) +{ + uint32_t req_pages = attr->buf_list_pages; + uint32_t min_pages = g_virtcca_mig_caps.nonmem_state_pages; + + if (req_pages > VIRTCCA_MIG_BUF_LIST_PAGES_MAX) { + stream->buf_list_pages = VIRTCCA_MIG_BUF_LIST_PAGES_MAX; + pr_warn("Cut the buf_list_npages to the max supported num\n"); + } else if (req_pages < min_pages) { + stream->buf_list_pages = min_pages; + } else { + stream->buf_list_pages = req_pages; + } + + return 0; +} + +static uint32_t crc32_table[CRC_LEN]; + +void crc32_init(void) +{ + for (uint32_t i = 0; i < CRC_LEN; i++) { + uint32_t c = i; + + for (size_t j = 0; j < CRC_SHIFT; j++) { + if (c & 1) + c = CRC_POLYNOMIAL ^ (c >> 1); + else + c >>= 1; + } + crc32_table[i] = c; + } +} + +uint32_t crc32_compute(const uint8_t *data, size_t len) +{ + uint32_t crc = 0xFFFFFFFF; + + for (size_t i = 0; i < len; i++) { + uint8_t index = (crc ^ data[i]) & 0xFF; + + crc = crc32_table[index] ^ (crc >> CRC_SHIFT); + } + return crc ^ 0xFFFFFFFF; +} + +int virtcca_config_crc(uint64_t crc_addr_start, uint64_t crc_addr_end, + uint64_t crc_granularity, bool is_secure) +{ + struct crc_config *config = &g_crc_configs[is_secure]; + const char *mem_type = is_secure ? SEC_MEM : NON_SEC_MEM; + + /* disable crc check */ + if (crc_granularity == 0) { + memset(config, 0, sizeof(*config)); + pr_info("Virtcca migration %s memory crc check disabled", mem_type); + return 0; + } + + if (crc_addr_start >= crc_addr_end || + (crc_granularity != SZ_2M && crc_granularity != SZ_4K) || + crc_addr_end - crc_addr_start < crc_granularity) { + pr_err("%s: invalid input parameters", __func__); + return -EINVAL; + } + + config->ipa_start = ALIGN(crc_addr_start, crc_granularity); + config->ipa_end = ALIGN_DOWN(crc_addr_end, crc_granularity); + config->granularity = crc_granularity; + config->enabled = true; + + pr_info("Virtcca migration %s memory crc check enabled", mem_type); + return 0; +} +EXPORT_SYMBOL_GPL(virtcca_config_crc); + +bool is_valid_crc_params_for_cvm(struct kvm *kvm, bool is_secure) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct crc_config *config = &g_crc_configs[is_secure]; + uint64_t cvm_addr_start = is_secure ? cvm->ipa_start : cvm->swiotlb_start; + uint64_t cvm_addr_end = is_secure ? (DEFAULT_IPA_START + cvm->ram_size) : cvm->swiotlb_end; + + if (config->ipa_start < cvm_addr_start || config->ipa_end > cvm_addr_end) + return false; + return true; +} + +static int virtcca_prepare_crc_file(struct virtcca_cvm *cvm, char *file_name, + size_t name_size, bool is_secure) +{ + struct file *file_p = NULL; + loff_t pos = 0; + int ret = 0; + const char *crc_file_path = is_secure ? SEC_CRC_PATH : NS_CRC_PATH; + + if (!file_name) { + pr_err("Invalid file name"); + return -EINVAL; + } + + snprintf(file_name, name_size, "%s_%u", crc_file_path, cvm->cvm_vmid); + file_p = filp_open(file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (IS_ERR(file_p)) { + ret = PTR_ERR(file_p); + pr_err("Failed to open file %s: %d\n", file_name, ret); + return -EIO; + } + + ret = kernel_write(file_p, "=== crc check start ===\n", + strlen("=== crc check start ===\n"), &pos); + if (ret < 0) + pr_err("Failed to write file header: %d\n", ret); + + filp_close(file_p, NULL); + return ret; +} + +int virtcca_dump_array_to_file(uint64_t *gpa_list, uint64_t *crc_result, + int gpa_nums, char *file_name) +{ + loff_t pos = 0; + char *buf = NULL; + int i, len; + int ret = 0; + struct file *file_p = NULL; + + if (file_name == NULL) { + pr_err("dump_array_to_file error: invalid input."); + return -EINVAL; + } + + file_p = filp_open(file_name, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (IS_ERR(file_p)) { + pr_err("dump_array_to_file: failed to open file"); + return -EIO; + } + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto cleanup; + } + + for (i = 0; i < gpa_nums; i++) { + len = snprintf(buf, PAGE_SIZE, "gpa = 0x%llx crc = 0x%llx\n", + gpa_list[i], crc_result[i]); + ret = kernel_write(file_p, buf, len, &pos); + if (ret < 0) { + pr_err("dump_array_to_file: write error at %d\n", i); + ret = -EIO; + goto cleanup; + } + } + +cleanup: + kfree(buf); + if (file_p) + filp_close(file_p, NULL); + return ret; +} + +uint32_t __execute_ns_crc_dump(struct kvm *kvm, uint64_t target_ipa, + unsigned char *crc_buf, uint64_t crc_granularity) +{ + uint64_t crc_buf_offset = 0; + int ret; + + while (crc_buf_offset < crc_granularity) { + gfn_t gfn = target_ipa >> PAGE_SHIFT; + /* The default granularity of the swiotlb range is 4K. */ + ret = kvm_read_guest_page(kvm, gfn, crc_buf + crc_buf_offset, 0, SZ_4K); + if (ret < 0) { + pr_err("read swiotlb page failed, ret = %d", ret); + return 0; + } + crc_buf_offset += SZ_4K; + } + + return crc32_compute((uint8_t *)crc_buf, crc_granularity); +} + +static int virtcca_execute_crc_dump(struct kvm *kvm, struct crc_config *config, char *file_name) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t crc_granularity = config->granularity; + uint64_t gpa_start = config->ipa_start; + uint64_t gpa_end = config->ipa_end; + uint64_t crc_addr = gpa_start; + uint64_t *gpa_list = NULL; + uint64_t *crc_result = NULL; + unsigned char *crc_buf = NULL; + uint64_t actual_chunk_size = 0; + uint64_t valid_count = 0; + int ret = 0; + + if (!file_name || !config) { + pr_err("execute_crc_dump_new: invalid input"); + return -EINVAL; + } + + gpa_list = kcalloc(CRC_DUMP_CHUNK_SIZE, sizeof(uint64_t), GFP_KERNEL); + crc_result = kcalloc(CRC_DUMP_CHUNK_SIZE, sizeof(uint64_t), GFP_KERNEL); + crc_buf = kzalloc(crc_granularity, GFP_KERNEL); + if (!crc_result || !gpa_list || !crc_buf) { + pr_err("execute_crc_dump_new: memory allocation failed"); + ret = -ENOMEM; + goto cleanup; + } + + while (crc_addr < gpa_end) { + valid_count = 0; + actual_chunk_size = min_t(uint64_t, CRC_DUMP_CHUNK_SIZE, + (gpa_end - crc_addr) / crc_granularity); + if (actual_chunk_size <= 0) + break; + if (config->is_secure) { + for (int i = 0; i < actual_chunk_size; i++) { + uint64_t addr = crc_addr + i * crc_granularity; + + if (addr >= UEFI_SIZE && addr < DEFAULT_IPA_START) + continue; /* skip the uefi reversed area */ + gpa_list[valid_count++] = addr; + } + + if (valid_count == 0) { + crc_addr += actual_chunk_size * crc_granularity; + continue; + } + ret = tmi_dump_checksum(cvm->rd, virt_to_phys(gpa_list), + virt_to_phys(crc_result), crc_granularity); + if (ret) { + pr_err("tmi_dump_checksum failed: %d", ret); + ret = -EIO; + goto cleanup; + } + } else { + for (int i = 0; i < actual_chunk_size; i++) { + gpa_list[i] = crc_addr + i * crc_granularity; + crc_result[i] = __execute_ns_crc_dump(kvm, gpa_list[i], + crc_buf, crc_granularity); + valid_count++; + } + } + + ret = virtcca_dump_array_to_file(gpa_list, crc_result, valid_count, file_name); + if (ret < 0) { + pr_err("dump crc to file failed: %d", ret); + ret = -EIO; + goto cleanup; + } + + memset(gpa_list, 0, actual_chunk_size * sizeof(uint64_t)); + memset(crc_result, 0, actual_chunk_size * sizeof(uint64_t)); + crc_addr += actual_chunk_size * crc_granularity; + touch_softlockup_watchdog(); + } + + ret = 0; +cleanup: + kfree(crc_result); + kfree(gpa_list); + kfree(crc_buf); + return ret; +} + +int virtcca_dump_crc(struct kvm *kvm, bool is_secure) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct crc_config *config = &g_crc_configs[is_secure]; + char file_name[FILE_NAME_LEN]; + const char *mem_type = is_secure ? SEC_MEM : NON_SEC_MEM; + int ret = 0; + + if (!cvm->mig_state) { + pr_err("%s: invalid mig_state", __func__); + return -EINVAL; + } + + if (!config->enabled) { + pr_err("%s disabled!", __func__); + return 0; + } + + if (!is_valid_crc_params_for_cvm(kvm, config->is_secure)) { + pr_err("%s: invalid input parameters", __func__); + return -EINVAL; + } + + ret = virtcca_prepare_crc_file(cvm, file_name, sizeof(file_name), config->is_secure); + if (ret < 0) { + pr_err("%s: create file failed", __func__); + return -EIO; + } + + ret = virtcca_execute_crc_dump(kvm, config, file_name); + if (ret) { + pr_err("%s: CRC dump execution failed", __func__); + return -EIO; + } + + pr_info("virtcca dump %s crc success", mem_type); + return 0; +} + +static int virtcca_mig_stream_mbmd_setup(struct virtcca_mig_mbmd *mbmd) +{ + struct page *page; + unsigned long mbmd_size = PAGE_SIZE; + int order = get_order(mbmd_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mbmd->data = page_address(page); + mbmd->hpa_and_size = page_to_phys(page) | (mbmd_size - 1) << 52; + + return 0; +} + +static void virtcca_mig_stream_buf_list_cleanup(struct virtcca_mig_buf_list *buf_list) +{ + int i; + kvm_pfn_t pfn; + struct page *page; + + if (!buf_list->entries) + return; + + for (i = 0; i < MAX_BUF_PAGES; i++) { + pfn = buf_list->entries[i].pfn; + if (!pfn) + break; + page = pfn_to_page(pfn); + __free_page(page); + } + free_page((unsigned long)buf_list->entries); +} + +static int virtcca_mig_stream_buf_list_alloc(struct virtcca_mig_buf_list *buf_list) +{ + struct page *page; + + /* + * Allocate the buf list page, which has 512 entries pointing to up to + * 512 pages used as buffers to export/import migration data. + */ + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + buf_list->entries = page_address(page); + buf_list->hpa = page_to_phys(page); + + return 0; +} + +static int virtcca_mig_stream_buf_list_setup(struct virtcca_mig_buf_list *buf_list, uint32_t npages) +{ + int i; + struct page *page; + + if (!npages) { + pr_err("Userspace should set_attr on the device first\n"); + return -EINVAL; + } + + if (virtcca_mig_stream_buf_list_alloc(buf_list)) + return -ENOMEM; + + for (i = 0; i < npages; i++) { + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) { + virtcca_mig_stream_buf_list_cleanup(buf_list); + return -ENOMEM; + } + buf_list->entries[i].pfn = page_to_pfn(page); + } + + /* Mark unused entries as invalid */ + for (i = npages; i < MAX_BUF_PAGES; i++) + buf_list->entries[i].invalid = true; + + return 0; +} + +static int +virtcca_mig_stream_page_list_setup(struct virtcca_mig_page_list *page_list, + struct virtcca_mig_buf_list *buf_list, uint32_t npages) +{ + struct page *page; + uint32_t i; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + page_list->entries = page_address(page); + page_list->info.pfn = page_to_pfn(page); + + /* Reuse the buffers from the buffer list for pages list */ + for (i = 0; i < npages; i++) + page_list->entries[i] = __pfn_to_phys(buf_list->entries[i].pfn); + page_list->info.last_entry = npages - 1; + + return 0; +} + +/* this function is used to setup the page list for migration */ +static int virtcca_mig_stream_gpa_list_setup(struct virtcca_mig_gpa_list *gpa_list) +{ + struct page *page; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + gpa_list->info.pfn = page_to_pfn(page); + gpa_list->entries = page_address(page); + + return 0; +} + +static int virtcca_mig_stream_mac_list_setup(struct virtcca_mig_mac_list *mac_list) +{ + struct page *page; + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0); + if (!page) + return -ENOMEM; + + mac_list->entries = page_address(page); + mac_list->hpa = page_to_phys(page); + + return 0; +} + +static int virtcca_mig_stream_setup(struct virtcca_mig_stream *stream, bool mig_src) +{ + int ret; + + ret = virtcca_mig_stream_mbmd_setup(&stream->mbmd); + if (ret) + goto err_mbmd; + + ret = virtcca_mig_stream_buf_list_setup(&stream->mem_buf_list, stream->buf_list_pages); + if (ret) + goto err_mem_buf_list; + + ret = virtcca_mig_stream_page_list_setup(&stream->page_list, + &stream->mem_buf_list, stream->buf_list_pages); + if (ret) + goto err_page_list; + + ret = virtcca_mig_stream_gpa_list_setup(&stream->gpa_list); + if (ret) + goto err_gpa_list; + + ret = virtcca_mig_stream_mac_list_setup(&stream->mac_list[0]); + if (ret) + goto err_mac_list0; + /* + * The 2nd mac list is needed only when the buf list uses more than + * 256 entries + */ + if (stream->buf_list_pages > MAX_MAC_PAGES_PER_ARR) { + ret = virtcca_mig_stream_mac_list_setup(&stream->mac_list[1]); + if (ret) + goto err_mac_list1; + } + + /* The lists used by the destination rd only */ + if (!mig_src) { + ret = virtcca_mig_stream_buf_list_alloc(&stream->dst_buf_list); + if (ret) + goto err_dst_buf_list; + ret = virtcca_mig_stream_buf_list_alloc(&stream->import_mem_buf_list); + if (ret) + goto err_import_mem_buf_list; + } + + return 0; +err_import_mem_buf_list: + free_page((unsigned long)stream->dst_buf_list.entries); +err_dst_buf_list: + if (stream->mac_list[1].entries) + free_page((unsigned long)stream->mac_list[1].entries); +err_mac_list1: + free_page((unsigned long)stream->mac_list[0].entries); +err_mac_list0: + free_page((unsigned long)stream->gpa_list.entries); +err_gpa_list: + free_page((unsigned long)stream->page_list.entries); +err_page_list: + virtcca_mig_stream_buf_list_cleanup(&stream->mem_buf_list); +err_mem_buf_list: + free_page((unsigned long)stream->mbmd.data); +err_mbmd: + pr_err("%s failed\n", __func__); + return ret; +} + +/* check the attr is enough */ +static int virtcca_mig_stream_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + struct virtcca_cvm *cvm = dev->kvm->arch.virtcca_cvm; + struct virtcca_mig_stream *stream = dev->private; + u64 __user *uaddr = (u64 __user *)(long)attr->addr; + int ret; + + switch (attr->group) { + case KVM_DEV_VIRTCCA_MIG_ATTR: { + struct kvm_dev_virtcca_mig_attr virtcca_mig_attr; + + if (copy_from_user(&virtcca_mig_attr, uaddr, sizeof(virtcca_mig_attr))) + return -EFAULT; + + if (virtcca_mig_attr.version != KVM_DEV_VIRTCCA_MIG_ATTR_VERSION) + return -EINVAL; + + ret = virtcca_mig_stream_set_virtcca_mig_attr(stream, &virtcca_mig_attr); + if (ret) + break; + + ret = virtcca_mig_stream_setup(stream, + virtcca_is_migration_source(cvm)); + break; + } + default: + return -EINVAL; + } + + return ret; +} + +static bool virtcca_mig_stream_in_mig_buf_list(uint32_t i, uint32_t max_pages) +{ + if (i >= VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET && + i < VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET + max_pages) + return true; + + return false; +} + +static vm_fault_t virtcca_mig_stream_fault(struct vm_fault *vmf) +{ + struct kvm_device *dev = vmf->vma->vm_file->private_data; + struct virtcca_mig_stream *stream = dev->private; + struct page *page; + kvm_pfn_t pfn; + uint32_t i; + + /* See linear_page_index for pgoff */ + if (vmf->pgoff == VIRTCCA_MIG_STREAM_MBMD_MAP_OFFSET) { + page = virt_to_page(stream->mbmd.data); + } else if (vmf->pgoff == VIRTCCA_MIG_STREAM_GPA_LIST_MAP_OFFSET) { + page = virt_to_page(stream->gpa_list.entries); + } else if (vmf->pgoff == VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET || + vmf->pgoff == VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET + 1) { + i = vmf->pgoff - VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET; + if (stream->mac_list[i].entries) { + page = virt_to_page(stream->mac_list[i].entries); + } else { + pr_err("%s: mac list page %d not allocated\n", + __func__, i); + return VM_FAULT_SIGBUS; + } + } else if (virtcca_mig_stream_in_mig_buf_list(vmf->pgoff, stream->buf_list_pages)) { + i = vmf->pgoff - VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET; + pfn = stream->mem_buf_list.entries[i].pfn; + page = pfn_to_page(pfn); + } else { + pr_err("%s: VM_FAULT_SIGBUS\n", __func__); + return VM_FAULT_SIGBUS; + } + + get_page(page); + vmf->page = page; + return 0; +} + +static const struct vm_operations_struct virtcca_mig_stream_ops = { + .fault = virtcca_mig_stream_fault, +}; + +static int virtcca_mig_stream_mmap(struct kvm_device *dev, struct vm_area_struct *vma) +{ + vma->vm_ops = &virtcca_mig_stream_ops; + return 0; +} + +static int virtcca_mig_export_state_immutable(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_mig_page_list *page_list = &stream->page_list; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct arm_smccc_res ret; + + ret = tmi_export_immutable(cvm->rd, stream->mbmd.hpa_and_size, + page_list->info.val, stream_info.val); + if (ret.a1 == TMI_SUCCESS) { + stream->idx = stream->mbmd.data->migs_index; + if (copy_to_user(data, &ret.a2, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; + } + return 0; +} + +static int virtcca_mig_import_state_immutable(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_mig_page_list *page_list = &stream->page_list; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct mig_cvm_update_info *update_info = NULL; + uint64_t ret, npages; + int res = 0; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + page_list->info.last_entry = npages - 1; + + ret = tmi_import_immutable(cvm->rd, stream->mbmd.hpa_and_size, + page_list->info.val, stream_info.val); + if (ret == TMI_SUCCESS) { + stream->idx = stream->mbmd.data->migs_index; + } else { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + update_info = kmalloc(sizeof(struct mig_cvm_update_info), GFP_KERNEL); + if (!update_info) + return -ENOMEM; + + ret = tmi_update_cvm_info(cvm->rd, (uint64_t)update_info); + if (ret) { + pr_err("tmi_update_cvm_info failed, err=%llx", ret); + res = -EIO; + goto out; + } + + cvm->swiotlb_start = update_info->swiotlb_start; + cvm->swiotlb_end = update_info->swiotlb_end; + cvm->ipa_start = update_info->ipa_start; + + ret = kvm_cvm_mig_map_range(kvm); + if (ret) { + pr_err("kvm_cvm_mig_map_range: failed, err=%llx\n", ret); + res = -EIO; + } + +out: + kfree(update_info); + return res; +} + +static void virtcca_mig_buf_list_set_valid(struct virtcca_mig_buf_list *mem_buf_list, + uint64_t num) +{ + int i; + + for (i = 0; i < num; i++) + mem_buf_list->entries[i].invalid = false; + + for (i = num; i < MAX_BUF_PAGES; i++) { + if (!mem_buf_list->entries[i].invalid) + mem_buf_list->entries[i].invalid = true; + else + break; + } +} + +static int virtcca_mig_mem_param_setup(struct tmi_mig_mem *mig_mem_param) +{ + struct page *page; + unsigned long mig_mem_param_size = PAGE_SIZE; + int order = get_order(mig_mem_param_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mig_mem_param->data = page_address(page); + mig_mem_param->addr_and_size = page_to_phys(page) | (mig_mem_param_size - 1) << 52; + + return 0; +} + +static int64_t virtcca_mig_stream_export_mem(struct kvm *kvm, + struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + union virtcca_mig_stream_info stream_info = {.val = 0}; + + struct tmi_mig_mem mig_mem_param = {0}; + struct tmi_mig_mem_data *mig_mem_param_data; + uint64_t npages, gpa_list_info_val; + struct arm_smccc_res tmi_res = { 0 }; + int ret; + + if (mig_state->bugged) + return -EBADF; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + if (npages > stream->buf_list_pages) + return -EINVAL; + + ret = virtcca_mig_mem_param_setup(&mig_mem_param); + if (ret) + goto out; + + mig_mem_param_data = mig_mem_param.data; + if (!mig_mem_param_data) { + ret = -ENOMEM; + goto out; + } + + gpa_list->info.first_entry = 0; + gpa_list->info.last_entry = npages - 1; + virtcca_mig_buf_list_set_valid(&stream->mem_buf_list, npages); + stream_info.index = stream->idx; + + mig_mem_param_data->gpa_list_info = gpa_list->info.val; + mig_mem_param_data->mig_buff_list_pa = stream->mem_buf_list.hpa; + mig_mem_param_data->mig_cmd = stream_info.val; + mig_mem_param_data->mbmd_hpa_and_size = stream->mbmd.hpa_and_size; + mig_mem_param_data->mac_pa0 = stream->mac_list[0].hpa; + mig_mem_param_data->mac_pa1 = stream->mac_list[1].hpa; + + tmi_res = tmi_export_mem(cvm->rd, mig_mem_param.addr_and_size); + + ret = tmi_res.a1; + gpa_list_info_val = tmi_res.a2; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &gpa_list_info_val, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: err=%d, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + +out: + if (mig_mem_param.data) + free_pages((unsigned long)mig_mem_param.data, get_order(PAGE_SIZE)); + + return ret; +} + +static int virtcca_mig_stream_import_mem(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + union virtcca_mig_stream_info stream_info = {.val = 0}; + + struct tmi_mig_mem mig_mem_param = {0}; + struct tmi_mig_mem_data *mig_mem_param_data; + + uint64_t npages = 0; + uint64_t gpa_list_info_val = 0; + uint64_t ret = 0; + struct arm_smccc_res tmi_res; + + if (mig_state->bugged) + return -EBADF; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + if (npages > stream->buf_list_pages) + return -EINVAL; + + ret = virtcca_mig_mem_param_setup(&mig_mem_param); + if (ret) + goto out; + + mig_mem_param_data = mig_mem_param.data; + if (!mig_mem_param_data) { + ret = -ENOMEM; + goto out; + } + + gpa_list->info.first_entry = 0; + gpa_list->info.last_entry = npages - 1; + virtcca_mig_buf_list_set_valid(&stream->mem_buf_list, npages); + stream_info.index = stream->idx; + + mig_mem_param_data->gpa_list_info = gpa_list->info.val; + mig_mem_param_data->mig_buff_list_pa = stream->mem_buf_list.hpa; + mig_mem_param_data->mig_cmd = stream_info.val; + mig_mem_param_data->mbmd_hpa_and_size = stream->mbmd.hpa_and_size; + mig_mem_param_data->mac_pa0 = stream->mac_list[0].hpa; + mig_mem_param_data->mac_pa1 = stream->mac_list[1].hpa; + + tmi_res = tmi_import_mem(cvm->rd, mig_mem_param.addr_and_size); + + ret = tmi_res.a1; + gpa_list_info_val = tmi_res.a2; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &gpa_list_info_val, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: err=%llx, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + +out: + if (mig_mem_param.data) + free_pages((unsigned long)mig_mem_param.data, get_order(PAGE_SIZE)); + + return ret; +} + +static int virtcca_mig_memslot_param_setup(struct tmi_mig_memslot *mig_mem_param) +{ + struct page *page; + unsigned long mig_mem_param_size = PAGE_SIZE; + int order = get_order(mig_mem_param_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mig_mem_param->data = page_address(page); + mig_mem_param->addr_and_size = page_to_phys(page) | (mig_mem_param_size - 1) << 52; + + return 0; +} + +void virtcca_set_tmm_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_mig_memslot mig_memslot_param = {0}; + struct tmi_mig_memslot_data *mig_memslot_param_data; + struct page *dirty_bitmap_page; + unsigned int dirty_bitmap_list_len; + uint64_t dirty_bitmap_addr; + int ret; + + if (memslot->base_gfn << PAGE_SHIFT < cvm->ipa_start) + return; + + ret = virtcca_mig_memslot_param_setup(&mig_memslot_param); + if (ret) + return; + + mig_memslot_param_data = mig_memslot_param.data; + if (!mig_memslot_param_data) { + ret = -ENOMEM; + return; + } + + unsigned long bitmap_size_bytes = kvm_dirty_bitmap_bytes(memslot); + + dirty_bitmap_list_len = DIV_ROUND_UP(bitmap_size_bytes, SZ_2M); + + dirty_bitmap_addr = (uint64_t)memslot->dirty_bitmap; + for (int i = 0; i < dirty_bitmap_list_len; i++) { + dirty_bitmap_page = vmalloc_to_page((uint64_t *)dirty_bitmap_addr); + mig_memslot_param_data->dirty_bitmap_list[i] = page_to_phys(dirty_bitmap_page); + dirty_bitmap_addr += SZ_2M; + } + mig_memslot_param_data->base_gfn = memslot->base_gfn; + mig_memslot_param_data->npages = memslot->npages; + mig_memslot_param_data->memslot_id = memslot->id; + + tmi_set_tmm_memslot(cvm->rd, mig_memslot_param.addr_and_size); +} + +static int virtcca_mig_export_track(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + unsigned long timeout = jiffies + msecs_to_jiffies(TMI_IMPORT_TIMEOUT_MS); + uint64_t in_order, ret; + + if (copy_from_user(&in_order, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + /* + * Set the in_order bit if userspace requests to generate a start + * token by sending a non-0 value through tdx_cmd.data. + */ + stream_info.in_order = !!in_order; + do { + ret = tmi_export_track(cvm->rd, stream->mbmd.hpa_and_size, stream_info.val); + msleep(TMI_TRACK_TIMEOUT_MS); + + if (time_after(jiffies, timeout)) { + pr_err("tmi_export_track timeout (%d ms)", TMI_IMPORT_TIMEOUT_MS); + ret = ETIMEDOUT; + break; + } + } while (ret == TMI_IMPORT_INCOMPLETE); + + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + return 0; +} + +static inline bool +virtcca_mig_epoch_is_start_token(struct virtcca_mig_mbmd_data *data) +{ + return data->mig_epoch == VIRTCCA_MIG_EPOCH_START_TOKEN; +} + +static int virtcca_mig_import_track(struct kvm *kvm, + struct virtcca_mig_stream *stream) +{ + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t ret; + + ret = tmi_import_track(cvm->rd, stream->mbmd.hpa_and_size, stream_info.val); + if (ret != TMI_SUCCESS) { + pr_err("tmi_import_track failed, err=%llx\n", ret); + return -EIO; + } + return 0; +} + +static int virtcca_mig_import_end(struct kvm *kvm) +{ + uint64_t ret; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + unsigned long timeout = jiffies + msecs_to_jiffies(TMI_IMPORT_TIMEOUT_MS); + + if (!cvm) { + pr_err("%s: cvm is not initialized\n", __func__); + return -EINVAL; + } + + do { + ret = tmi_import_commit(cvm->rd); + msleep(TMI_TRACK_TIMEOUT_MS); + + if (time_after(jiffies, timeout)) { + pr_err("tmi_import_commit timeout (%d ms)", TMI_IMPORT_TIMEOUT_MS); + ret = ETIMEDOUT; + break; + } + } while (ret == TMI_IMPORT_INCOMPLETE); + + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + virtcca_mig_state_release(cvm); + + WRITE_ONCE(cvm->state, CVM_STATE_ACTIVE); + + return 0; +} + +static int virtcca_mig_export_state_tec(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct kvm_vcpu *vcpu; + struct virtcca_cvm_tec *tec; + struct virtcca_mig_state *mig_state = kvm->arch.virtcca_cvm->mig_state; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct arm_smccc_res ret; + + if (mig_state->vcpu_export_next_idx >= atomic_read(&kvm->online_vcpus)) { + pr_err("%s: vcpu_export_next_idx %d >= online_vcpus %d\n", + __func__, mig_state->vcpu_export_next_idx, + atomic_read(&kvm->online_vcpus)); + return -EINVAL; + } + + vcpu = kvm_get_vcpu(kvm, mig_state->vcpu_export_next_idx); + tec = &vcpu->arch.tec; + + stream_info.index = stream->idx; + + + ret = tmi_export_tec(tec->tec, stream->mbmd.hpa_and_size, + stream->page_list.info.val, stream_info.val); + + if (ret.a1 == TMI_SUCCESS) { + mig_state->vcpu_export_next_idx++; + if (copy_to_user(data, &(ret.a2), sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; + } + + return 0; +} + +static uint16_t tdx_mig_mbmd_get_vcpu_idx(struct virtcca_mig_mbmd_data *data) +{ + return *(uint16_t *)data->type_specific_info; +} + +static int virtcca_mig_import_state_tec(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct kvm_vcpu *vcpu; + struct virtcca_cvm_tec *tec; + union virtcca_mig_stream_info stream_info = {.val = 0}; + uint64_t ret; + uint64_t npages; + uint16_t vcpu_idx; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + stream->page_list.info.last_entry = npages - 1; + + vcpu_idx = tdx_mig_mbmd_get_vcpu_idx(stream->mbmd.data); + vcpu = kvm_get_vcpu(kvm, vcpu_idx); + tec = &vcpu->arch.tec; + + ret = tmi_import_tec(tec->tec, stream->mbmd.hpa_and_size, + stream->page_list.info.val, stream_info.val); + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + return 0; +} + +static int virtcca_mig_get_mig_info(struct kvm *kvm, uint64_t __user *data) +{ + struct virtCCAMigInfo migInfo; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + + migInfo.swiotlb_start = cvm->swiotlb_start; + migInfo.swiotlb_end = cvm->swiotlb_end; + + if (copy_to_user(data, &(migInfo), sizeof(struct virtCCAMigInfo))) + return -EFAULT; + + return 0; +} + +static int virtcca_mig_is_zero_page(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + + int ret; + struct arm_smccc_res tmi_res = { 0 }; + bool is_zero_page = false; + + tmi_res = tmi_is_zero_page(cvm->rd, gpa_list->info.val); + + ret = tmi_res.a1; + if (tmi_res.a2) + is_zero_page = true; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &is_zero_page, sizeof(bool))) + return -EFAULT; + } else { + pr_err("%s: err=%d, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + + return ret; +} + +static int virtcca_mig_import_zero_page(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + int ret; + struct arm_smccc_res tmi_res = { 0 }; + uint64_t gpa; + + gpa = (uint64_t)data; + + tmi_res = tmi_import_zero_page(cvm->rd, gpa); + + ret = tmi_res.a1; + + if (ret) { + pr_err("%s: err=%d\n", + __func__, ret); + return -EIO; + } + + return ret; +} + +static int virtcca_mig_export_pause(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct arm_smccc_res tmi_res = { 0 }; + + tmi_res = tmi_export_pause(cvm->rd); + if (tmi_res.a1) { + pr_err("%s: err=%lu\n", + __func__, tmi_res.a1); + return -EIO; + } + + return tmi_res.a1; +} + +/* add qemu ioctl struct to fit this func */ +static long virtcca_mig_stream_ioctl(struct kvm_device *dev, unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = dev->kvm; + struct virtcca_mig_stream *stream = dev->private; + void __user *argp = (void __user *)arg; + struct kvm_virtcca_mig_cmd cvm_cmd; + int r; + + if (copy_from_user(&cvm_cmd, argp, sizeof(struct kvm_virtcca_mig_cmd))) + return -EFAULT; + + if (ioctl != KVM_CVM_MIG_IOCTL) + return -EINVAL; + + switch (cvm_cmd.id) { + case KVM_CVM_MIG_EXPORT_STATE_IMMUTABLE: + r = virtcca_mig_export_state_immutable(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_STATE_IMMUTABLE: + r = virtcca_mig_import_state_immutable(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_MEM: + r = virtcca_mig_stream_export_mem(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_MEM: + r = virtcca_mig_stream_import_mem(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_TRACK: + r = virtcca_mig_export_track(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_TRACK: + r = virtcca_mig_import_track(kvm, stream); + break; + case KVM_CVM_MIG_EXPORT_STATE_TEC: + r = virtcca_mig_export_state_tec(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_STATE_TEC: + r = virtcca_mig_import_state_tec(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_END: + r = virtcca_mig_import_end(kvm); + break; + case KVM_CVM_MIG_CRC: + r = 0; + virtcca_dump_crc(kvm, true); + virtcca_dump_crc(kvm, false); + break; + case KVM_CVM_MIG_GET_MIG_INFO: + r = virtcca_mig_get_mig_info(kvm, (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IS_ZERO_PAGE: + r = virtcca_mig_is_zero_page(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_ZERO_PAGE: + r = virtcca_mig_import_zero_page(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_PAUSE: + r = virtcca_mig_export_pause(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + default: + r = -EINVAL; + } + + return r; +} + +static int virtcca_mig_do_stream_create(struct kvm *kvm, + struct virtcca_mig_stream *stream, hpa_t *migsc_addr) +{ + u64 numa_set = kvm_get_host_numa_set_by_vcpu(0, kvm); + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + hpa_t migsc_pa = 0; + struct mig_cvm_update_info *update_info = NULL; + uint64_t ret = 0; + + /* + * This migration stream has been created, e.g. the previous migration + * session is aborted and the migration stream is retained during the + * TD guest lifecycle (required by the TDX migration architecture for + * later re-migration). No need to proceed to the creation in this + * case. + */ + if (!migsc_addr) { + pr_err("invalid migsc_addr!"); + return -1; + } + + /* now just create stream in tmm */ + migsc_pa = tmi_mig_stream_create(cvm->rd, numa_set); + if (!migsc_pa) + kvm_err("virtcca mig stream create failed!\n"); + + *migsc_addr = migsc_pa; + + update_info = kmalloc(sizeof(struct mig_cvm_update_info), GFP_KERNEL); + if (!update_info) + return -ENOMEM; + + ret = tmi_update_cvm_info(cvm->rd, (uint64_t)update_info); + if (ret) { + pr_err("tmi_update_cvm_info failed, err=%llx", ret); + kfree(update_info); + return -EIO; + } + cvm->swiotlb_start = update_info->swiotlb_start; + cvm->swiotlb_end = update_info->swiotlb_end; + + kfree(update_info); + return 0; +} + +static int virtcca_mig_session_init(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *blockw_gpa_list = &mig_state->blockw_gpa_list; + int ret = 0; + + if (virtcca_mig_do_stream_create(kvm, &mig_state->backward_stream, + &mig_state->backward_migsc_paddr)) + return -EIO; + + if (virtcca_is_migration_source(cvm)) + ret = virtcca_mig_stream_gpa_list_setup(blockw_gpa_list); + + return ret; +} + +static void virtcca_mig_session_exit(struct virtcca_mig_state *mig_state) +{ + if (mig_state->blockw_gpa_list.entries) { + free_pages((uint64_t)mig_state->blockw_gpa_list.entries, 0); + mig_state->blockw_gpa_list.entries = NULL; + mig_state->blockw_gpa_list.info.pfn = 0; + } +} + +static int virtcca_mig_stream_create(struct kvm_device *dev, u32 type) +{ + struct kvm *kvm = dev->kvm; + struct virtcca_cvm *cvm = dev->kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_stream *stream; + int ret; + + stream = kzalloc(sizeof(struct virtcca_mig_stream), GFP_KERNEL_ACCOUNT); + if (!stream) + return -ENOMEM; + + dev->private = stream; + /* set the stream idx of the cvm */ + stream->idx = atomic_inc_return(&mig_state->streams_created) - 1; + + if (!stream->idx) { + ret = virtcca_mig_session_init(kvm); /* if is the first stream, call this func */ + if (ret) + goto err_mig_session_init; + mig_state->default_stream = stream; + } + + ret = virtcca_mig_do_stream_create(kvm, stream, &mig_state->migsc_paddrs[stream->idx]); + if (ret) + goto err_stream_create; + + return 0; +err_stream_create: + virtcca_mig_session_exit(mig_state); +err_mig_session_init: + atomic_dec(&mig_state->streams_created); + kfree(stream); + return ret; +} + +void virtcca_mig_state_release(struct virtcca_cvm *cvm) +{ + struct virtcca_mig_state *mig_state = cvm->mig_state; + + if (!mig_state) + return; + + mig_state->vcpu_export_next_idx = 0; + mig_state->backward_migsc_paddr = 0; + + atomic_dec(&mig_state->streams_created); + if (!atomic_read(&mig_state->streams_created)) + virtcca_mig_session_exit(mig_state); +} + + +static void virtcca_mig_stream_release(struct kvm_device *dev) +{ + struct virtcca_mig_stream *stream = dev->private; + + free_page((unsigned long)stream->mbmd.data); + virtcca_mig_stream_buf_list_cleanup(&stream->mem_buf_list); + free_page((unsigned long)stream->page_list.entries); + free_page((unsigned long)stream->gpa_list.entries); + free_page((unsigned long)stream->mac_list[0].entries); + /* + * The 2nd mac list page is allocated conditionally when + * stream->buf_list_pages is larger than 256. + */ + if (stream->mac_list[1].entries) + free_page((unsigned long)stream->mac_list[1].entries); + if (stream->dst_buf_list.entries) + free_page((unsigned long)stream->dst_buf_list.entries); + if (stream->import_mem_buf_list.entries) + free_page((unsigned long)stream->import_mem_buf_list.entries); + /*print the elements of the stream*/ + kfree(stream); +} + +int virtcca_mig_state_create(struct virtcca_cvm *cvm) +{ + struct virtcca_mig_state *mig_state = cvm->mig_state; + hpa_t *migsc_paddrs = NULL; + + mig_state = NULL; + cvm->mig_cvm_info = NULL; + mig_state = kzalloc(sizeof(struct virtcca_mig_state), GFP_KERNEL_ACCOUNT); + if (!mig_state) + goto out; + + migsc_paddrs = kcalloc(g_virtcca_mig_caps.max_migs, sizeof(hpa_t), GFP_KERNEL_ACCOUNT); + if (!migsc_paddrs) + goto out; + + cvm->mig_cvm_info = kzalloc(sizeof(struct mig_cvm), GFP_KERNEL_ACCOUNT); + mig_state->mig_src = cvm->params->mig_src; + if (!cvm->mig_cvm_info) + goto out; + + mig_state->migsc_paddrs = migsc_paddrs; + cvm->mig_state = mig_state; + + return 0; + +out: + pr_err("%s failed", __func__); + kfree(mig_state); + kfree(migsc_paddrs); + kfree(cvm->mig_cvm_info); + return -ENOMEM; +} + +int virtcca_save_migvm_cid(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct mig_cvm *mig_cvm_usr; + struct mig_cvm *mig_cvm_info = cvm->mig_cvm_info; + + pr_info("calling %s\n", __func__); + if (mig_cvm_info == NULL) { + pr_info("guest_mig_cvm_info is NULL\n"); + return -EINVAL; + } + + mig_cvm_usr = kmalloc(sizeof(struct mig_cvm), GFP_KERNEL); + if (!mig_cvm_usr) + return -ENOMEM; + if (copy_from_user(mig_cvm_usr, (void __user *)cmd->data, + sizeof(struct mig_cvm))) { + kfree(mig_cvm_usr); + return -EFAULT; + } + + if (cmd->flags || mig_cvm_usr->version != KVM_CVM_MIGVM_VERSION) { + kfree(mig_cvm_usr); + return -EINVAL; + } + + memcpy(&mig_cvm_info->migvm_cid, &mig_cvm_usr->migvm_cid, sizeof(uint64_t)); + g_migcvm_agent_listen_cid.cid = mig_cvm_info->migvm_cid; + + kfree(mig_cvm_usr); + return 0; +} + +static int send_and_wait_ack(struct socket *sock, struct bind_msg_s *req_msg) +{ + int ret; + struct kvec vec; + struct msghdr hdr; + struct bind_msg_s ack_msg = {0}; + int retry = 0; + + memset(&hdr, 0, sizeof(hdr)); + vec.iov_base = req_msg; + vec.iov_len = sizeof(*req_msg); + /* set vsock hdr */ + hdr.msg_flags = MSG_NOSIGNAL; + iov_iter_kvec(&hdr.msg_iter, WRITE, &vec, 1, vec.iov_len); + + if (req_msg->payload_len > MAX_PAYLOAD_SIZE) { + pr_err("Payload size %u exceeds limit\n", req_msg->payload_len); + ret = -EINVAL; + goto out; + } + + /* send request to migcvm agent, expect ack from migcvm agent */ + retry = 0; + do { + ret = kernel_sendmsg(sock, &hdr, &vec, 1, vec.iov_len); + if (ret == -EINTR) { + pr_warn("sendmsg interrupted by signal, retry %d\n", retry); + retry++; + continue; + } + break; + } while (retry < SEND_RETRY_LIMIT); + + if (ret < 0) { + pr_err("Failed to send request, ret=%d\n", ret); + goto out; + } else if (ret != sizeof(*req_msg)) { + pr_err("Partial send, ret=%d\n", ret); + ret = -EIO; + goto out; + } + + /* reset ack buffer */ + vec.iov_base = &ack_msg; + vec.iov_len = sizeof(ack_msg); + + retry = 0; + do { + ret = kernel_recvmsg(sock, &hdr, &vec, 1, sizeof(ack_msg), hdr.msg_flags); + if (ret == -EINTR) { + pr_warn("recvmsg interrupted by signal, retry %d\n", retry); + retry++; + continue; + } + break; + } while (retry < RECV_RETRY_LIMIT); + + if (ret < 0) { + pr_err("Failed to recv ack, ret=%d\n", ret); + goto out; + } else if (ret != sizeof(ack_msg)) { + pr_err("Partial ack recv ret=%d\n", ret); + ret = -EIO; + goto out; + } + + /* validate ack message */ + if (ack_msg.payload_type != VSOCK_MSG_ACK || + ack_msg.session_id != req_msg->session_id || + ack_msg.success == 0) { + pr_err("ACK validation failed, the payload_type=%d, session_id=%llu, success=%d\n", + ack_msg.payload_type, ack_msg.session_id, ack_msg.success); + ret = -EPROTO; + goto out; + } + pr_info("ACK validation passed\n"); + ret = 0; + +out: + return ret; +} + +/* vsock connection*/ +/* step 1: send to mig-cvm agent: the migrated rd, the destination platform ip*/ +/* step 2: wait for mig-cvm agent's response */ +static int notify_migcvm_agent(uint64_t cid, struct virtcca_dst_host_info *dst_host_info, + uint64_t guest_rd, bool is_src) +{ + struct socket *sock = NULL; + int ret = 0; + int retry_count = 3; + int error = 0, len = sizeof(error); + int connect_retry = 0; + long old_sndtimeo = 0, old_rcvtimeo = 0; + const unsigned long timeout = 5 * HZ; + + struct sockaddr_vm sa = { + .svm_family = AF_VSOCK, + .svm_cid = cid, + .svm_port = is_src ? MIGCVM_AGENT_PORT_SRC : MIGCVM_AGENT_PORT_DST + }; + struct bind_msg_s bind_msg = {0}; + + pr_info("calling %s, cid=%llu, port=%d\n", __func__, cid, sa.svm_port); + ret = sock_create_kern(&init_net, AF_VSOCK, SOCK_STREAM, 0, &sock); + if (ret < 0) { + pr_err("Failed to create socket, error: %d\n", ret); + return ret; + } + + /* save original receive timeout, and set 5s timeout for migcvm ack */ + if (sock->sk) { + old_sndtimeo = sock->sk->sk_sndtimeo; + old_rcvtimeo = sock->sk->sk_rcvtimeo; + sock->sk->sk_sndtimeo = timeout; + sock->sk->sk_rcvtimeo = timeout; + } + + connect_retry = 0; + do { + ret = kernel_connect(sock, (struct sockaddr *)&sa, sizeof(sa), O_NONBLOCK); + if (ret == -EINTR) { + pr_warn("connect interrupted by signal, retry %d\n", connect_retry); + schedule_timeout_uninterruptible(HZ / 10); + connect_retry++; + continue; + } + break; + } while (connect_retry < CONNECT_RETRY_LIMIT); + + if (ret < 0) { + if (sock->ops && sock->ops->getsockopt) + sock->ops->getsockopt(sock, SOL_SOCKET, SO_ERROR, (char *)&error, &len); + if (error) { + pr_err("Connect failed (cid=%llu, port=%d, err=%d)\n", + cid, sa.svm_port, error); + ret = -error; + goto cleanup; + } + } + + if (strscpy(bind_msg.cmd, is_src ? "START_CLIENT" : "START_SERVER", + sizeof(bind_msg.cmd)) < 0) { + pr_err("Command string too long\n"); + ret = -EINVAL; + goto cleanup; + } + + bind_msg.session_id = get_jiffies_64(); + bind_msg.payload_type = is_src ? PAYLOAD_TYPE_ALL : PAYLOAD_TYPE_ULL; + bind_msg.payload.ull_payload = guest_rd; + bind_msg.payload_len = MAX_PAYLOAD_SIZE; + if (is_src) { + if (!dst_host_info) { + pr_err("No destination host info provided for source\n"); + ret = -EINVAL; + goto cleanup; + } + bind_msg.payload_len = strlen(dst_host_info->dst_ip) + 1; + if (strscpy(bind_msg.payload.char_payload, dst_host_info->dst_ip, + sizeof(bind_msg.payload.char_payload)) < 0) { + pr_err("Destination IP too long\n"); + ret = -EINVAL; + goto cleanup; + } + } + + do { + ret = send_and_wait_ack(sock, &bind_msg); + if (!ret) + break; + pr_warn("Send/ACK failed, retrying (%d left)\n", retry_count - 1); + retry_count--; + /* a delay time */ + schedule_timeout_uninterruptible(HZ / 10); + } while (retry_count > 0); + + if (ret) + pr_err("Failed to get bind info after retries, error=%d\n", ret); + +cleanup: + if (sock) { + if (sock->sk) { + sock->sk->sk_sndtimeo = old_sndtimeo; + sock->sk->sk_rcvtimeo = old_rcvtimeo; + } + if (ret == 0) + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + return ret; +} + +int virtcca_migvm_agent_ratstls_dst(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_dst_host_info dst_info; + struct arm_smccc_res tmi_ret; + int ret = 0; + + if (g_migcvm_agent_listen_cid.cid == 0) { + pr_err("there is no cid of migcvm, cannot migrate virtCCA cVM\n"); + return -EINVAL; + } + + if (copy_from_user(&dst_info, (void __user *)cmd->data, + sizeof(struct virtcca_dst_host_info))) { + return -EFAULT; + } + + if (cmd->flags || dst_info.version != KVM_CVM_MIGVM_VERSION) { + pr_err("invalid flags or version, flags is %x, version is %x\n", + cmd->flags, dst_info.version); + return -EINVAL; + } + pr_info("calling tmi_bind_peek"); + /* check if the slot is binded*/ + tmi_ret = tmi_bind_peek(cvm->rd); + if (tmi_ret.a1 == TMI_SUCCESS) { + if (tmi_ret.a2 <= SLOT_NOT_BINDED) { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + } else { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + + ret = notify_migcvm_agent(g_migcvm_agent_listen_cid.cid, NULL, cvm->rd, false); + if (ret != 0) { + pr_err("%s: notify_migcvm_agent failed, ret=%d\n", __func__, ret); + return -EINVAL; + } + + return ret; +} + +int virtcca_migvm_agent_ratstls(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_dst_host_info dst_info; + struct arm_smccc_res tmi_ret; + int ret = 0; + + if (g_migcvm_agent_listen_cid.cid == 0) { + pr_err("there is no cid of migcvm, cannot migrate virtCCA cVM\n"); + return -EINVAL; + } + + if (copy_from_user(&dst_info, (void __user *)cmd->data, + sizeof(struct virtcca_dst_host_info))) { + return -EFAULT; + } + + if (cmd->flags || dst_info.version != KVM_CVM_MIGVM_VERSION) { + pr_err("invalid flags or version, flags is %x, version is %x\n", + cmd->flags, dst_info.version); + return -EINVAL; + } + + /* now the dst ip is none, and dst port is 0, + * it should be add check into this (after qemu input) + */ + if (strscpy(cvm->mig_cvm_info->dst_ip, dst_info.dst_ip, + sizeof(cvm->mig_cvm_info->dst_ip)) < 0) { + pr_err("save dst_ip failed\n"); + return -EINVAL; + } + cvm->mig_cvm_info->dst_port = dst_info.dst_port; + /* check if the slot is binded*/ + tmi_ret = tmi_bind_peek(cvm->rd); + if (tmi_ret.a1 == TMI_SUCCESS) { + if (tmi_ret.a2 <= SLOT_NOT_BINDED) { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + } else { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + + ret = notify_migcvm_agent(g_migcvm_agent_listen_cid.cid, &dst_info, cvm->rd, true); + if (ret != 0) { + pr_info("%s: notify_migcvm_agent failed, ret=%d\n", __func__, ret); + return -EINVAL; + } + return ret; +} + +int virtcca_get_bind_info(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_bind_info info; + struct arm_smccc_res ret; + + if (copy_from_user(&info, (void __user *)cmd->data, + sizeof(struct virtcca_bind_info))) { + return -EFAULT; + } + + if (cmd->flags || info.version != KVM_CVM_MIGVM_VERSION) + return -EINVAL; + + ret = tmi_bind_peek(cvm->rd); + if (ret.a1 == TMI_SUCCESS) { + if (ret.a2 == SLOT_IS_READY) + info.premig_done = true; + else + info.premig_done = false; + if (copy_to_user((void __user *)cmd->data, &info, + sizeof(struct virtcca_bind_info))) { + return -EFAULT; + } + return ret.a1; + } + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; +} + +static struct kvm_device_ops kvm_virtcca_mig_stream_ops = { + .name = "kvm-virtcca-mig-stream", + .get_attr = virtcca_mig_stream_get_attr, + .set_attr = virtcca_mig_stream_set_attr, + .mmap = virtcca_mig_stream_mmap, + .ioctl = virtcca_mig_stream_ioctl, + .create = virtcca_mig_stream_create, + .destroy = virtcca_mig_stream_release, +}; + +static atomic_t g_mig_streams_used = ATOMIC_INIT(0); + +int kvm_virtcca_mig_stream_ops_init(void) +{ + int ret = 0; + + if (!atomic_read(&g_mig_streams_used)) + ret = kvm_register_device_ops(&kvm_virtcca_mig_stream_ops, + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM); + + if (!ret) + atomic_inc(&g_mig_streams_used); + + return ret; +} + +void kvm_virtcca_mig_stream_ops_exit(void) +{ + atomic_dec(&g_mig_streams_used); + if (!atomic_read(&g_mig_streams_used)) + kvm_unregister_device_ops(KVM_DEV_TYPE_VIRTCCA_MIG_STREAM); +} + +void virtcca_enable_log_dirty(struct kvm *kvm, uint64_t start, uint64_t end) +{ + struct arm_smccc_res res; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t s_start = cvm->ipa_start; + uint64_t s_end = cvm->ipa_start + cvm->ram_size; + + if (end <= s_start || start >= s_end) + return; + + res = tmi_mem_region_protect(cvm->rd, start, end); + if (res.a1 != 0) + pr_err("tmi_mem_region_protect failed!\n"); +} + +int virtcca_mig_export_abort(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + phys_addr_t target_ipa = cvm->swiotlb_start; + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + int level; + uint64_t granule; + uint64_t ret = 0; + + while (target_ipa != 0 && target_ipa < cvm->swiotlb_end) { + ret = kvm_pgtable_get_leaf(pgt, target_ipa, NULL, &level); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + granule = kvm_granule_size(level); + ret = virtcca_stage2_update_leaf_attrs(pgt, target_ipa, granule, + KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W, 0, NULL, NULL, 0); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, target_ipa, level); + target_ipa += granule; + } + + ret = tmi_export_abort(cvm->rd); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + virtcca_mig_state_release(cvm); + return ret; +} + +static int virtcca_kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) +{ + unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot); + unsigned long dirty_bitmap_size = ALIGN(dirty_bytes, SZ_2M); + int current_node; + + current_node = numa_node_id(); + if (current_node < 0) { + pr_err("Failed to get current NUMA node."); + return -EINVAL; + } + + memslot->dirty_bitmap = __vmalloc_node_range(dirty_bitmap_size * 2, SZ_2M, + VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, + VM_ALLOW_HUGE_VMAP, current_node, __builtin_return_address(0)); + if (!memslot->dirty_bitmap) + return -ENOMEM; + + return 0; +} + +int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new) +{ + int r; + struct kvm_memory_slot *memslot = (struct kvm_memory_slot *)new; + + r = virtcca_kvm_alloc_dirty_bitmap(memslot); + if (r) + return r; + + virtcca_set_tmm_memslot(kvm, memslot); + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + bitmap_set(memslot->dirty_bitmap, 0, memslot->npages); + return 0; +} + +void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush) +{ + struct kvm_memory_slot *memslot = (struct kvm_memory_slot *)new; + + if (!kvm_is_realm(kvm)) + return; + + spin_lock((spinlock_t *)&(kvm)->mmu_lock); + + phys_addr_t start = (memslot->base_gfn) << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + *flush = true; + virtcca_enable_log_dirty(kvm, start, end); + + spin_unlock((spinlock_t *)&(kvm)->mmu_lock); +} + +bool kvm_check_realm(struct kvm *kvm) +{ + return kvm_is_realm(kvm); +} #endif diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h index c5d7779b1ffcd15e4a55f4abe75629d2a8b9cc64..588b07030b8bb2a806fdc846fe127a00ad4f49cf 100644 --- a/include/linux/virtcca_cvm_domain.h +++ b/include/linux/virtcca_cvm_domain.h @@ -55,6 +55,9 @@ void virtcca_dev_destroy(u64 dev_num, u64 clean); bool is_virtcca_pci_cc_dev(struct device *dev); int virtcca_create_vdev(struct device *dev); bool is_virtcca_secure_vf(struct device *dev, struct device_driver *drv); +int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new); +void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush); +bool kvm_check_realm(struct kvm *kvm); #else static inline size_t virtcca_pci_get_rom_size(void *pdev, void __iomem *rom, @@ -89,5 +92,20 @@ static inline bool is_virtcca_secure_vf(struct device *dev, struct device_driver { return false; } + +static inline int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new) +{ + return 0; +} + +static inline void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush) +{ + ; +} + +static inline bool kvm_check_realm(struct kvm *kvm) +{ + return false; +} #endif /* CONFIG_HISI_VIRTCCA_CODA */ #endif /* __VIRTCCA_CVM_DOMAIN_H */ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 0036cfaf5d69e0d59e8fd68656bc728a99a8aeb2..609c24bef3814b9f65049e2d875204b8bd35be60 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1499,6 +1499,8 @@ enum kvm_device_type { #define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_RISCV_AIA, #define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM = 0x00C, +#define KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_LOONGARCH_IPI, #define KVM_DEV_TYPE_LOONGARCH_IPI KVM_DEV_TYPE_LOONGARCH_IPI KVM_DEV_TYPE_LOONGARCH_EIOINTC, @@ -1551,6 +1553,8 @@ struct kvm_numa_info { #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO, 0x48, __u64) #define KVM_LOAD_USER_DATA _IOW(KVMIO, 0x49, struct kvm_user_data) +/*virtcca migration*/ +#define KVM_CVM_MIG_IOCTL _IOWR(KVMIO, 0xf2, struct kvm_virtcca_mig_cmd) #define KVM_CAP_ARM_TMM 300 /* FIXME: Large number to prevent conflicts */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b44ab7919428fbe4d7c051512101f7a675d153f8..f252cc39495fe9d9236d5f86ac4bdab867c3027e 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1804,6 +1804,7 @@ void __init swiotlb_cvm_update_mem_attributes(void) bytes = PAGE_ALIGN(io_tlb_default_mem.defpool.nslabs << IO_TLB_SHIFT); set_cvm_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); memset(vaddr, 0, bytes); + swiotlb_unmap_notify(io_tlb_default_mem.defpool.start, bytes); io_tlb_default_mem.for_alloc = true; } #endif diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index bd1a496b5448f192b319221f568895b03d44c5a7..34f3243767c193a2242d03c72257db463b8071dd 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -1448,6 +1448,8 @@ enum kvm_device_type { #define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_RISCV_AIA, #define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM = 0x00C, +#define KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_LA_IOAPIC = 0x100, #define KVM_DEV_TYPE_LA_IOAPIC KVM_DEV_TYPE_LA_IOAPIC KVM_DEV_TYPE_LA_IPI, @@ -1668,6 +1670,9 @@ struct kvm_enc_region { __u64 size; }; +/*virtcca migration*/ +#define KVM_CVM_MIG_IOCTL _IOWR(KVMIO, 0xf2, struct kvm_virtcca_mig_cmd) + #define KVM_MEMORY_ENCRYPT_REG_REGION _IOR(KVMIO, 0xbb, struct kvm_enc_region) #define KVM_MEMORY_ENCRYPT_UNREG_REGION _IOR(KVMIO, 0xbc, struct kvm_enc_region) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d8cb9c60db2034fd30270ee021fc8b94c35e3b8e..830d445dad8468ad7fef20a61c3f594065e9d55b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1703,6 +1703,13 @@ static int kvm_prepare_memory_region(struct kvm *kvm, if (change != KVM_MR_DELETE) { if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) new->dirty_bitmap = NULL; +#ifdef CONFIG_HISI_VIRTCCA_HOST + else if (kvm_check_realm(kvm) && kvm_use_dirty_bitmap(kvm)) { + r = virtcca_kvm_prepare_dirty_bitmap(kvm, (void *)new); + if (r) + return r; + } +#endif else if (old && old->dirty_bitmap) new->dirty_bitmap = old->dirty_bitmap; else if (kvm_use_dirty_bitmap(kvm)) { @@ -2241,6 +2248,12 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) n = kvm_dirty_bitmap_bytes(memslot); flush = false; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (!kvm_check_realm(kvm)) + kvm->manual_dirty_log_protect = false; +#endif + if (kvm->manual_dirty_log_protect) { /* * Unlike kvm_get_dirty_log, we always return false in *flush, @@ -2274,6 +2287,10 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) KVM_MMU_UNLOCK(kvm); } +#ifdef CONFIG_HISI_VIRTCCA_HOST + virtcca_kvm_enable_log_dirty(kvm, (void *)memslot, &flush); +#endif + if (flush) kvm_flush_remote_tlbs_memslot(kvm, memslot);