kubevirt在360的探索之路（k8s接管虛擬化）

type DomainManager interface { //SyncVMI 為創(chuàng)建虛擬機(jī)SyncVMI(*v1.VirtualMachineInstance, bool, *cmdv1.VirtualMachineOptions) (*api.DomainSpec, error)//暫停VMIPauseVMI(*v1.VirtualMachineInstance) error//恢復(fù)暫停的VMIUnpauseVMI(*v1.VirtualMachineInstance) errorKillVMI(*v1.VirtualMachineInstance) error//刪除VMIDeleteVMI(*v1.VirtualMachineInstance) errorSignalShutdownVMI(*v1.VirtualMachineInstance) errorMarkGracefulShutdownVMI(*v1.VirtualMachineInstance) errorListAllDomains() ([]*api.Domain, error)//遷移VMIMigrateVMI(*v1.VirtualMachineInstance, *cmdclient.MigrationOptions) errorPrepareMigrationTarget(*v1.VirtualMachineInstance, bool) errorGetDomainStats() ([]*stats.DomainStats, error)//取消遷移CancelVMIMigration(*v1.VirtualMachineInstance) error//如下需要啟用Qemu guest agent，沒啟用會(huì)包VMI does not have guest agent connectedGetGuestInfo() (v1.VirtualMachineInstanceGuestAgentInfo, error)GetUsers() ([]v1.VirtualMachineInstanceGuestOSUser, error)GetFilesystems() ([]v1.VirtualMachineInstanceFileSystem, error)SetGuestTime(*v1.VirtualMachineInstance) error}

[root@openstack825 ~]# kubectl get vmi -o wideNAME                          AGE     PHASE     IP             NODENAME        LIVE-MIGRATABLEtest100.foo.demo.example.com     8d   Running   192.168.10.30  10.10.67.244   Truetest200.foo.demo.example.com     8d   Running   192.168.10.31  10.10.67.245   True

獲取已安裝的kubevirt pod[root@openstack825 ~]# kubectl  -n kubevirt get podNAME                               READY   STATUS    RESTARTS   AGEvirt-api-68c958dd-6sx4n            1/1     Running   0          14dvirt-api-68c958dd-sldgr            1/1     Running   0          14dvirt-controller-647d666bd5-gsnzf   1/1     Running   1          14dvirt-controller-647d666bd5-hshnz   1/1     Running   1          14dvirt-handler-4g7ck                 1/1     Running   3          14dvirt-handler-kzv86                 1/1     Running   0          14dvirt-handler-m2ppb                 1/1     Running   0          14dvirt-handler-v6fgt                 1/1     Running   0          14dvirt-operator-65ccf74f56-b82kz     1/1     Running   0          14dvirt-operator-65ccf74f56-zs2xq     1/1     Running   0          14dvirtvnc-947874d99-hn7k5            1/1     Running   0          6d19h

//主要通過調(diào)用磁盤、網(wǎng)絡(luò)相關(guān)函數(shù)，來(lái)判斷當(dāng)前VMI 是否適合遷移func (d *VirtualMachineController) calculateLiveMigrationCondition(vmi *v1.VirtualMachineInstance, hasHotplug bool) (*v1.VirtualMachineInstanceCondition, bool) {liveMigrationCondition := v1.VirtualMachineInstanceCondition{Type:   v1.VirtualMachineInstanceIsMigratable,Status: k8sv1.ConditionTrue,}//調(diào)用 checkvolume 方法isBlockMigration, err := d.checkVolumesForMigration(vmi)if err != nil {//如果返回錯(cuò)誤信息則會(huì)限制遷移liveMigrationCondition.Status = k8sv1.ConditionFalseliveMigrationCondition.Message = err.Error()liveMigrationCondition.Reason = v1.VirtualMachineInstanceReasonDisksNotMigratablereturn &liveMigrationCondition, isBlockMigration}//調(diào)用網(wǎng)絡(luò)模式檢查方法err = d.checkNetworkInterfacesForMigration(vmi)if err != nil {liveMigrationCondition = v1.VirtualMachineInstanceCondition{Type:    v1.VirtualMachineInstanceIsMigratable,Status:  k8sv1.ConditionFalse,Message: err.Error(),Reason:  v1.VirtualMachineInstanceReasonInterfaceNotMigratable,}return &liveMigrationCondition, isBlockMigration}if hasHotplug {liveMigrationCondition = v1.VirtualMachineInstanceCondition{Type:    v1.VirtualMachineInstanceIsMigratable,Status:  k8sv1.ConditionFalse,Message: "VMI has hotplugged disks",Reason:  v1.VirtualMachineInstanceReasonHotplugNotMigratable,}return &liveMigrationCondition, isBlockMigration}return &liveMigrationCondition, isBlockMigration}

//checkvolume 定義/檢查所有VMI卷共享可以在源和實(shí)時(shí)遷移的目的地之間熱遷移//當(dāng)所有卷均已共享且VMI沒有本地磁盤時(shí)，blockMigrate才返回True//某些磁盤組合使VMI不適合實(shí)時(shí)遷移， 在這種情況下，將返回相關(guān)錯(cuò)誤func (d *VirtualMachineController) checkVolumesForMigration(vmi *v1.VirtualMachineInstance) (blockMigrate bool, err error) {for _, volume := range vmi.Spec.Volumes {volSrc := volume.VolumeSourceif volSrc.PersistentVolumeClaim != nil || volSrc.DataVolume != nil {var volName stringif volSrc.PersistentVolumeClaim != nil {volName = volSrc.PersistentVolumeClaim.ClaimName} else {volName = volSrc.DataVolume.Name}//pvcutils.IsSharedPVCFromClient_, shared, err := pvcutils.IsSharedPVCFromClient(d.clientset, vmi.Namespace, volName)if errors.IsNotFound(err) {return blockMigrate, fmt.Errorf("persistentvolumeclaim %v not found", volName)} else if err != nil {return blockMigrate, err}if !shared {return true, fmt.Errorf("cannot migrate VMI with non-shared PVCs")}} else if volSrc.HostDisk != nil {shared := volSrc.HostDisk.Shared != nil && *volSrc.HostDisk.Sharedif !shared {return true, fmt.Errorf("cannot migrate VMI with non-shared HostDisk")}} else {blockMigrate = true}}return}func IsSharedPVCFromClient(client kubecli.KubevirtClient, namespace string, claimName string) (pvc *k8sv1.PersistentVolumeClaim, isShared bool, err error) {pvc, err = client.CoreV1().PersistentVolumeClaims(namespace).Get(claimName, v1.GetOptions{})if err == nil {//IsPVCSharedisShared = IsPVCShared(pvc)}return}//IsPVCShared Shared 判斷，函數(shù)返回bool 類型，成功則返回truefunc IsPVCShared(pvc *k8sv1.PersistentVolumeClaim) bool {//循環(huán)PVC的accessModesfor _, accessMode := range pvc.Spec.AccessModes {if accessMode == k8sv1.ReadWriteMany {return true}}return false}

[root@kubevirt01 ~]# kubectl get podNAME                                               READY   STATUS    RESTARTS   AGEcsi-rbdplugin-7bprd                                3/3     Running   0          14dcsi-rbdplugin-fl5c9                                3/3     Running   0          14dcsi-rbdplugin-ggj9q                                3/3     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-7qtnh         6/6     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-sdscf         6/6     Running   0          14dcsi-rbdplugin-provisioner-84bb9bdd56-xjz2r         6/6     Running   0          14dcsi-rbdplugin-svfv2

---apiVersion: v1kind: PersistentVolumeClaimmetadata: name: testzhangsanlisispec: accessModes:   - ReadWriteMany volumeMode: Block resources:   requests:     storage: 10Gi storageClassName: csi-rbd-sckubectl apply -f pvc-test.yaml[root@kubevirt01 ~]# kubectl get pvcNAME                           STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGEtestzhangsanlisi                       Bound    pvc-2c98391d-a2eb-4dd1-a1f0-2a8ef27d4ca3   10Gi       RWX            csi-rbd-sc     3s

(ceph-mon)[root@example100 /]# ceph -s cluster:   id:     d8ab2087-f55c-4b8f-913d-fc60d6fc455d   health: HEALTH_OK services:   mon: 3 daemons, quorum 192.168.10.100 192.168.20.100 192.168.30.100 (age 4d)   mgr: ceph100(active, since 4d), standbys: ceph200， ceph300   osd: 27 osds: 27 up (since 2w), 27 in (since 2w) data:   pools:   1 pools, 1024 pgs   objects: 55.91k objects, 218 GiB   usage:   682 GiB used, 98 TiB / 98 TiB avail   pgs:     1024 active+clean io:   client:   2.2 KiB/s wr, 0 op/s rd, 0 op/s wr(ceph-mon)[root@example100 /]# ceph dfRAW STORAGE:   CLASS     SIZE       AVAIL      USED        RAW USED     %RAW USED   hdd       98 TiB     98 TiB     655 GiB      682 GiB          0.68   TOTAL     98 TiB     98 TiB     655 GiB      682 GiB          0.68

//Demo Yaml//IP 地址段來(lái)自源與網(wǎng)絡(luò)物理設(shè)備分配時(shí)spec: cidrBlock: 192.168.10.0/23 default: true excludeIps: - 192.168.10.1 gateway: 192.168.10.1 gatewayNode: "" gatewayType: distributed // 需要設(shè)置成false natOutgoing: false private: false protocol: IPv4 provider: ovn //需要設(shè)置成true，若為false，會(huì)在主機(jī)側(cè)加上route，導(dǎo)致net不通 underlayGateway: true vlan: ovn-vlan

[root@kubevirt01 ~]# kubectl -n kube-system get podNAME                                   READY   STATUS    RESTARTS   AGEcoredns-65dbdb44db-8bxlr               1/1     Running   33         17dkube-ovn-cni-4v4xb                     1/1     Running   0          18dkube-ovn-cni-kvgrj                     1/1     Running   0          18dkube-ovn-cni-nj2pr                     1/1     Running   0          18dkube-ovn-cni-xv476                     1/1     Running   0          18dkube-ovn-controller-7f6db69b48-6c7w8   1/1     Running   0          18dkube-ovn-controller-7f6db69b48-82kjt   1/1     Running   0          18dkube-ovn-controller-7f6db69b48-mhkfc   1/1     Running   0          18dkube-ovn-pinger-n2rn4                  1/1     Running   0          18dkube-ovn-pinger-s4hrz                  1/1     Running   0          18dkube-ovn-pinger-tccz5                  1/1     Running   0          18dkube-ovn-pinger-x2tqq                  1/1     Running   0          18dovn-central-775c4ff46d-4nqjw           1/1     Running   1          18dovn-central-775c4ff46d-822v2           1/1     Running   0          18dovn-central-775c4ff46d-txkn8           1/1     Running   0          18dovs-ovn-mbpv2                          1/1     Running   0          18dovs-ovn-r9mvc                          1/1     Running   0          18dovs-ovn-wkxld                          1/1     Running   0          18dovs-ovn-z89hw                          1/1     Running   0          18d

Invalid value: 0x0: must be specified for an update

https://www.codeleading.com/article/27252474726/

https://github.com/kubevirt/client-pythonhttps://github.com/kubevirt/client-go

創(chuàng)建虛擬機(jī)( create_namespaced_virtual_machine)  注意：body是json格式，官方sdk的example有誤刪除虛擬機(jī)( delete_namespaced_virtual_machine)展示某個(gè)namespace下的vm資源( list_namespaced_virtual_machine)展示某個(gè)namespace下的vmi資源( list_namespaced_virtual_machine_instance)展示所有namespace下的vm資源( list_virtual_machine_for_all_namespaces)展示所有namespace下的vmi資源( list_virtual_machine_instance_for_all_namespaces)獲取某個(gè)namespace某個(gè)name下的vm資源( read_namespaced_virtual_machine)獲取某個(gè)namespace某個(gè)name下的vmi資源( read_namespaced_virtual_machine_instance) 注意：在獲取vm資源時(shí)無(wú)法獲取到node_name,uuid,ip的數(shù)據(jù)，獲取vmi資源時(shí)無(wú)法獲取到disk以及image_url的數(shù)據(jù)，筆者認(rèn)為vm是虛擬機(jī)資源，vmi是虛擬機(jī)實(shí)例資源，只有在running狀態(tài)下的vm才是vmi，由于k8s中ip是動(dòng)態(tài)分配的，因此才會(huì)出現(xiàn)node_name,uuid,ip數(shù)據(jù)在vm中獲取不到啟動(dòng)虛擬機(jī)( v1alpha3_start)停止虛擬機(jī)( v1alpha3_stop)重啟虛擬機(jī)( v1alpha3_restart) 注意: 重啟虛擬機(jī)只能在虛擬機(jī)狀態(tài)是running是才能調(diào)用，否則會(huì)失敗修改虛擬機(jī)名稱( v1alpha3_rename)替換虛擬機(jī)的配置文件( replace_namespaced_virtual_machine_instance)

def v1alpha3_rename_with_http_info(self, name, newName, namespace, **kwargs):   """   Rename a stopped VirtualMachine object.   This method makes a synchronous HTTP request by default. To make an   asynchronous HTTP request, please define a `callback` function   to be invoked when receiving the response.   >>> def callback_function(response):   >>>     pprint(response)   >>>   >>> thread = api.v1alpha3_rename_with_http_info(name, namespace, newName, callback=callback_function)   :param callback function: The callback function       for asynchronous request. (optional)   :param str name: Name of the resource (required)   :param str namespace: Object name and auth scope, such as for teams and projects (required)   :param str newName: NewName of the resource (required)   :return: str            If the method is called asynchronously,            returns the request thread.   """   all_params = ['name', 'namespace', 'newName']   all_params.append('callback')   all_params.append('_return_http_data_only')   all_params.append('_preload_content')   all_params.append('_request_timeout')   params = locals()   for key, val in iteritems(params['kwargs']):       if key not in all_params:           raise TypeError(               "Got an unexpected keyword argument '%s'"               " to method v1alpha3_rename" % key           )       params[key] = val   del params['kwargs']   # verify the required parameter 'name' is set   if ('name' not in params) or (params['name'] is None):       raise ValueError("Missing the required parameter `name` when calling `v1alpha3_rename`")   # verify the required parameter 'namespace' is set   if ('namespace' not in params) or (params['namespace'] is None):       raise ValueError("Missing the required parameter `namespace` when calling `v1alpha3_rename`")   collection_formats = {}   path_params = {}   # if 'name' in params:   #     path_params['name'] = params['name']   # if 'namespace' in params:   #     path_params['namespace'] = params['namespace']   query_params = []   header_params = {}   form_params = []   local_var_files = {}   body_params = {"newName": params["newName"]}   # Authentication setting   auth_settings = []   api_route = "/apis/subresources.kubevirt.io/v1alpha3/namespaces/{namespace}/virtualmachines/{name}/rename".format(namespace=params["namespace"], name=params["name"])   return self.api_client.call_api(api_route, 'PUT',                                   path_params,                                   query_params,                                   header_params,                                   body=body_params,                                   post_params=form_params,                                   files=local_var_files,                                   response_type='str',                                   auth_settings=auth_settings,                                   callback=params.get('callback'),                                   _return_http_data_only=params.get('_return_http_data_only'),                                   _preload_content=params.get('_preload_content', True),                                   _request_timeout=params.get('_request_timeout'),                                   collection_formats=collection_formats)