K8S CSI容器存儲接口(一)：介紹以及原理

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/container-storage-interface/spec/blob/master/spec.md","title":""},"content":[{"type":"text","text":"容器存儲接口","attrs":{}}]},{"type":"text","text":"（CSI）是用於將任意塊和文件存儲系統暴露給諸如Kubernetes之類的容器編排系統（CO）上的容器化工作負載的標準。 使用CSI的第三方存儲提供商可以編寫和部署在Kubernetes中公開新存儲系統的插件，而無需接觸核心的Kubernetes代碼。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"具體來說，Kubernetes針對CSI規定了以下內容：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Kubelet到CSI驅動程序的通信","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - Kubelet通過Unix域套接字直接向CSI驅動程序發起CSI調用（例如","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"NodeStageVolume","attrs":{}}],"attrs":{}},{"type":"text","text":"，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"NodePublishVolume","attrs":{}}],"attrs":{}},{"type":"text","text":"等），以掛載和卸載卷。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - Kubelet通過kubelet插件註冊機制發現CSI驅動程序（以及用於與CSI驅動程序進行交互的Unix域套接字）。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - 因此，部署在Kubernetes上的所有CSI驅動程序","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"必須","attrs":{}},{"type":"text","text":"在每個受支持的節點上使用kubelet插件註冊機制進行註冊。","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Master到CSI驅動程序的通信","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - Kubernetes master組件不會直接（通過Unix域套接字或其他方式）與CSI驅動程序通信。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - Kubernetes master組件僅與Kubernetes API交互。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" - 因此，需要依賴於Kubernetes API的操作的CSI驅動程序（例如卷創建，卷attach，卷快照等）必須監聽Kubernetes API並針對它觸發適當的CSI操作（例如下面的一系列的external組件）。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"組件","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/8b/8b17561f8acee239cbc14a416396280b.png","alt":"CSI調用說明","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"CSI實現中的組件分爲兩部分：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由k8s官方維護的一系列external組件負責註冊CSI driver 或監聽k8s對象資源，從而發起csi driver調用，比如（node-driver-registrar，external-attacher，external-provisioner，external-resizer，external-snapshotter，livenessprobe）","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"各雲廠商or開發者自行開發的組件（需要實現CSI Identity，CSI Controller，CSI Node 接口）","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"RPC接口(開發商實現)","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Identity Service","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"service Identity {\n //返回driver的信息，比如名字，版本\n rpc GetPluginInfo(GetPluginInfoRequest)\n returns (GetPluginInfoResponse) {}\n //返回driver提供的能力，比如是否提供Controller Service,volume 訪問能能力\n rpc GetPluginCapabilities(GetPluginCapabilitiesRequest)\n returns (GetPluginCapabilitiesResponse) {}\n //探針\n rpc Probe (ProbeRequest)\n returns (ProbeResponse) {}\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Controller service","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"service Controller {\n //創建卷\n rpc CreateVolume (CreateVolumeRequest)\n returns (CreateVolumeResponse) {}\n //刪除卷\n rpc DeleteVolume (DeleteVolumeRequest)\n returns (DeleteVolumeResponse) {}\n //attach 卷\n rpc ControllerPublishVolume (ControllerPublishVolumeRequest)\n returns (ControllerPublishVolumeResponse) {}\n //unattach卷\n rpc ControllerUnpublishVolume (ControllerUnpublishVolumeRequest)\n returns (ControllerUnpublishVolumeResponse) {}\n //返回存儲卷的功能點，如是否支持掛載到多個節點上，是否支持多個節點同時讀寫\n rpc ValidateVolumeCapabilities (ValidateVolumeCapabilitiesRequest)\n returns (ValidateVolumeCapabilitiesResponse) {}\n //列出所有卷\n rpc ListVolumes (ListVolumesRequest)\n returns (ListVolumesResponse) {}\n //返回存儲資源池的可用空間大小\n rpc GetCapacity (GetCapacityRequest)\n returns (GetCapacityResponse) {}\n //返回controller插件的功能點，如是否支持GetCapacity接口，是否支持snapshot功能等\n rpc ControllerGetCapabilities (ControllerGetCapabilitiesRequest)\n returns (ControllerGetCapabilitiesResponse) {}\n //創建快照\n rpc CreateSnapshot (CreateSnapshotRequest)\n returns (CreateSnapshotResponse) {}\n //刪除快照\n rpc DeleteSnapshot (DeleteSnapshotRequest)\n returns (DeleteSnapshotResponse) {}\n //列出快照\n rpc ListSnapshots (ListSnapshotsRequest)\n returns (ListSnapshotsResponse) {}\n //擴容\n rpc ControllerExpandVolume (ControllerExpandVolumeRequest)\n returns (ControllerExpandVolumeResponse) {}\n //獲得卷\n rpc ControllerGetVolume (ControllerGetVolumeRequest)\n returns (ControllerGetVolumeResponse) {\n option (alpha_method) = true;\n }\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"Node Service","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"service Node {\n //如果存儲卷沒有格式化，首先要格式化。然後把存儲卷mount到一個臨時的目錄（這個目錄通常是節點上的一個全局目錄）。再通過NodePublishVolume將存儲卷mount到pod的目錄中。mount過程分爲2步，原因是爲了支持多個pod共享同一個volume（如NFS）。\n rpc NodeStageVolume (NodeStageVolumeRequest)\n returns (NodeStageVolumeResponse) {}\n //NodeStageVolume的逆操作，將一個存儲卷從臨時目錄umount掉\n rpc NodeUnstageVolume (NodeUnstageVolumeRequest)\n returns (NodeUnstageVolumeResponse) {}\n //將存儲卷從臨時目錄mount到目標目錄（pod目錄）\n rpc NodePublishVolume (NodePublishVolumeRequest)\n returns (NodePublishVolumeResponse) {}\n //將存儲卷從pod目錄umount掉\n rpc NodeUnpublishVolume (NodeUnpublishVolumeRequest)\n returns (NodeUnpublishVolumeResponse) {}\n //返回可用於該卷的卷容量統計信息。\n rpc NodeGetVolumeStats (NodeGetVolumeStatsRequest)\n returns (NodeGetVolumeStatsResponse) {}\n\n //noe上執行卷擴容\n rpc NodeExpandVolume(NodeExpandVolumeRequest)\n returns (NodeExpandVolumeResponse) {}\n\n //返回Node插件的功能點，如是否支持stage/unstage功能\n rpc NodeGetCapabilities (NodeGetCapabilitiesRequest)\n returns (NodeGetCapabilitiesResponse) {}\n //返回節點信息\n rpc NodeGetInfo (NodeGetInfoRequest)\n returns (NodeGetInfoResponse) {}\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"External 組件（k8s Team）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這部分組件是由k8s官方提供的，作爲k8s api跟csi driver的橋樑：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"node-driver-registrar","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" CSI node-driver-registrar是一個sidecar容器，可從CSI driver獲取驅動程序信息（使用NodeGetInfo），並使用kubelet插件註冊機制在該節點上的kubelet中對其進行註冊。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"external-attacher","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 它是一個sidecar容器，用於監視Kubernetes VolumeAttachment對象並針對驅動程序端點觸發CSI ControllerPublish和ControllerUnpublish操作","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"external-provisioner","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 它是一個sidecar容器，用於監視Kubernetes PersistentVolumeClaim對象並針對驅動程序端點觸發CSI CreateVolume和DeleteVolume操作。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" external-attacher還支持快照數據源。 如果將快照CRD資源指定爲PVC對象上的數據源，則此sidecar容器通過獲取SnapshotContent對象獲取有關快照的信息，並填充數據源字段，該字段向存儲系統指示應使用指定的快照填充新卷 。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"external-resizer","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 它是一個sidecar容器，用於監視Kubernetes API服務器上的PersistentVolumeClaim對象的改動，如果用戶請求在PersistentVolumeClaim對象上請求更多存儲，則會針對CSI端點觸發ControllerExpandVolume操作。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"external-snapshotter","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 它是一個sidecar容器，用於監視Kubernetes API服務器上的VolumeSnapshot和VolumeSnapshotContent CRD對象。創建新的VolumeSnapshot對象（引用與此驅動程序對應的SnapshotClass CRD對象）將導致sidecar容器提供新的快照。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 該Sidecar偵聽指示成功創建VolumeSnapshot的服務，並立即創建VolumeSnapshotContent資源。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"livenessprobe","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 它是一個sidecar容器，用於監視CSI驅動程序的運行狀況，並通過","attrs":{}},{"type":"link","attrs":{"href":"https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/","title":""},"content":[{"type":"text","text":"Liveness Probe機制","attrs":{}}]},{"type":"text","text":"將其報告給Kubernetes。 這使Kubernetes能夠自動檢測驅動程序問題並重新啓動Pod以嘗試解決問題。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"參考","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://kubernetes-csi.github.io/docs/introduction.html","title":""},"content":[{"type":"text","text":"kubernetes-csi-introduction","attrs":{}}]}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://draveness.me/kubernetes-volume/","title":""},"content":[{"type":"text","text":"詳解 Kubernetes Volume 的實現原理","attrs":{}}]}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://www.dazhuanlan.com/2020/01/31/5e33a33ba05d1/","title":""},"content":[{"type":"text","text":"CSI存儲接口解釋","attrs":{}}]}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"關注公衆號，獲取最新文章推送：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5d/5deb67509e4bddb6d90e87cd91a03562.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}