{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"去年寫過一篇博客：","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s/5UXhXpwPDBh2xuGKq9Nqig","title":"","type":null},"content":[{"type":"text","text":"控制 Pod 內容器的啓動順序","attrs":{}}]},{"type":"text","text":"，分析了 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/tektoncd","title":"","type":null},"content":[{"type":"text","text":"TektonCD","attrs":{}}]},{"type":"text","text":" 的容器啓動控制的原理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"爲什麼要做容器啓動順序控制？我們都知道 Pod 中除了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"init-container","attrs":{}}],"attrs":{}},{"type":"text","text":" 之外，是允許添加多個容器的。類似 TektonCD 中 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"task","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 的概念就分別與 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"container","attrs":{}}],"attrs":{}},{"type":"text","text":" 對應，而 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 是按照順序執行的。此外還有服務網格的場景，sidecar 容器需要在服務容器啓動之前完成配置的加載，也需要對容器的啓動順序加以控制。否則，服務容器先啓動，而 sidecar 還無法提供網絡上的支持。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"現實","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/79/7906ea7faa727462209f32c60e7748b3.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"期望","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ee/ee1f04c7c54394134aea2f85a69c86a6.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"到了這裏肯定有同學會問，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"spec.containers[]","attrs":{}}],"attrs":{}},{"type":"text","text":" 是一個數組，數組是有順序的。Kubernetes 也確實是按照順序來創建和啓動容器，但是 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"容器啓動成功，並不表示容器可以對外提供服務","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 Kubernetes 1.18 非正式版中曾在 Lifecycle 層面提供了對 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sidecar 類型容器的","attrs":{}}],"attrs":{}},{"type":"text","text":" 支持，但是最終該功能並","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/kubernetes/enhancements/issues/753#issuecomment-713471597","title":"","type":null},"content":[{"type":"text","text":"沒有落地","attrs":{}}]},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那到底該怎麼做？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"TL;DR","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"筆者準備了一個簡單的 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample","title":"","type":null},"content":[{"type":"text","text":"go 項目","attrs":{}}]},{"type":"text","text":"，用於模擬 sidecar 的啓動及配置加載。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"克隆代碼後可以通過 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make build","attrs":{}}],"attrs":{}},{"type":"text","text":" 構建出鏡像，假如你是用的 minikube 進行的實驗，可以通過命令 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make load-2-minikube","attrs":{}}],"attrs":{}},{"type":"text","text":" 將鏡像加載到 minikube 節點中。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用 Deployment 的方式進行部署，直接用 Pod 也可以。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"yaml"},"content":[{"type":"text","text":"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n creationTimestamp: null\n labels:\n app: sample\n name: sample\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: sample\n strategy: {}\n template:\n metadata:\n creationTimestamp: null\n labels:\n app: sample\n spec:\n containers:\n - image: addozhang/k8s-container-sequence-sidecar:latest\n name: sidecar\n imagePullPolicy: IfNotPresent\n lifecycle:\n postStart:\n exec:\n command:\n - /entrypoint\n - wait\n - image: busybox:latest\n name: app\n imagePullPolicy: IfNotPresent\n command: [\"/bin/sh\",\"-c\"]\n args: [\"date; echo 'app container started'; tail -f /dev/null\"]\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面的截圖中，演示了在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sample","attrs":{}}],"attrs":{}},{"type":"text","text":" 命名空間中，pod 內兩個容器的執行順序。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb0a9570114a484ea586087d69771f2a.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Kubernetes 源碼","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 kubelet 的源碼 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pkg/kubelet/kuberuntime/kuberuntime_manager.go","attrs":{}}],"attrs":{}},{"type":"text","text":" 中，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#SyncPod","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法用於創建 Pod，步驟比較繁瑣，直接看第 7 步：創建普通容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// SyncPod syncs the running pod into the desired pod by executing following steps:\n//\n// 1. Compute sandbox and container changes.\n// 2. Kill pod sandbox if necessary.\n// 3. Kill any containers that should not be running.\n// 4. Create sandbox if necessary.\n// 5. Create ephemeral containers.\n// 6. Create init containers.\n// 7. Create normal containers.\nfunc (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {\n \n ...\n \n // Step 7: start containers in podContainerChanges.ContainersToStart.\n for _, idx := range podContainerChanges.ContainersToStart {\n start(\"container\", containerStartSpec(&pod.Spec.Containers[idx]))\n }\n\n return\n}\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#start","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法中調用了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#startContainer","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法，該方法會啓動容器，並返回容器啓動的結果。注意，這裏的結果還 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"包含了容器的 Lifecycle hooks 調用","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"也就是說，假如容器的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" hook 沒有正確的返回，kubelet 便不會去創建下一個容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// startContainer starts a container and returns a message indicates why it is failed on error.\n// It starts the container through the following steps:\n// * pull the image\n// * create the container\n// * start the container\n// * run the post start lifecycle hooks (if applicable)\nfunc (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {\n \n ...\n \n // Step 4: execute the post start hook.\n if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {\n kubeContainerID := kubecontainer.ContainerID{\n Type: m.runtimeName,\n ID: containerID,\n }\n msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)\n if handlerErr != nil {\n m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)\n if err := m.killContainer(pod, kubeContainerID, container.Name, \"FailedPostStartHook\", reasonFailedPostStartHook, nil); err != nil {\n klog.ErrorS(fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr), \"Failed to kill container\", \"pod\", klog.KObj(pod),\n \"podUID\", pod.UID, \"containerName\", container.Name, \"containerID\", kubeContainerID.String())\n }\n return msg, fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr)\n }\n }\n\n return \"\", nil\n}\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"實現方案","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/18/188be79f1012b4df36f8256fba3de21c.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample/blob/main/cmd/entrypoint/wait.go#L26","title":"","type":null},"content":[{"type":"text","text":"cmd/entrypoint/wait.go#L26","attrs":{}}]},{"type":"text","text":" （這裏參考了 Istio 的 pilot-agent 實現 ）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" 中持續的去檢查 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 斷點，可以 hold 住當前容器的創建流程。保證 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 返回 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"200","attrs":{}}],"attrs":{}},{"type":"text","text":" 後，kubelet 纔會去創建下一個容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這樣就達到了前面截圖中演示的效果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"for time.Now().Before(timeoutAt) {\n err = checkIfReady(client, url)\n if err == nil {\n log.Println(\"sidecar is ready\")\n return nil\n }\n log.Println(\"sidecar is not ready\")\n time.Sleep(time.Duration(periodMillis) * time.Millisecond)\n}\nreturn fmt.Errorf(\"sidecar is not ready in %d second(s)\", timeoutSeconds)\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"參考","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://banzaicloud.com/blog/k8s-sidecars/","title":"","type":null},"content":[{"type":"text","text":"Sidecar container lifecycle changes in Kubernetes 1.18","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74","title":"","type":null},"content":[{"type":"text","text":"Delaying application start until sidecar is ready","attrs":{}}]}]}]}],"attrs":{}}]}
Kubernetes 上如何控制容器的啓動順序？
{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"去年寫過一篇博客：","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s/5UXhXpwPDBh2xuGKq9Nqig","title":"","type":null},"content":[{"type":"text","text":"控制 Pod 內容器的啓動順序","attrs":{}}]},{"type":"text","text":"，分析了 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/tektoncd","title":"","type":null},"content":[{"type":"text","text":"TektonCD","attrs":{}}]},{"type":"text","text":" 的容器啓動控制的原理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"爲什麼要做容器啓動順序控制？我們都知道 Pod 中除了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"init-container","attrs":{}}],"attrs":{}},{"type":"text","text":" 之外，是允許添加多個容器的。類似 TektonCD 中 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"task","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 的概念就分別與 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"container","attrs":{}}],"attrs":{}},{"type":"text","text":" 對應，而 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 是按照順序執行的。此外還有服務網格的場景，sidecar 容器需要在服務容器啓動之前完成配置的加載，也需要對容器的啓動順序加以控制。否則，服務容器先啓動，而 sidecar 還無法提供網絡上的支持。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"現實","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/79/7906ea7faa727462209f32c60e7748b3.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"期望","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ee/ee1f04c7c54394134aea2f85a69c86a6.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"到了這裏肯定有同學會問，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"spec.containers[]","attrs":{}}],"attrs":{}},{"type":"text","text":" 是一個數組，數組是有順序的。Kubernetes 也確實是按照順序來創建和啓動容器，但是 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"容器啓動成功，並不表示容器可以對外提供服務","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 Kubernetes 1.18 非正式版中曾在 Lifecycle 層面提供了對 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sidecar 類型容器的","attrs":{}}],"attrs":{}},{"type":"text","text":" 支持，但是最終該功能並","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/kubernetes/enhancements/issues/753#issuecomment-713471597","title":"","type":null},"content":[{"type":"text","text":"沒有落地","attrs":{}}]},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那到底該怎麼做？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"TL;DR","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"筆者準備了一個簡單的 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample","title":"","type":null},"content":[{"type":"text","text":"go 項目","attrs":{}}]},{"type":"text","text":"，用於模擬 sidecar 的啓動及配置加載。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"克隆代碼後可以通過 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make build","attrs":{}}],"attrs":{}},{"type":"text","text":" 構建出鏡像，假如你是用的 minikube 進行的實驗，可以通過命令 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make load-2-minikube","attrs":{}}],"attrs":{}},{"type":"text","text":" 將鏡像加載到 minikube 節點中。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用 Deployment 的方式進行部署，直接用 Pod 也可以。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"yaml"},"content":[{"type":"text","text":"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n creationTimestamp: null\n labels:\n app: sample\n name: sample\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: sample\n strategy: {}\n template:\n metadata:\n creationTimestamp: null\n labels:\n app: sample\n spec:\n containers:\n - image: addozhang/k8s-container-sequence-sidecar:latest\n name: sidecar\n imagePullPolicy: IfNotPresent\n lifecycle:\n postStart:\n exec:\n command:\n - /entrypoint\n - wait\n - image: busybox:latest\n name: app\n imagePullPolicy: IfNotPresent\n command: [\"/bin/sh\",\"-c\"]\n args: [\"date; echo 'app container started'; tail -f /dev/null\"]\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面的截圖中，演示了在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sample","attrs":{}}],"attrs":{}},{"type":"text","text":" 命名空間中，pod 內兩個容器的執行順序。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb0a9570114a484ea586087d69771f2a.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Kubernetes 源碼","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 kubelet 的源碼 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pkg/kubelet/kuberuntime/kuberuntime_manager.go","attrs":{}}],"attrs":{}},{"type":"text","text":" 中，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#SyncPod","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法用於創建 Pod，步驟比較繁瑣，直接看第 7 步：創建普通容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// SyncPod syncs the running pod into the desired pod by executing following steps:\n//\n// 1. Compute sandbox and container changes.\n// 2. Kill pod sandbox if necessary.\n// 3. Kill any containers that should not be running.\n// 4. Create sandbox if necessary.\n// 5. Create ephemeral containers.\n// 6. Create init containers.\n// 7. Create normal containers.\nfunc (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {\n \n ...\n \n // Step 7: start containers in podContainerChanges.ContainersToStart.\n for _, idx := range podContainerChanges.ContainersToStart {\n start(\"container\", containerStartSpec(&pod.Spec.Containers[idx]))\n }\n\n return\n}\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#start","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法中調用了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#startContainer","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法，該方法會啓動容器，並返回容器啓動的結果。注意，這裏的結果還 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"包含了容器的 Lifecycle hooks 調用","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"也就是說，假如容器的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" hook 沒有正確的返回，kubelet 便不會去創建下一個容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// startContainer starts a container and returns a message indicates why it is failed on error.\n// It starts the container through the following steps:\n// * pull the image\n// * create the container\n// * start the container\n// * run the post start lifecycle hooks (if applicable)\nfunc (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {\n \n ...\n \n // Step 4: execute the post start hook.\n if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {\n kubeContainerID := kubecontainer.ContainerID{\n Type: m.runtimeName,\n ID: containerID,\n }\n msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)\n if handlerErr != nil {\n m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)\n if err := m.killContainer(pod, kubeContainerID, container.Name, \"FailedPostStartHook\", reasonFailedPostStartHook, nil); err != nil {\n klog.ErrorS(fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr), \"Failed to kill container\", \"pod\", klog.KObj(pod),\n \"podUID\", pod.UID, \"containerName\", container.Name, \"containerID\", kubeContainerID.String())\n }\n return msg, fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr)\n }\n }\n\n return \"\", nil\n}\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"實現方案","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/18/188be79f1012b4df36f8256fba3de21c.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample/blob/main/cmd/entrypoint/wait.go#L26","title":"","type":null},"content":[{"type":"text","text":"cmd/entrypoint/wait.go#L26","attrs":{}}]},{"type":"text","text":" （這裏參考了 Istio 的 pilot-agent 實現 ）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" 中持續的去檢查 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 斷點，可以 hold 住當前容器的創建流程。保證 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 返回 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"200","attrs":{}}],"attrs":{}},{"type":"text","text":" 後，kubelet 纔會去創建下一個容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這樣就達到了前面截圖中演示的效果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"for time.Now().Before(timeoutAt) {\n err = checkIfReady(client, url)\n if err == nil {\n log.Println(\"sidecar is ready\")\n return nil\n }\n log.Println(\"sidecar is not ready\")\n time.Sleep(time.Duration(periodMillis) * time.Millisecond)\n}\nreturn fmt.Errorf(\"sidecar is not ready in %d second(s)\", timeoutSeconds)\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"參考","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://banzaicloud.com/blog/k8s-sidecars/","title":"","type":null},"content":[{"type":"text","text":"Sidecar container lifecycle changes in Kubernetes 1.18","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74","title":"","type":null},"content":[{"type":"text","text":"Delaying application start until sidecar is ready","attrs":{}}]}]}]}],"attrs":{}}]}
發表評論
所有評論
還沒有人評論，想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章