{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"去年写过一篇博客：","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s/5UXhXpwPDBh2xuGKq9Nqig","title":"","type":null},"content":[{"type":"text","text":"控制 Pod 内容器的启动顺序","attrs":{}}]},{"type":"text","text":"，分析了 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/tektoncd","title":"","type":null},"content":[{"type":"text","text":"TektonCD","attrs":{}}]},{"type":"text","text":" 的容器启动控制的原理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为什么要做容器启动顺序控制？我们都知道 Pod 中除了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"init-container","attrs":{}}],"attrs":{}},{"type":"text","text":" 之外，是允许添加多个容器的。类似 TektonCD 中 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"task","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 的概念就分别与 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"container","attrs":{}}],"attrs":{}},{"type":"text","text":" 对应，而 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 是按照顺序执行的。此外还有服务网格的场景，sidecar 容器需要在服务容器启动之前完成配置的加载，也需要对容器的启动顺序加以控制。否则，服务容器先启动，而 sidecar 还无法提供网络上的支持。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"现实","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/79/7906ea7faa727462209f32c60e7748b3.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"期望","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ee/ee1f04c7c54394134aea2f85a69c86a6.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"到了这里肯定有同学会问，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"spec.containers[]","attrs":{}}],"attrs":{}},{"type":"text","text":" 是一个数组，数组是有顺序的。Kubernetes 也确实是按照顺序来创建和启动容器，但是 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"容器启动成功，并不表示容器可以对外提供服务","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 Kubernetes 1.18 非正式版中曾在 Lifecycle 层面提供了对 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sidecar 类型容器的","attrs":{}}],"attrs":{}},{"type":"text","text":" 支持，但是最终该功能并","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/kubernetes/enhancements/issues/753#issuecomment-713471597","title":"","type":null},"content":[{"type":"text","text":"没有落地","attrs":{}}]},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那到底该怎么做？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"TL;DR","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"笔者准备了一个简单的 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample","title":"","type":null},"content":[{"type":"text","text":"go 项目","attrs":{}}]},{"type":"text","text":"，用于模拟 sidecar 的启动及配置加载。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"克隆代码后可以通过 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make build","attrs":{}}],"attrs":{}},{"type":"text","text":" 构建出镜像，假如你是用的 minikube 进行的实验，可以通过命令 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make load-2-minikube","attrs":{}}],"attrs":{}},{"type":"text","text":" 将镜像加载到 minikube 节点中。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用 Deployment 的方式进行部署，直接用 Pod 也可以。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"yaml"},"content":[{"type":"text","text":"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n creationTimestamp: null\n labels:\n app: sample\n name: sample\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: sample\n strategy: {}\n template:\n metadata:\n creationTimestamp: null\n labels:\n app: sample\n spec:\n containers:\n - image: addozhang/k8s-container-sequence-sidecar:latest\n name: sidecar\n imagePullPolicy: IfNotPresent\n lifecycle:\n postStart:\n exec:\n command:\n - /entrypoint\n - wait\n - image: busybox:latest\n name: app\n imagePullPolicy: IfNotPresent\n command: [\"/bin/sh\",\"-c\"]\n args: [\"date; echo 'app container started'; tail -f /dev/null\"]\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面的截图中，演示了在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sample","attrs":{}}],"attrs":{}},{"type":"text","text":" 命名空间中，pod 内两个容器的执行顺序。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb0a9570114a484ea586087d69771f2a.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Kubernetes 源码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 kubelet 的源码 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pkg/kubelet/kuberuntime/kuberuntime_manager.go","attrs":{}}],"attrs":{}},{"type":"text","text":" 中，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#SyncPod","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法用于创建 Pod，步骤比较繁琐，直接看第 7 步：创建普通容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// SyncPod syncs the running pod into the desired pod by executing following steps:\n//\n// 1. Compute sandbox and container changes.\n// 2. Kill pod sandbox if necessary.\n// 3. Kill any containers that should not be running.\n// 4. Create sandbox if necessary.\n// 5. Create ephemeral containers.\n// 6. Create init containers.\n// 7. Create normal containers.\nfunc (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {\n \n ...\n \n // Step 7: start containers in podContainerChanges.ContainersToStart.\n for _, idx := range podContainerChanges.ContainersToStart {\n start(\"container\", containerStartSpec(&pod.Spec.Containers[idx]))\n }\n\n return\n}\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#start","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法中调用了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#startContainer","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法，该方法会启动容器，并返回容器启动的结果。注意，这里的结果还 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"包含了容器的 Lifecycle hooks 调用","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"也就是说，假如容器的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" hook 没有正确的返回，kubelet 便不会去创建下一个容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// startContainer starts a container and returns a message indicates why it is failed on error.\n// It starts the container through the following steps:\n// * pull the image\n// * create the container\n// * start the container\n// * run the post start lifecycle hooks (if applicable)\nfunc (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {\n \n ...\n \n // Step 4: execute the post start hook.\n if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {\n kubeContainerID := kubecontainer.ContainerID{\n Type: m.runtimeName,\n ID: containerID,\n }\n msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)\n if handlerErr != nil {\n m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)\n if err := m.killContainer(pod, kubeContainerID, container.Name, \"FailedPostStartHook\", reasonFailedPostStartHook, nil); err != nil {\n klog.ErrorS(fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr), \"Failed to kill container\", \"pod\", klog.KObj(pod),\n \"podUID\", pod.UID, \"containerName\", container.Name, \"containerID\", kubeContainerID.String())\n }\n return msg, fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr)\n }\n }\n\n return \"\", nil\n}\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"实现方案","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/18/188be79f1012b4df36f8256fba3de21c.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample/blob/main/cmd/entrypoint/wait.go#L26","title":"","type":null},"content":[{"type":"text","text":"cmd/entrypoint/wait.go#L26","attrs":{}}]},{"type":"text","text":" （这里参考了 Istio 的 pilot-agent 实现 ）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" 中持续的去检查 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 断点，可以 hold 住当前容器的创建流程。保证 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 返回 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"200","attrs":{}}],"attrs":{}},{"type":"text","text":" 后，kubelet 才会去创建下一个容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这样就达到了前面截图中演示的效果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"for time.Now().Before(timeoutAt) {\n err = checkIfReady(client, url)\n if err == nil {\n log.Println(\"sidecar is ready\")\n return nil\n }\n log.Println(\"sidecar is not ready\")\n time.Sleep(time.Duration(periodMillis) * time.Millisecond)\n}\nreturn fmt.Errorf(\"sidecar is not ready in %d second(s)\", timeoutSeconds)\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"参考","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://banzaicloud.com/blog/k8s-sidecars/","title":"","type":null},"content":[{"type":"text","text":"Sidecar container lifecycle changes in Kubernetes 1.18","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74","title":"","type":null},"content":[{"type":"text","text":"Delaying application start until sidecar is ready","attrs":{}}]}]}]}],"attrs":{}}]}
Kubernetes 上如何控制容器的启动顺序？
{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"去年写过一篇博客：","attrs":{}},{"type":"link","attrs":{"href":"https://mp.weixin.qq.com/s/5UXhXpwPDBh2xuGKq9Nqig","title":"","type":null},"content":[{"type":"text","text":"控制 Pod 内容器的启动顺序","attrs":{}}]},{"type":"text","text":"，分析了 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/tektoncd","title":"","type":null},"content":[{"type":"text","text":"TektonCD","attrs":{}}]},{"type":"text","text":" 的容器启动控制的原理。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"为什么要做容器启动顺序控制？我们都知道 Pod 中除了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"init-container","attrs":{}}],"attrs":{}},{"type":"text","text":" 之外，是允许添加多个容器的。类似 TektonCD 中 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"task","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 的概念就分别与 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pod","attrs":{}}],"attrs":{}},{"type":"text","text":" 和 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"container","attrs":{}}],"attrs":{}},{"type":"text","text":" 对应，而 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"step","attrs":{}}],"attrs":{}},{"type":"text","text":" 是按照顺序执行的。此外还有服务网格的场景，sidecar 容器需要在服务容器启动之前完成配置的加载，也需要对容器的启动顺序加以控制。否则，服务容器先启动，而 sidecar 还无法提供网络上的支持。","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"现实","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/79/7906ea7faa727462209f32c60e7748b3.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"期望","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ee/ee1f04c7c54394134aea2f85a69c86a6.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"到了这里肯定有同学会问，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"spec.containers[]","attrs":{}}],"attrs":{}},{"type":"text","text":" 是一个数组，数组是有顺序的。Kubernetes 也确实是按照顺序来创建和启动容器，但是 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"容器启动成功，并不表示容器可以对外提供服务","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 Kubernetes 1.18 非正式版中曾在 Lifecycle 层面提供了对 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sidecar 类型容器的","attrs":{}}],"attrs":{}},{"type":"text","text":" 支持，但是最终该功能并","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/kubernetes/enhancements/issues/753#issuecomment-713471597","title":"","type":null},"content":[{"type":"text","text":"没有落地","attrs":{}}]},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"那到底该怎么做？","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"TL;DR","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"笔者准备了一个简单的 ","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample","title":"","type":null},"content":[{"type":"text","text":"go 项目","attrs":{}}]},{"type":"text","text":"，用于模拟 sidecar 的启动及配置加载。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"克隆代码后可以通过 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make build","attrs":{}}],"attrs":{}},{"type":"text","text":" 构建出镜像，假如你是用的 minikube 进行的实验，可以通过命令 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"make load-2-minikube","attrs":{}}],"attrs":{}},{"type":"text","text":" 将镜像加载到 minikube 节点中。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用 Deployment 的方式进行部署，直接用 Pod 也可以。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"yaml"},"content":[{"type":"text","text":"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n creationTimestamp: null\n labels:\n app: sample\n name: sample\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: sample\n strategy: {}\n template:\n metadata:\n creationTimestamp: null\n labels:\n app: sample\n spec:\n containers:\n - image: addozhang/k8s-container-sequence-sidecar:latest\n name: sidecar\n imagePullPolicy: IfNotPresent\n lifecycle:\n postStart:\n exec:\n command:\n - /entrypoint\n - wait\n - image: busybox:latest\n name: app\n imagePullPolicy: IfNotPresent\n command: [\"/bin/sh\",\"-c\"]\n args: [\"date; echo 'app container started'; tail -f /dev/null\"]\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面的截图中，演示了在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"sample","attrs":{}}],"attrs":{}},{"type":"text","text":" 命名空间中，pod 内两个容器的执行顺序。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/cb/cb0a9570114a484ea586087d69771f2a.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"Kubernetes 源码","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 kubelet 的源码 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"pkg/kubelet/kuberuntime/kuberuntime_manager.go","attrs":{}}],"attrs":{}},{"type":"text","text":" 中，","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#SyncPod","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法用于创建 Pod，步骤比较繁琐，直接看第 7 步：创建普通容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// SyncPod syncs the running pod into the desired pod by executing following steps:\n//\n// 1. Compute sandbox and container changes.\n// 2. Kill pod sandbox if necessary.\n// 3. Kill any containers that should not be running.\n// 4. Create sandbox if necessary.\n// 5. Create ephemeral containers.\n// 6. Create init containers.\n// 7. Create normal containers.\nfunc (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {\n \n ...\n \n // Step 7: start containers in podContainerChanges.ContainersToStart.\n for _, idx := range podContainerChanges.ContainersToStart {\n start(\"container\", containerStartSpec(&pod.Spec.Containers[idx]))\n }\n\n return\n}\n","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#start","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法中调用了 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"#startContainer","attrs":{}}],"attrs":{}},{"type":"text","text":" 方法，该方法会启动容器，并返回容器启动的结果。注意，这里的结果还 ","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"包含了容器的 Lifecycle hooks 调用","attrs":{}},{"type":"text","text":"。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"也就是说，假如容器的 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" hook 没有正确的返回，kubelet 便不会去创建下一个容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"// startContainer starts a container and returns a message indicates why it is failed on error.\n// It starts the container through the following steps:\n// * pull the image\n// * create the container\n// * start the container\n// * run the post start lifecycle hooks (if applicable)\nfunc (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {\n \n ...\n \n // Step 4: execute the post start hook.\n if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {\n kubeContainerID := kubecontainer.ContainerID{\n Type: m.runtimeName,\n ID: containerID,\n }\n msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)\n if handlerErr != nil {\n m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)\n if err := m.killContainer(pod, kubeContainerID, container.Name, \"FailedPostStartHook\", reasonFailedPostStartHook, nil); err != nil {\n klog.ErrorS(fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr), \"Failed to kill container\", \"pod\", klog.KObj(pod),\n \"podUID\", pod.UID, \"containerName\", container.Name, \"containerID\", kubeContainerID.String())\n }\n return msg, fmt.Errorf(\"%s: %v\", ErrPostStartHook, handlerErr)\n }\n }\n\n return \"\", nil\n}\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"实现方案","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/18/188be79f1012b4df36f8256fba3de21c.jpeg","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/addozhang/k8s-container-sequence-sample/blob/main/cmd/entrypoint/wait.go#L26","title":"","type":null},"content":[{"type":"text","text":"cmd/entrypoint/wait.go#L26","attrs":{}}]},{"type":"text","text":" （这里参考了 Istio 的 pilot-agent 实现 ）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"PostStart","attrs":{}}],"attrs":{}},{"type":"text","text":" 中持续的去检查 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 断点，可以 hold 住当前容器的创建流程。保证 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"/ready","attrs":{}}],"attrs":{}},{"type":"text","text":" 返回 ","attrs":{}},{"type":"codeinline","content":[{"type":"text","text":"200","attrs":{}}],"attrs":{}},{"type":"text","text":" 后，kubelet 才会去创建下一个容器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这样就达到了前面截图中演示的效果。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":"go"},"content":[{"type":"text","text":"for time.Now().Before(timeoutAt) {\n err = checkIfReady(client, url)\n if err == nil {\n log.Println(\"sidecar is ready\")\n return nil\n }\n log.Println(\"sidecar is not ready\")\n time.Sleep(time.Duration(periodMillis) * time.Millisecond)\n}\nreturn fmt.Errorf(\"sidecar is not ready in %d second(s)\", timeoutSeconds)\n","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"参考","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://banzaicloud.com/blog/k8s-sidecars/","title":"","type":null},"content":[{"type":"text","text":"Sidecar container lifecycle changes in Kubernetes 1.18","attrs":{}}]}]}]},{"type":"listitem","attrs":{"listStyle":null},"content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://medium.com/@marko.luksa/delaying-application-start-until-sidecar-is-ready-2ec2d21a7b74","title":"","type":null},"content":[{"type":"text","text":"Delaying application start until sidecar is ready","attrs":{}}]}]}]}],"attrs":{}}]}
發表評論
所有評論
還沒有人評論，想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章