Kubernetes:kubelet 源碼分析之 pod 創建流程

---

0. 前言

kubelet 是運行在 Kubernetes 節點上的“節點代理”，用來管理節點。

kubelet 主要負責所在節點上的資源對象的管理，例如 Pod 資源對象的創建，刪除，監控，驅逐及生命週期管理等。

1. kubelet 源碼分析

1.1 kubelet 模塊

kubelet 包括的模塊如下圖：

從圖中可以看出，kubelet 的模塊衆多，每個模塊負責不同的功能。本文將圍繞創建 Pod 流程有取捨的介紹 kubelet 各個模塊。

在開始流程介紹前，讓我們通過 kubelet 工作原理圖將各個模塊串聯起來，這對於我們的源碼分析是相當有幫助的。

1.2 kubelet 啓動及調試

下載 Kubernetes 源碼，配置調試參數：

{
    "version": "0.2.0",
    "configurations": [
        {
            "name": "Kubelet",
            "type": "go",
            "request": "launch",
            "mode": "auto",
            "program": "${fileDirname}",
            "args": [
                "--container-runtime-endpoint=/run/k3s/containerd/containerd.sock",
                "-v=5",
                "--port=10251",
                "--kubeconfig=/root/.kube/config",
            ]
        }
    ]
}

打斷點進入 kubelet:

kubelet 使用 Cobra 作爲應用命令行框架，和 kube-scheduler，kube-apiserver 初始化過程類似，其流程如下：

這裏，簡要給出初始化示例代碼：

// kubernetes/cmd/kubelet/app/server.go
func NewKubeletCommand() *cobra.Command {
    // 解析 flags
	cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)
	cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
	kubeletFlags := options.NewKubeletFlags()

    // 獲取 kubelet 配置
    kubeletConfig, err := options.NewKubeletConfiguration()

    cmd := &cobra.Command{
		...
        RunE: func(cmd *cobra.Command, args []string) error {
			...

            // 構建 kubeletServer
            kubeletServer := &options.KubeletServer{
				KubeletFlags:         *kubeletFlags,
				KubeletConfiguration: *kubeletConfig,
			}

            // 構建 kubeletDeps，kubeletDeps 是運行 kubelet 需要的依賴項
            kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate)

            ...
            return Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)
        }
    }
}

進入 Run 函數運行 kubelet：

// kubernetes/cmd/kubelet/app/server.go
func Run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) error {
	...
	if err := run(ctx, s, kubeDeps, featureGate); err != nil {
		return fmt.Errorf("failed to run Kubelet: %w", err)
	}
	return nil
}

func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) (err error) {
    ...
    if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil {
		return err
	}
    ...
}

run 函數的內容比較多，我們直接忽略，有重點的看 RunKubelet：

// kubernetes/cmd/kubelet/app/server.go
func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencies, runOnce bool) error {
    ...
    k, err := createAndInitKubelet(kubeServer,
		kubeDeps,
		hostname,
		hostnameOverridden,
		nodeName,
		nodeIPs)
    if err != nil {
		return fmt.Errorf("failed to create kubelet: %w", err)
	}

    ...
    if runOnce {
		...
	} else {
		startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableServer)
		klog.InfoS("Started kubelet")
	}
	return nil
}

這裏 createAndInitKubelet 創建 kubelet 對象，該對象在 startKubelet 中運行：

// kubernetes/cmd/kubelet/app/server.go
func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, enableServer bool) {
	// start the kubelet
	go k.Run(podCfg.Updates())

	// start the kubelet server
	if enableServer {
		go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, kubeDeps.TracerProvider)
	}
	if kubeCfg.ReadOnlyPort > 0 {
		go k.ListenAndServeReadOnly(netutils.ParseIPSloppy(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort))
	}
	go k.ListenAndServePodResources()
}

startKubelet 調用 kubelet.Run 方法運行 kubelet。我們直接進入 kubelet.Run 方法看其中做了什麼。

// kubernetes/pkg/kubelet/kubelet.go
func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
    ...
    // 初始化模塊是初始化不依賴於 container runtime 的模塊
    if err := kl.initializeModules(); err != nil {
		kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.KubeletSetupFailed, err.Error())
		klog.ErrorS(err, "Failed to initialize internal modules")
		os.Exit(1)
	}

    ...
    kl.syncLoop(ctx, updates, kl)
}

Kubelet.Run 中包括了不少操作，這裏還是抓重點看 Kubelet.syncLoop 主邏輯做了什麼。

// kubernetes/pkg/kubelet/kubelet.go
func (kl *Kubelet) syncLoop(ctx context.Context, updates <-chan kubetypes.PodUpdate, handler SyncHandler) {
	klog.InfoS("Starting kubelet main sync loop")

    // syncTicker 每秒檢測一次是否有需要同步的 pod workers
    syncTicker := time.NewTicker(time.Second)
	defer syncTicker.Stop()

    // 每兩秒檢測一次是否有需要清理的 pod
	housekeepingTicker := time.NewTicker(housekeepingPeriod)
	defer housekeepingTicker.Stop()
    ...
	for {
		...
		kl.syncLoopMonitor.Store(kl.clock.Now())
		if !kl.syncLoopIteration(ctx, updates, handler, syncTicker.C, housekeepingTicker.C, plegCh) {
			break
		}
		kl.syncLoopMonitor.Store(kl.clock.Now())
	}
}

func (kl *Kubelet) syncLoopIteration(ctx context.Context, configCh <-chan kubetypes.PodUpdate, handler SyncHandler,
	syncCh <-chan time.Time, housekeepingCh <-chan time.Time, plegCh <-chan *pleg.PodLifecycleEvent) bool {
	select {
    case u, open := <-configCh:
        ...
        switch u.Op {
		case kubetypes.ADD:
			klog.V(2).InfoS("SyncLoop ADD", "source", u.Source, "pods", klog.KObjSlice(u.Pods))
			// After restarting, kubelet will get all existing pods through
			// ADD as if they are new pods. These pods will then go through the
			// admission process and *may* be rejected. This can be resolved
			// once we have checkpointing.
			handler.HandlePodAdditions(u.Pods)
            ...
        }
    }
}

Kubelet.syncLoopIteration 包括多個操作管道的行爲，這裏僅以 configCh 管道爲例，看創建 pod 的行爲。

在 handler.HandlePodAdditions(u.Pods) 這裏打斷點，然後創建 pod：

# helm install test .
NAME: test
LAST DEPLOYED: Sun May 19 15:34:54 2024
NAMESPACE: default
STATUS: deployed

I0519 15:34:54.577769 1801325 kubelet.go:2410] "SyncLoop ADD" source="api" pods=["default/test-6d47479b6b-pphb2"]

進入 handler.HandlePodAdditions

func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) {
	...
    for _, pod := range pods {
        // 獲取 podManager 模塊中記錄的 pods
		existingPods := kl.podManager.GetPods()

        // 更新 podManager 中的 pod
        kl.podManager.AddPod(pod)

        // 根據 pod 的屬性判斷當前 pod 是不是 mirrorPod
        // mirrorPod 是僅受 kubelet 管理的，對 kubernetes 不可見的 pod
        pod, mirrorPod, wasMirror := kl.podManager.GetPodAndMirrorPod(pod)
        if wasMirror {
            ...
        }

        // 判斷 pod 是否處於 termination 狀態
        if !kl.podWorkers.IsPodTerminationRequested(pod.UID) {
            activePods := kl.filterOutInactivePods(existingPods)
            if utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) {
                ...
            } else {
                // 判斷 pod 是否可以運行在當前 node
                if ok, reason, message := kl.canAdmitPod(activePods, pod); !ok {
					kl.rejectPod(pod, reason, message)
					continue
				}
            }
        }

        kl.podWorkers.UpdatePod(UpdatePodOptions{
			Pod:        pod,
			MirrorPod:  mirrorPod,
			UpdateType: kubetypes.SyncPodCreate,
			StartTime:  start,
		})
    }
}

這裏，podManager 模塊負責存儲和訪問 pod 的信息，維持 static pod 和 mirror pods 的關係，podManager 會被 statusManager/volumeManager/runtimeManager 調用，podManger 記錄所有被管理的 pod。

繼續往下看 podWorkers.UpdatePod：

# kubernetes/pkg/kubelet/pod_workers.go
func (p *podWorkers) UpdatePod(options UpdatePodOptions) {
    ...
    status, ok := p.podSyncStatuses[uid]
    if !ok {
        klog.V(4).InfoS("Pod is being synced for the first time", "pod", klog.KRef(ns, name), "podUID", uid, "updateType", options.UpdateType)
        firstTime = true
		status = &podSyncStatus{
			syncedAt: now,
			fullname: kubecontainer.BuildPodFullName(name, ns),
		}
        ...
        p.podSyncStatuses[uid] = status
    }

    ...
    // 創建一個 pod worker 協程，如果該協程不存在的話
    podUpdates, exists := p.podUpdates[uid]
	if !exists {
        podUpdates = make(chan struct{}, 1)
		p.podUpdates[uid] = podUpdates
        ...
        go func() {
			defer runtime.HandleCrash()
			defer klog.V(3).InfoS("Pod worker has stopped", "podUID", uid)
			p.podWorkerLoop(uid, outCh)
		}()
    }
}

func (p *podWorkers) podWorkerLoop(podUID types.UID, podUpdates <-chan struct{}) {
    var lastSyncTime time.Time
    for range podUpdates {
        // startPodSync 判斷 pod 是否可以被同步
		ctx, update, canStart, canEverStart, ok := p.startPodSync(podUID)

        ...
        err := func() error {
            var status *kubecontainer.PodStatus
            var err error
            switch {
			case update.Options.RunningPod != nil:
            default:
                status, err = p.podCache.GetNewerThan(update.Options.Pod.UID, lastSyncTime)
                ...
            }
        }

        switch {
			case update.WorkType == TerminatedPod:
            ...
            default:
				isTerminal, err = p.podSyncer.SyncPod(ctx, update.Options.UpdateType, update.Options.Pod, update.Options.MirrorPod, status)
			}

            lastSyncTime = p.clock.Now()
			return err
		}()

        ...
    }
}

這裏，要注意的是 podWorkers.podCache.GetNewerThan 獲取的是最新的 pod 狀態。其中，PLEG 獲取 container runtime 的 pod 狀態，存入 podCache 中。podCache 中的 pod 狀態和 kubelet 從 kube-apiserver 獲取的 pod 狀態做對比，以獲取最新的 pod 狀態。

接着，進入 podWorkers.podSyncer.SyncPod 同步 pod：

func (kl *Kubelet) SyncPod(ctx context.Context, updateType kubetypes.SyncPodType, pod, mirrorPod *v1.Pod, podStatus *kubecontainer.PodStatus) (isTerminal bool, err error) {
    ...
	klog.V(4).InfoS("SyncPod enter", "pod", klog.KObj(pod), "podUID", pod.UID)
    ...
    // 生成 apiPodStatus 以同步至 statusManager
    apiPodStatus := kl.generateAPIPodStatus(pod, podStatus, false)
    ...
    // 獲取 statusManager 中存儲的 pod 狀態
    existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID)
    ...
    // 調用 statusManager 同步 pod 狀態
    kl.statusManager.SetPodStatus(pod, apiPodStatus)

    ...
    // ensure the kubelet knows about referenced secrets or configmaps used by the pod
    if !kl.podWorkers.IsPodTerminationRequested(pod.UID) {
		if kl.secretManager != nil {
			kl.secretManager.RegisterPod(pod)
		}
		if kl.configMapManager != nil {
			kl.configMapManager.RegisterPod(pod)
		}
	}

    // 創建 pod container manager
    pcm := kl.containerManager.NewPodContainerManager()
    ...

    // Make data directories for the pod
	if err := kl.makePodDataDirs(pod); err != nil {
		...
	}

    // Wait for volumes to attach/mount
	if err := kl.volumeManager.WaitForAttachAndMount(ctx, pod); err != nil {
		...
	}

    // Fetch the pull secrets for the pod
	pullSecrets := kl.getPullSecretsForPod(pod)

	// Ensure the pod is being probed
	kl.probeManager.AddPod(pod)

    ...
    result := kl.containerRuntime.SyncPod(sctx, pod, podStatus, pullSecrets, kl.backOff)
    ...
}

Kubelet.SyncPod 首先更新 statusManager 中 pod 的狀態信息，接着開始創建 pod 所需要的資源，如 data directories，volumes，secrets。在調用 container runtime 同步 pod 前，將 pod 添加到 probeManger 模塊，以檢測 pod 狀態。這裏關於 probeManger 模塊的詳細內容可參考。

進入 Kubelet.containerRuntime.SyncPod 查看 container runtime 是怎麼同步 pod 的。

// kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager.go
func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {
	// Step 1: Compute sandbox and container changes.
	podContainerChanges := m.computePodActions(ctx, pod, podStatus)
    ...
    // Step 2: Kill the pod if the sandbox has changed.
	if podContainerChanges.KillPod {
        ...
    } else {
        // Step 3: kill any running containers in this pod which are not to keep.
        for containerID, containerInfo := range podContainerChanges.ContainersToKill {
            ...
        }
    }

    ...
    // Step 4: Create a sandbox for the pod if necessary.
    podSandboxID := podContainerChanges.SandboxID
	if podContainerChanges.CreateSandbox {
        ...
        createSandboxResult := kubecontainer.NewSyncResult(kubecontainer.CreatePodSandbox, format.Pod(pod))
		result.AddSyncResult(createSandboxResult)
        ...
        podSandboxID, msg, err = m.createPodSandbox(ctx, pod, podContainerChanges.Attempt)
        if err != nil {
            ...
        }

        // 調用 runtime cri 接口查詢創建的 pod sandbox 狀態
        resp, err := m.runtimeService.PodSandboxStatus(ctx, podSandboxID, false)
        ...
    }

    configPodSandboxResult := kubecontainer.NewSyncResult(kubecontainer.ConfigPodSandbox, podSandboxID)
	result.AddSyncResult(configPodSandboxResult)

    start := func(ctx context.Context, typeName, metricLabel string, spec *startSpec) error {
        ...
        klog.V(4).InfoS("Creating container in pod", "containerType", typeName, "container", spec.container, "pod", klog.KObj(pod))
        ...
        if msg, err := m.startContainer(ctx, podSandboxID, podSandboxConfig, spec, pod, podStatus, pullSecrets, podIP, podIPs); err != nil {
            ...
        }
        ...
    }

    // Step 5: start ephemeral containers
	for _, idx := range podContainerChanges.EphemeralContainersToStart {
		start(ctx, "ephemeral container", metrics.EphemeralContainer, ephemeralContainerStartSpec(&pod.Spec.EphemeralContainers[idx]))
	}

    if !utilfeature.DefaultFeatureGate.Enabled(features.SidecarContainers) {
        ...
    } else {
        // Step 6: start init containers.
		for _, idx := range podContainerChanges.InitContainersToStart {
            container := &pod.Spec.InitContainers[idx]
			// Start the next init container.
			if err := start(ctx, "init container", metrics.InitContainer, containerStartSpec(container)); err != nil {
                ...
            }

            // Successfully started the container; clear the entry in the failure
			klog.V(4).InfoS("Completed init container for pod", "containerName", container.Name, "pod", klog.KObj(pod))
        }
    }

    // Step 7: For containers in podContainerChanges.ContainersToUpdate[CPU,Memory] list, invoke UpdateContainerResources
	if isInPlacePodVerticalScalingAllowed(pod) {
		if len(podContainerChanges.ContainersToUpdate) > 0 || podContainerChanges.UpdatePodResources {
			m.doPodResizeAction(pod, podStatus, podContainerChanges, result)
		}
	}

    // Step 8: start containers in podContainerChanges.ContainersToStart.
	for _, idx := range podContainerChanges.ContainersToStart {
		start(ctx, "container", metrics.Container, containerStartSpec(&pod.Spec.Containers[idx]))
	}

	return

在 Kubelet.containerRuntime.SyncPod 中通過調用 runtime cri 接口創建 pod sandbox 和 container。以創建 pod sandbox 爲例，在 kubeGenericRuntimeManager.createPodSandbox 中調用 kubeGenericRuntimeManager.instrumentedRuntimeService.RunPodSandbox 創建 pod sandbox：

func (m *kubeGenericRuntimeManager) createPodSandbox(ctx context.Context, pod *v1.Pod, attempt uint32) (string, string, error) {
    podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt)
    if err != nil {
        ...
    }

    // 創建 pod 的 log 目錄
    err = m.osInterface.MkdirAll(podSandboxConfig.LogDirectory, 0755)
    ...

    podSandBoxID, err := m.runtimeService.RunPodSandbox(ctx, podSandboxConfig, runtimeHandler)
    if err != nil {
        ...
    }

    return podSandBoxID, "", nil
}

func (in instrumentedRuntimeService) RunPodSandbox(ctx context.Context, config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error) {
	...
	out, err := in.service.RunPodSandbox(ctx, config, runtimeHandler)
	...
	return out, err
}

func (r *remoteRuntimeService) RunPodSandbox(ctx context.Context, config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error) {
	...
	klog.V(10).InfoS("[RemoteRuntimeService] RunPodSandbox", "config", config, "runtimeHandler", runtimeHandler, "timeout", timeout)
	...
	resp, err := r.runtimeClient.RunPodSandbox(ctx, &runtimeapi.RunPodSandboxRequest{
		Config:         config,
		RuntimeHandler: runtimeHandler,
	})
    ...
}

// kubernetes/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go
func (c *runtimeServiceClient) RunPodSandbox(ctx context.Context, in *RunPodSandboxRequest, opts ...grpc.CallOption) (*RunPodSandboxResponse, error) {
	out := new(RunPodSandboxResponse)
	err := c.cc.Invoke(ctx, "/runtime.v1.RuntimeService/RunPodSandbox", in, out, opts...)
	if err != nil {
		return nil, err
	}
	return out, nil
}

可以看到，這裏通過調用 cri 接口的 /runtime.v1.RuntimeService/RunPodSandbox 創建 pod sandbox，至於創建 container 也是類似，調用 runtime cri 的接口實現創建 pod 的 container。

2. 小結

本文從 kubelet 源碼層面介紹了 pod 創建的流程，後續將重點看 runtime 是如何工作的。

---