【containerd 源碼分析】containerd cri 啓動註冊流程源碼分析

 

 

CRI

      CRI（Container Runtime Interface）是 Kubernetes 定義的與 contianer runtime 進行交互的接口，將 Kubernetes 與特定的容器解耦。Kubernetes早期的版本，對於容器環境的支持是通過 hard code 方式直接調用 Docker API，支持更多的容器運行時和更精簡的容器運行時，Kubernetes 提出了CRI。

 

 

 

    Containerd 在 1.0 及以前版本將 dockershim 和 docker daemon 替換爲 cri-containerd + containerd，在 1.1 版本直接將 cri-containerd 內置在 Containerd 中，作爲一個 CRI 插件

     Containerd 內置的 CRI 插件實現了 Kubelet CRI 接口中的 Image Service 和 Runtime Service，通過內部接口管理容器和鏡像，調用 CNI 插件給 Pod 配置網絡

// grpcServices are all the grpc services provided by cri containerd.
type grpcServices interface {
   runtime.RuntimeServiceServer
   runtime.ImageServiceServer
}

 

1. init 函數註冊 CRI 服務插件

     路徑 github.com/containerd/cri/cri.go，註冊 CRI 服務插件，類型爲 io.containerd.grpc.v1，ID 爲 cri

# ctr plugins ls
TYPE                            ID                    PLATFORMS      STATUS    
io.containerd.content.v1        content               -              ok        
io.containerd.grpc.v1           version               -              ok        
io.containerd.grpc.v1           cri                   linux/amd64    ok

// Register CRI service plugin
func init() {
	config := criconfig.DefaultConfig()
	plugin.Register(&plugin.Registration{
		Type:   plugin.GRPCPlugin,
		ID:     "cri",
		Config: &config,
		Requires: []plugin.Type{
			plugin.ServicePlugin,
		},
		InitFn: initCRIService,
	})
}

     1.1 initCRIService 函數

      配置信息，配置文件 /etc/containerd 目錄下 config.toml

• Snapshotter:overlayfs
• DefaultRuntimeName:runc
• ContainerdRootDir：/var/lib/containerd
• ContainerdEndpoint：/run/containerd/containerd.sock
• RootDir：/var/lib/containerd/io.containerd.grpc.v1.cri
• StateDir：/run/containerd/io.containerd.grpc.v1.cri

func initCRIService(ic *plugin.InitContext) (interface{}, error) {
	ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
	ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion}
	ctx := ic.Context
	pluginConfig := ic.Config.(*criconfig.PluginConfig)
	c := criconfig.Config{
		PluginConfig:       *pluginConfig,
		ContainerdRootDir:  filepath.Dir(ic.Root),
		ContainerdEndpoint: ic.Address,
		RootDir:            ic.Root,
		StateDir:           ic.State,
	}

     1.1.1 getServicesOpts

     getServicesOpts 函數從 plugin 上下文獲取服務選項，類型爲 io.containerd.service.v1，這個時實現了內部的服務

// getServicesOpts get service options from plugin context.
func getServicesOpts(ic *plugin.InitContext) ([]containerd.ServicesOpt, error) {
	plugins, err := ic.GetByType(plugin.ServicePlugin)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get service plugin")
	}

     1.1.1.1 定義 map 一系列服務

• content-service
• images-service
• snapshots-service
• containers-service
• tasks-service
• diff-service
• namespaces-service
• leases-service

for s, fn := range map[string]func(interface{}) containerd.ServicesOpt{
	services.ContentService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithContentStore(s.(content.Store))
	},
	services.ImagesService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithImageService(s.(images.ImagesClient))
	},
	services.SnapshotsService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithSnapshotters(s.(map[string]snapshots.Snapshotter))
	},
	services.ContainersService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithContainerService(s.(containers.ContainersClient))
	},
	services.TasksService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithTaskService(s.(tasks.TasksClient))
	},
	services.DiffService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithDiffService(s.(diff.DiffClient))
	},
	services.NamespacesService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithNamespaceService(s.(namespaces.NamespacesClient))
	},
	services.LeasesService: func(s interface{}) containerd.ServicesOpt {
		return containerd.WithLeasesService(s.(leases.Manager))
	},
}

      1.1.2 containerd.New 函數

      路徑 github.com/containerd/containerd/client.go，創建一個 client 連接到 containerd 實例

client, err := containerd.New(
	"",
	containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
	containerd.WithDefaultPlatform(criplatforms.Default()),
	containerd.WithServices(servicesOpts...),
)

      1.1.3 NewCRIService 函數實例化 criService 

// NewCRIService returns a new instance of CRIService
func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIService, error) {
	var err error
	c := &criService{
		config:             config,
		client:             client,
		os:                 osinterface.RealOS{},
		sandboxStore:       sandboxstore.NewStore(),
		containerStore:     containerstore.NewStore(),
		imageStore:         imagestore.NewStore(client),
		snapshotStore:      snapshotstore.NewStore(),
		sandboxNameIndex:   registrar.NewRegistrar(),
		containerNameIndex: registrar.NewRegistrar(),
		initialized:        atomic.NewBool(false),
	}

     實現了接口 CRIService，包括如下：定義 pkg/server/services.go

// CRIService is the interface implement CRI remote service server.
type CRIService interface {
   Run() error
   // io.Closer is used by containerd to gracefully stop cri service.
   io.Closer
   plugin.Service
   grpcServices
}

     grpcServices 實現了接口 Runtime 和 image 服務

// grpcServices are all the grpc services provided by cri containerd.
type grpcServices interface {
   runtime.RuntimeServiceServer
   runtime.ImageServiceServer
}

 

2. Run 函數啓動 CRI 服務

// Run starts the CRI service.
func (c *criService) Run() error {
	logrus.Info("Start subscribing containerd event")
	c.eventMonitor.subscribe(c.client)

	logrus.Infof("Start recovering state")
	if err := c.recover(ctrdutil.NamespacedContext()); err != nil {
		return errors.Wrap(err, "failed to recover state")
	}

	// Start event handler.
	logrus.Info("Start event monitor")
	eventMonitorErrCh := c.eventMonitor.start()

    2.1 recover 函數

     當 CRI down 掉重啓進行恢復工作

// NOTE: The recovery logic has following assumption: when the cri plugin is down:
// 1) Files (e.g. root directory, netns) and checkpoint maintained by the plugin MUST NOT be
// touched. Or else, recovery logic for those containers/sandboxes may return error.
// 2) Containerd containers may be deleted, but SHOULD NOT be added. Or else, recovery logic
// for the newly added container/sandbox will return error, because there is no corresponding root
// directory created.
// 3) Containerd container tasks may exit or be stoppped, deleted. Even though current logic could
// tolerant tasks being created or started, we prefer that not to happen.

// recover recovers system state from containerd and status checkpoint.
func (c *criService) recover(ctx context.Context) error {

     2.1.1 對所有 sandbox 進行恢復

    client.Containers 列出所有 sandbox 的容器，根據 lable io.cri-containerd.kind = sandbox

    loadSandbox 加載 sandbox 的 metadata，NewSandbox 實例化 Sandbox 包括 metadata，status， 網絡 namespace

    sandboxStore.Add 把 sandbox 加入到緩存中

// Recover all sandboxes.
sandboxes, err := c.client.Containers(ctx, filterLabel(containerKindLabel, containerKindSandbox))
	if err != nil {
		return errors.Wrap(err, "failed to list sandbox containers")
	}

for _, sandbox := range sandboxes {
	sb, err := c.loadSandbox(ctx, sandbox)
	if err != nil {
		log.G(ctx).WithError(err).Errorf("Failed to load sandbox %q", sandbox.ID())
		continue
	}
	log.G(ctx).Debugf("Loaded sandbox %+v", sb)
	if err := c.sandboxStore.Add(sb); err != nil {
		return errors.Wrapf(err, "failed to add sandbox %q to store", sandbox.ID())
	}
	if err := c.sandboxNameIndex.Reserve(sb.Name, sb.ID); err != nil {
		return errors.Wrapf(err, "failed to reserve sandbox name %q", sb.Name)
	}
}

     2.1.2 恢復所有 container

     列出所有 container，根據 label io.cri-containerd.kind = container 過濾

// Recover all containers.
containers, err := c.client.Containers(ctx, filterLabel(containerKindLabel, containerKindContainer))
if err != nil {
	return errors.Wrap(err, "failed to list containers")
}
for _, container := range containers {
	cntr, err := c.loadContainer(ctx, container)
	if err != nil {
		log.G(ctx).WithError(err).Errorf("Failed to load container %q", container.ID())
		continue
	}
	log.G(ctx).Debugf("Loaded container %+v", cntr)
	if err := c.containerStore.Add(cntr); err != nil {
		return errors.Wrapf(err, "failed to add container %q to store", container.ID())
	}
	if err := c.containerNameIndex.Reserve(cntr.Name, cntr.ID); err != nil {
		return errors.Wrapf(err, "failed to reserve container name %q", cntr.Name)
	}
}

     2.1.3 恢復所有 image，加載到 cache

// Recover all images.
cImages, err := c.client.ListImages(ctx)
if err != nil {
	return errors.Wrap(err, "failed to list images")
}
c.loadImages(ctx, cImages)