【kubeedge源碼分析】edgecore源碼分析之二edged模塊

描述

• Edged：一個運行在 edge 節點的 agent 程序，管理邊緣的容器化應用程序，目前支持 docker 運行時，將來會支持更多的運行時，比如 containerd等

                                               Fig 1: EdgeD Functionalities

 

1. Register 函數

    根據 registerModules 中模塊註冊，將調用 edged 模塊，路徑 kubeedge/edge/pkg/edged/edged.go，註冊 edged 模塊

// Register register edged
func Register() {
	edged, err := newEdged()
	if err != nil {
		klog.Errorf("init new edged error, %v", err)
		return
	}
	core.Register(edged)
}

   1.1 newEged 函數

    newEged 實例化 eged 對象，並進行初始化，getConfig 函數主要是初始化 kubelet 啓動參數，包括 node 名字，namespace imageGC docker endpoint CRI 接口類型，dns，內存等等

    包括 pod manager 初始化，imageGC 出發策略等，實例化 eged 並初始化 kubelet 啓動參數，根目錄默認爲 /var/lib/edged

//newEdged creates new edged object and initialises it
func newEdged() (*edged, error) {
	conf := getConfig()
	backoff := flowcontrol.NewBackOff(backOffPeriod, MaxContainerBackOff)

	podManager := podmanager.NewPodManager()
	policy := images.ImageGCPolicy{
		HighThresholdPercent: conf.imageGCHighThreshold,
		LowThresholdPercent:  conf.imageGCLowThreshold,
		MinAge:               minAge,
	}
	// build new object to match interface
	recorder := record.NewEventRecorder()

     1.1.1 如果未設置運行時 endpoint，則設置默認未 unix:///var/run/dockershim.sock

if conf.remoteRuntimeEndpoint != "" {
	// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
	if conf.remoteImageEndpoint == "" {
		conf.remoteImageEndpoint = conf.remoteRuntimeEndpoint
	}
}

     1.1.2 如果運行時 endpoint 爲 docker 的情況

     目前支持的 runtime dockershim，初始化 docker 服務，啓動 dockershim GRPC 服務，並註冊 runtime 和 image服務

//create and start the docker shim running as a grpc server
if conf.remoteRuntimeEndpoint == DockerShimEndpoint || conf.remoteRuntimeEndpoint == DockerShimEndpointDeprecated {
	streamingConfig := &streaming.Config{}
	......
	if err := server.Start(); err != nil {
		return nil, err
	}

}

     1.1.3 使用 socket 建立 GRPC 連接 runtime 和 image 服務

containerRefManager := kubecontainer.NewRefManager()
httpClient := &http.Client{}
runtimeService, imageService, err := getRuntimeAndImageServices(conf.remoteRuntimeEndpoint, conf.remoteRuntimeEndpoint, conf.RuntimeRequestTimeout)
if err != nil {
	return nil, err
}
if ed.os == nil {
	ed.os = kubecontainer.RealOS{}
}

     1.1.4 容器運行時 manager 實例化

      複用了 kubelet 的 NewKubeGerericRuntimeManager 函數，實現在 kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager.go

var machineInfo cadvisorapi.MachineInfo
machineInfo.MemoryCapacity = uint64(conf.memoryCapacity)
containerRuntime, err := kuberuntime.NewKubeGenericRuntimeManager(
	recorder,
	ed.livenessManager,
	"",
	containerRefManager,
	&machineInfo,
	ed,
	ed.os,
	ed,
	httpClient,
	backoff,
	false,
	0,
	0,
	false,
	metav1.Duration{Duration: 100 * time.Millisecond},
	runtimeService,
	imageService,
	ed.clcm.InternalContainerLifecycle(),
	nil,
	nil,
)

     1.1.5 實例化 cadvisor manager

      複用了 kubelet 使用 cadvisor 模塊，實現在 kubernetes/pkg/kubelet/cm/container_manager_linux.go

cadvisorInterface, err := cadvisor.New("")
containerManager, err := cm.NewContainerManager(mount.New(""),
	cadvisorInterface,
	cm.NodeConfig{
		CgroupDriver:       conf.cgroupDriver,
		SystemCgroupsName:  conf.cgroupDriver,
		KubeletCgroupsName: conf.cgroupDriver,
		ContainerRuntime:   conf.runtimeType,
		KubeletRootDir:     DefaultRootDir,
	},
	false,
	conf.devicePluginEnabled,
	recorder)

     1.1.6 實例化 imageGC containerGC manager

     同樣使用了 kubelet 中該代碼，不再說明，可以參看 imageGC containerGC 文章

     https://blog.csdn.net/zhonglinzhang/article/details/92796057

     https://blog.csdn.net/zhonglinzhang/article/details/92771215

imageGCManager, err := images.NewImageGCManager(ed.containerRuntime, statsProvider, recorder, nodeRef, policy, conf.PodSandboxImage)
if err != nil {
	return nil, fmt.Errorf("failed to initialize image manager: %v", err)
}
ed.imageGCManager = imageGCManager

containerGCManager, err := kubecontainer.NewContainerGC(containerRuntime, containerGCPolicy, &containers.KubeSourcesReady{})
if err != nil {
	return nil, fmt.Errorf("init Container GC Manager failed with error %s", err.Error())
}

     1.1.7 實例化 volume plugin manager

     包裹了 kubernetes/pkg/volume/plugins.go，初始化 plugin

      可以參考：https://blog.csdn.net/zhonglinzhang/article/details/82800287

// NewInitializedVolumePluginMgr returns a new instance of volume.VolumePluginMgr
func NewInitializedVolumePluginMgr(
	edge *edged,
	plugins []volume.VolumePlugin) (*volume.VolumePluginMgr, error) {
	evh := &edgedVolumeHost{
		edge:            edge,
		volumePluginMgr: volume.VolumePluginMgr{},
	}

	if err := evh.volumePluginMgr.InitPlugins(plugins, nil, evh); err != nil {
		return nil, fmt.Errorf(
			"Could not initialize volume plugins for KubeletVolumePluginMgr: %v",
			err)
	}

	return &evh.volumePluginMgr, nil
}

   1.2 註冊 edged 模塊加入到全局變量 modules map中

// Register register module
func Register(m Module) {
	if isModuleEnabled(m.Name()) {
		modules[m.Name()] = m
		klog.Infof("Module %v registered", m.Name())
	} else {
		disabledModules[m.Name()] = m
		klog.Warningf("Module %v is not register, please check modules.yaml", m.Name())
	}
}

 

edged 同樣實現了  Module 接口 

// Module interface
type Module interface {
   Name() string
   Group() string
   Start()
   Cleanup()
}

 

2. edged 的 name 設置爲 edged，group 設置爲 edged

func (e *edged) Name() string {
	return modules.EdgedModuleName
}

func (e *edged) Group() string {
	return modules.EdgedGroup
}

 

   啓動模塊分別調用不同的 Start() 方法

3. edged 的 Start 函數

func (e *edged) Start() {
	e.metaClient = client.New()
	var ctx context.Context

	ctx, e.cancel = context.WithCancel(context.Background())
	// use self defined client to replace fake kube client
	e.kubeClient = fakekube.NewSimpleClientset(e.metaClient)

   3.1 NewManager 函數包裹了 kubelet 的 status manager

    參考 status manager：https://blog.csdn.net/zhonglinzhang/article/details/75102193

//NewManager creates and returns a new manager object
func NewManager(kubeClient clientset.Interface, podManager podmanager.Manager, podDeletionSafety status.PodDeletionSafetyProvider, metaClient client.CoreInterface) status.Manager {
	kubeManager := status.NewManager(kubeClient, podManager, podDeletionSafety)
	return &manager{
		Manager:           kubeManager,
		metaClient:        metaClient,
		podManager:        podManager,
		apiStatusVersions: make(map[types.UID]*v1.PodStatus),
	}
}

   3.2 initializeModules 函數調用了 kubelet 中 container manager 模塊啓動服務

func (e *edged) initializeModules() error {
	node, _ := e.initialNode()
	if err := e.containerManager.Start(node, e.GetActivePods, edgedutil.NewSourcesReady(), e.statusManager, e.runtimeService); err != nil {
		klog.Errorf("Failed to start device plugin manager %v", err)
		return err
	}
	return nil
}

   3.3 volume manager 

    不用看了，使用了 kubernetes 的 volume manager 並啓動，還有下面的 probe manager 

e.volumeManager = volumemanager.NewVolumeManager(
	true,
	types.NodeName(e.nodeName),
	e.podManager,
	e.statusManager,
	e.kubeClient,
	e.volumePluginMgr,
	e.containerRuntime,
	e.mounter,
	e.getPodsDir(),
	record.NewEventRecorder(),
	false,
	false,
)
go e.volumeManager.Run(edgedutil.NewSourcesReady(), utilwait.NeverStop)

   3.4 podAddWorkerRun

     併發 5 個 worker 處理新增 pod 的請求，調用 consumePodAddition 函數創建 pod，創建 pod 的工作目錄

func (e *edged) consumePodAddition(namespacedName *types.NamespacedName) error {
	podName := namespacedName.Name
	klog.Infof("start to consume added pod [%s]", podName)
	pod, ok := e.podManager.GetPodByName(namespacedName.Namespace, podName)
	if !ok || pod.DeletionTimestamp != nil {
		return apis.ErrPodNotFound
	}

	if err := e.makePodDataDirs(pod); err != nil {
		klog.Errorf("Unable to make pod data directories for pod %q: %v", format.Pod(pod), err)
		return err
	}

 

總結：

     edged 模塊的 Register 函數 newEdged 初始化如 kubelet 啓動的參數     

 

參考：

     https://docs.kubeedge.io/en/latest/modules/edge/edged.html