【kubernetes/k8s源碼分析】operator controller-runtime源碼分析

operator的原理

      kubernetes 自定義擴展資源註冊到 controller-manager，通過 list / watch 的方式監聽資源的變化，然後在週期內的各個環節做相應的協調處理。所謂的處理就是operator實現由狀態的應用的核心部分

     operator-sdk 和 Kubebuilder 是目前開發 operator 兩種常用的 framework，生成相應的模板而已

 

kubebuilder 使用

    創建project

         在 ${GOPATH}/src 目錄下創建目錄，進入目錄執行

         kubebuilder --domain krome.io --license apache2 --owner "zzl"

 

         kubebuilder create api --group apps --version v1 --kind TestKind

     參考： https://github.com/kubernetes-sigs/kubebuilder

 

operator-sdk 使用

     operator-sdk new krome

     operator-sdk add api --api-version=apps.krome.io/v1 --kind=Statefulset

     operator-sdk add controller --api-version=apps.krome.io/v1 --kind=Statefulset

    更新並生成自定義資源的代碼

        每當types.go文件有變更時，因爲有一些代碼文件依賴於types.go中定義的類型，所以它們需要重新生成。

  $ operator-sdk generate k8s

 

    kubebuild 與 operator-sdk 都使用了 controller-runtime 框架

    實例化 Manager，定義在 controller-runtime/pkg/manager/manager.go 的 New 函數，GetConfigOrDie 函數如果指定了環境變量 KUBECONFIG，則從這個路徑下連接 kubernetes，或者 in-cluster 模式，指定 QPS

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
    Scheme:             scheme,
    MetricsBindAddress: metricsAddr,
    Port:               9443,
    LeaderElection:     enableLeaderElection,
    LeaderElectionID:   "2b0c98f1.krome.io",
})

1. New 函數實例化 Manager 對象，實現了 Manager 接口

// New returns a new Manager for creating Controllers.
func New(config *rest.Config, options Options) (Manager, error) {
	// Initialize a rest.config if none was specified
	if config == nil {
		return nil, fmt.Errorf("must specify Config")
	}

	// Set default values for options fields
	options = setOptionsDefaults(options)

     1.1 setOptionDefaults 函數

       這裏設置默認的 cache，這個說一下，核心結構 InformersMap 

// InformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs.
// It uses a standard parameter codec constructed based on the given generated Scheme.
type InformersMap struct {
	// we abstract over the details of structured vs unstructured with the specificInformerMaps

	structured   *specificInformersMap
	unstructured *specificInformersMap

	// Scheme maps runtime.Objects to GroupVersionKinds
	Scheme *runtime.Scheme
}

    1.1.1 對於 option 中的 provider，這個主要是創建事件廣播器

// NewProvider create a new Provider instance.
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, broadcaster record.EventBroadcaster) (recorder.Provider, error) {
	clientSet, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("failed to init clientSet: %w", err)
	}

	p := &provider{scheme: scheme, logger: logger, eventBroadcaster: broadcaster}
	p.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientSet.CoreV1().Events("")})
	p.eventBroadcaster.StartEventWatcher(
		func(e *corev1.Event) {
			p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message)
		})

	return p, nil
}

    1.2 實例化 controllerManager 對象

      定義在 controller-runtime/pkg/manager/internal.go 中

return &controllerManager{
	config:                config,
	scheme:                options.Scheme,
	cache:                 cache,
	fieldIndexes:          cache,
	client:                writeObj,
	apiReader:             apiReader,
	recorderProvider:      recorderProvider,
	resourceLock:          resourceLock,
	mapper:                mapper,
	metricsListener:       metricsListener,
	internalStop:          stop,
	internalStopper:       stop,
	port:                  options.Port,
	host:                  options.Host,
	certDir:               options.CertDir,
	leaseDuration:         *options.LeaseDuration,
	renewDeadline:         *options.RenewDeadline,
	retryPeriod:           *options.RetryPeriod,
	healthProbeListener:   healthProbeListener,
	readinessEndpointName: options.ReadinessEndpointName,
	livenessEndpointName:  options.LivenessEndpointName,
}, nil

 

    啓動 manager 

if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
    setupLog.Error(err, "problem running manager")
    os.Exit(1)
}

2. contrllerManager 的 Start 方法

    serveMetrics 啓動 http server，/metrics

func (cm *controllerManager) Start(stop <-chan struct{}) error {
	// join the passed-in stop channel as an upstream feeding into cm.internalStopper
	defer close(cm.internalStopper)

	// initialize this here so that we reset the signal channel state on every start
	cm.errSignal = &errSignaler{errSignal: make(chan struct{})}

	// Metrics should be served whether the controller is leader or not.
	// (If we don't serve metrics for non-leaders, prometheus will still scrape
	// the pod but will get a connection refused)
	if cm.metricsListener != nil {
		go cm.serveMetrics(cm.internalStop)
	}

	// Serve health probes
	if cm.healthProbeListener != nil {
		go cm.serveHealthProbes(cm.internalStop)
	}

     2.1 controller.New 函數

      定義在 controller-runtime/pkg/controller/controller.go 中

// New returns a new Controller registered with the Manager.  The Manager will ensure that shared Caches have
// been synced before the Controller is Started.
func New(name string, mgr manager.Manager, options Options) (Controller, error) {
	if options.Reconciler == nil {
		return nil, fmt.Errorf("must specify Reconciler")
	}

	if len(name) == 0 {
		return nil, fmt.Errorf("must specify Name for Controller")
	}

	if options.MaxConcurrentReconciles <= 0 {
		options.MaxConcurrentReconciles = 1
	}

     2.1.1 實例化 Controller 對象

      定義在 controller-runtime/pkg/internal/controller/controller.go 中

// Create controller with dependencies set
c := &controller.Controller{
	Do:       options.Reconciler,
	Cache:    mgr.GetCache(),
	Config:   mgr.GetConfig(),
	Scheme:   mgr.GetScheme(),
	Client:   mgr.GetClient(),
	Recorder: mgr.GetEventRecorderFor(name),
	MakeQueue: func() workqueue.RateLimitingInterface {
		return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name)
	},
	MaxConcurrentReconciles: options.MaxConcurrentReconciles,
	Name:                    name,
}

     2.1.1 manager 的 Add 方法

// Add sets dependencies on i, and adds it to the list of Runnables to start.
func (cm *controllerManager) Add(r Runnable) error {
	cm.mu.Lock()
	defer cm.mu.Unlock()

	// Set dependencies on the object
	if err := cm.SetFields(r); err != nil {
		return err
	}

	var shouldStart bool

	// Add the runnable to the leader election or the non-leaderelection list
	if leRunnable, ok := r.(LeaderElectionRunnable); ok && !leRunnable.NeedLeaderElection() {
		shouldStart = cm.started
		cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r)
	} else {
		shouldStart = cm.startedLeader
		cm.leaderElectionRunnables = append(cm.leaderElectionRunnables, r)
	}

	if shouldStart {
		// If already started, start the controller
		go func() {
			if err := r.Start(cm.internalStop); err != nil {
				cm.errSignal.SignalError(err)
			}
		}()
	}

	return nil
}

   

2 Controller 的 Start 方法

    每一個 controller 都有自己的 queue

// Start implements controller.Controller
func (c *Controller) Start(stop <-chan struct{}) error {
	// use an IIFE to get proper lock handling
	// but lock outside to get proper handling of the queue shutdown
	c.mu.Lock()

	c.Queue = c.MakeQueue()

     2.1 啓動 worker，與原生 controller manager 雷同

       從隊列取 key，調用 reconcileHandler 進行處理

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the reconcileHandler is never invoked concurrently with the same object.
func (c *Controller) worker() {
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem will read a single work item off the workqueue and
// attempt to process it, by calling the reconcileHandler.
func (c *Controller) processNextWorkItem() bool {
	obj, shutdown := c.Queue.Get()
	if shutdown {
		// Stop working
		return false
	}

	// We call Done here so the workqueue knows we have finished
	// processing this item. We also must remember to call Forget if we
	// do not want this work item being re-queued. For example, we do
	// not call Forget if a transient error occurs, instead the item is
	// put back on the workqueue and attempted again after a back-off
	// period.
	defer c.Queue.Done(obj)

	return c.reconcileHandler(obj)
}

    2.2 reconcileHandler 函數

func (c *Controller) reconcileHandler(obj interface{}) bool {
	// Update metrics after processing each item
	reconcileStartTS := time.Now()
	defer func() {
		c.updateMetrics(time.Since(reconcileStartTS))
	}()

	var req reconcile.Request
	var ok bool
	if req, ok = obj.(reconcile.Request); !ok {
		// As the item in the workqueue is actually invalid, we call
		// Forget here else we'd go into a loop of attempting to
		// process a work item that is invalid.
		c.Queue.Forget(obj)
		log.Error(nil, "Queue item was not a Request",
			"controller", c.Name, "type", fmt.Sprintf("%T", obj), "value", obj)
		// Return true, don't take a break
		return true
	}

    2.3 調用 Reconcile 進行處理

     也就是自己實現控制器的 Reconcile 函數

// RunInformersAndControllers the syncHandler, passing it the namespace/Name string of the
// resource to be synced.
if result, err := c.Do.Reconcile(req); err != nil {
	c.Queue.AddRateLimited(req)
	log.Error(err, "Reconciler error", "controller", c.Name, "request", req)
	ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "error").Inc()
	return false
} else if result.RequeueAfter > 0 {
	// The result.RequeueAfter request will be lost, if it is returned
	// along with a non-nil error. But this is intended as
	// We need to drive to stable reconcile loops before queuing due
	// to result.RequestAfter
	c.Queue.Forget(obj)
	c.Queue.AddAfter(req, result.RequeueAfter)
	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue_after").Inc()
	return true
} else if result.Requeue {
	c.Queue.AddRateLimited(req)
	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue").Inc()
	return true
}

 

如果要生成zz_generated.defaults.go文件的話

a、在自定義的類型上，多添加一個tag, // +k8s:defaulter-gen=true

b、修改generate-groups.sh腳本，添加對應的模塊，或者直接執行命令

// defaulter-gen --input-dirs krome/pkg/apis/apps/v1 --input-dirs krome/pkg/apis/apps
// -o $GOPATH/src --go-header-file boilerplate.go.txt
// -O zz_generated.defaults
// --extra-peer-dirs= k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/apimachinery/pkg/conversion,k8s.io/apimachinery/pkg/runtime