關於看kubelet啓動代碼，本來是想了解一個問題，就是關於CNI，二進制部署的時候，使用flannel組網，在啓動kublet時並沒有傳遞network-plugin參數，那必然有默認值。
其實flannel的本質也是使用brigde方式進行pod的網絡接口配置操作，這和docker本身默認的docker0是一致的，但是這裏需要明白一點的kubelet的network-plugin的參數最後是傳遞給dockershim的來處理的，因此dockershim去判斷使用docker自己的還是cni插件去做。無論是docker自己去做還是flannel去做，是dockershim決定的。如何決定的呢？
因此有兩個參數需要深究:
從kubelet的啓動參數
| --network-plugin | <Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. This docker-specific flag only works when container-runtime is set to docker. |
| --container-runtime | The container runtime to use. Possible values: 'docker', 'remote', 'rkt (deprecated)'. (default "docker") |
//pkg/kubelet/dockershim/docker_service.go
func NewDockerService(config *ClientConfig, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings,
...
// dockershim currently only supports CNI plugins.
pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)
cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs)
cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs))
netHost := &dockerNetworkHost{
&namespaceGetter{ds},
&portMappingGetter{ds},
}
plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)
if err != nil {
return nil, fmt.Errorf("didn't find compatible CNI plugin with given settings %+v: %v", pluginSettings, err)
}
ds.network = network.NewPluginManager(plug)
glog.Infof("Docker cri networking managed by %v", plug.Name())
...
}
// pkg/kubelet/dockershim/network/plugin.go
// InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.
func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) (NetworkPlugin, error) {
if networkPluginName == "" {
// default to the no_op plugin
plug := &NoopNetworkPlugin{}
plug.Sysctl = utilsysctl.New()
if err := plug.Init(host, hairpinMode, nonMasqueradeCIDR, mtu); err != nil {
return nil, err
}
return plug, nil
}
pluginMap := map[string]NetworkPlugin{}
從以上代碼可以看出，我們在二進制部署的時候，是沒有配置network-plugin參數的，即代碼中會進入NoopNetworkPlugin分支，
從/var/log/message中也能看到kubelet的日誌“Docker cri networking managed by kubernetes.io/no-op"。
一句話：dockershim會判斷傳遞的network-plugin參數，沒配怎麼處理，配了怎麼處理。
以下代碼是看kubelet的默認參數的是如何定義的。
（一） kubelet的啓動入口：
app.NewKubeletCommand（）
#kubernets/cmd/kubelet/kubelet.go
func main() {
rand.Seed(time.Now().UTC().UnixNano())
command := app.NewKubeletCommand(server.SetupSignalHandler())
logs.InitLogs()
defer logs.FlushLogs()
if err := command.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}
# kubernetes/cmd/app/server.go
func NewKubeletCommand(stopCh <-chan struct{}) *cobra.Command {
cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)
cleanFlagSet.SetNormalizeFunc(flag.WordSepNormalizeFunc)
kubeletFlags := options.NewKubeletFlags()
kubeletConfig, err := options.NewKubeletConfiguration()
// programmer error
if err != nil {
glog.Fatal(err)
}
... （中間將來會運行）
// keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
kubeletFlags.AddFlags(cleanFlagSet)
options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
options.AddGlobalFlags(cleanFlagSet)
cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))
以上代碼中，options.NewKubeletFlags，以及下面的kubeletFlays.AddFlags是將配置參數默認值的地方
這裏options.NewKubeletFlags中，會初始化kubelet的各種配置。
func NewKubeletFlags() *KubeletFlags {
remoteRuntimeEndpoint := ""
if runtime.GOOS == "linux" {
remoteRuntimeEndpoint = "unix:///var/run/dockershim.sock"
} else if runtime.GOOS == "windows" {
remoteRuntimeEndpoint = "tcp://localhost:3735"
}
return &KubeletFlags{
EnableServer: true,
ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
CertDirectory: "/var/lib/kubelet/pki",
RootDirectory: defaultRootDir,
MasterServiceNamespace: metav1.NamespaceDefault,
MaxContainerCount: -1,
MaxPerPodContainerCount: 1,
MinimumGCAge: metav1.Duration{Duration: 0},
NonMasqueradeCIDR: "10.0.0.0/8",
RegisterSchedulable: true,
ExperimentalKernelMemcgNotification: false,
RemoteRuntimeEndpoint: remoteRuntimeEndpoint,
NodeLabels: make(map[string]string),
VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
RegisterNode: true,
SeccompProfileRoot: filepath.Join(defaultRootDir, "seccomp"),
HostNetworkSources: []string{kubetypes.AllSource},
HostPIDSources: []string{kubetypes.AllSource},
HostIPCSources: []string{kubetypes.AllSource},
// TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated
AllowPrivileged: true,
// prior to the introduction of this flag, there was a hardcoded cap of 50 images
NodeStatusMaxImages: 50,
}
}
在這裏有linux環境下，RemoteRuntimeEndpoint的定義。也有ContaineRuntimeOptions的初始化
func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions {
dockerEndpoint := ""
if runtime.GOOS != "windows" {
dockerEndpoint = "unix:///var/run/docker.sock"
}
return &config.ContainerRuntimeOptions{
ContainerRuntime: kubetypes.DockerContainerRuntime,
RedirectContainerStreaming: false,
DockerEndpoint: dockerEndpoint,
DockershimRootDirectory: "/var/lib/dockershim",
PodSandboxImage: defaultPodSandboxImage,
ImagePullProgressDeadline: metav1.Duration{Duration: 1 * time.Minute},
ExperimentalDockershim: false,
//Alpha feature
CNIBinDir: "/opt/cni/bin",
CNIConfDir: "/etc/cni/net.d",
}
}
====================分割線===================
之所以有上面的東西，其實我本意是想知道，既然我們在二進制部署時，沒有配置network-plugin，那麼kubelet在創建pod時，究竟容器的網絡是如何創建的。
從配置中能看到的是flannel啓動時，將本節點的網段傳遞了docker啓動參數。docker啓動後會以此設置docker0網橋的網段。
其實我們有以下幾個疑問:
- flannel這個插件的工作原理，爲什麼network-plugin沒有體現flannel， flannel最終實現跨節點的pod通信？
- kubelet對network-plugin參數的默認值是多少？針對該配置，對pod的網絡流程的影響？
（一）先回答第一個問題：
flannel在cni中是一個相對於calico等第三方cni比較特殊的一個插件，特殊在哪呢？https://github.com/containernetworking/plugins中也有說明，flannel屬於Meta: other plugins類，不能獨立工作，需要有Main類支持纔可以。而flannel本質是調用Main類中的bridge，這個和docker 默認的NetworkMode是一致的。因此二進制部署的時候，kubelet沒有指定flannel相關的配置，pod的網絡也配好了，肯定是docker自己配置的。
（二）再來看第二個問題：