diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go index 52f75dfc2..a2c1f800d 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ b/cmd/kk/apis/kubekey/v1alpha2/default.go @@ -63,6 +63,7 @@ const ( DefaultMaxPods = 110 DefaultPodPidsLimit = 10000 DefaultNodeCidrMaskSize = 24 + DefaultNodeCidrMaskSizeIPv6 = 120 DefaultIPIPMode = "Always" DefaultVXLANMode = "Never" DefaultVethMTU = 0 @@ -139,6 +140,9 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*Kub if cfg.Kubernetes.NodeCidrMaskSize == 0 { clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize } + if cfg.Kubernetes.NodeCidrMaskSizeIPv6 == 0 { + clusterCfg.Kubernetes.NodeCidrMaskSizeIPv6 = DefaultNodeCidrMaskSizeIPv6 + } if cfg.Kubernetes.ProxyMode == "" { clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode } diff --git a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go index 9531f4817..6807bb129 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go @@ -32,6 +32,7 @@ type Kubernetes struct { MaxPods int `yaml:"maxPods" json:"maxPods,omitempty"` PodPidsLimit int `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"` NodeCidrMaskSize int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"` + NodeCidrMaskSizeIPv6 int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSizeIPv6,omitempty"` ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"` ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"` AutoRenewCerts *bool `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"` diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go index 727b4db81..d53fb770c 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/network_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/network_types.go @@ -28,14 +28,24 @@ type NetworkConfig struct { } type CalicoCfg struct { - IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"` - VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` - VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"` - Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"` - DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"` - EnableTypha *bool `yaml:"enableTypha" json:"enableTypha,omitempty"` - Replicas int `yaml:"replicas" json:"replicas,omitempty"` - NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` + IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"` + VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` + VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"` + Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"` + DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"` + Typha Typha `yaml:"typha" json:"typha,omitempty"` + Controller Controller `yaml:"controller" json:"controller,omitempty"` +} + +type Typha struct { + Replicas int `yaml:"replicas" json:"replicas,omitempty"` + NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` + Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` +} + +type Controller struct { + Replicas int `yaml:"replicas" json:"replicas,omitempty"` + NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` } type FlannelCfg struct { @@ -186,12 +196,12 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool { return *c.DefaultIPPOOL } -// Typha is used to determine whether to enable calico Typha -func (c *CalicoCfg) Typha() bool { - if c.EnableTypha == nil { +// EnableTypha is used to determine whether to enable calico Typha +func (c *CalicoCfg) EnableTypha() bool { + if c.Typha.Enabled == nil { return false } - return *c.EnableTypha + return *c.Typha.Enabled } // EnableInit is used to determine whether to create default network diff --git a/cmd/kk/pkg/bootstrap/os/module.go b/cmd/kk/pkg/bootstrap/os/module.go index ef34ae4ef..6d41df945 100644 --- a/cmd/kk/pkg/bootstrap/os/module.go +++ b/cmd/kk/pkg/bootstrap/os/module.go @@ -64,7 +64,8 @@ func (c *ConfigureOSModule) Init() { Template: templates.InitOsScriptTmpl, Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"), Data: util.Data{ - "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), + "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), + "IPv6Support": templates.EnabledIPv6(c.KubeConf), }, }, Parallel: true, diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go index d9042eff2..ad9e6f2c7 100644 --- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go +++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go @@ -99,11 +99,16 @@ echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf +{{- if .IPv6Support }} #add for ipv6 echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf +echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf +echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf +{{- end}} #See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf @@ -145,6 +150,18 @@ sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.d sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf +{{- if .IPv6Support }} +#add for ipv6 +sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf +{{- end}} + + tmpfile="$$.tmp" awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf mv $tmpfile /etc/sysctl.conf @@ -287,3 +304,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [ hostsList = append(hostsList, lbHost) return hostsList } + +func EnabledIPv6(kubeConf *common.KubeConf) bool { + if len(strings.Split(kubeConf.Cluster.Network.KubePodsCIDR, ",")) == 2 { + return true + } + return false +} diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go index 016b65d28..0f32c57d9 100644 --- a/cmd/kk/pkg/kubernetes/tasks.go +++ b/cmd/kk/pkg/kubernetes/tasks.go @@ -307,6 +307,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { "BootstrapToken": bootstrapToken, "CertificateKey": certificateKey, "IPv6Support": host.GetInternalIPv6Address() != "", + "NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6, }, } diff --git a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go index 6d30206a5..092d97816 100644 --- a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go +++ b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go @@ -98,7 +98,7 @@ controllerManager: extraArgs: {{- if .IPv6Support }} node-cidr-mask-size-ipv4: "{{ .NodeCidrMaskSize }}" - node-cidr-mask-size-ipv6: "64" + node-cidr-mask-size-ipv6: "{{ .NodeCidrMaskSizeIPv6 }}" {{- else }} node-cidr-mask-size: "{{ .NodeCidrMaskSize }}" {{- end }} diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go index c470d45f6..3773ce7dd 100644 --- a/cmd/kk/pkg/plugins/network/tasks.go +++ b/cmd/kk/pkg/plugins/network/tasks.go @@ -460,7 +460,7 @@ func (g *GenerateCalicoManifests) Execute(runtime connector.Runtime) error { "CalicoFlexvolImage": images.GetImage(runtime, g.KubeConf, "calico-flexvol").ImageName(), "CalicoControllersImage": images.GetImage(runtime, g.KubeConf, "calico-kube-controllers").ImageName(), "CalicoTyphaImage": images.GetImage(runtime, g.KubeConf, "calico-typha").ImageName(), - "TyphaEnabled": len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.Typha(), + "TyphaEnabled": len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.EnableTypha(), "VethMTU": g.KubeConf.Cluster.Network.Calico.VethMTU, "NodeCidrMaskSize": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, "IPIPMode": g.KubeConf.Cluster.Network.Calico.IPIPMode, @@ -469,8 +469,11 @@ func (g *GenerateCalicoManifests) Execute(runtime connector.Runtime) error { "IPV4POOLNATOUTGOING": g.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(), "DefaultIPPOOL": g.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(), "IPv6Support": IPv6Support, - "Replicas": g.KubeConf.Cluster.Network.Calico.Replicas, - "NodeSelector": g.KubeConf.Cluster.Network.Calico.NodeSelector, + "NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6, + "TyphaReplicas": g.KubeConf.Cluster.Network.Calico.Typha.Replicas, + "TyphaNodeSelector": g.KubeConf.Cluster.Network.Calico.Typha.NodeSelector, + "ControllerReplicas": g.KubeConf.Cluster.Network.Calico.Controller.Replicas, + "ControllerNodeSelector": g.KubeConf.Cluster.Network.Calico.Controller.NodeSelector, }, } templateAction.Init(nil, nil) diff --git a/cmd/kk/pkg/plugins/network/templates/calico.tmpl b/cmd/kk/pkg/plugins/network/templates/calico.tmpl index 23a50f784..c9ef0a055 100644 --- a/cmd/kk/pkg/plugins/network/templates/calico.tmpl +++ b/cmd/kk/pkg/plugins/network/templates/calico.tmpl @@ -88,9 +88,17 @@ data: "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, +{{- if .IPv6Support }} + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "true", + "assign_ipv6": "true" + }, +{{- else }} "ipam": { "type": "calico-ipam" }, +{{- end }} "policy": { "type": "k8s" }, @@ -5023,7 +5031,7 @@ spec: - name: CALICO_IPV6POOL_CIDR value: "{{ .KubePodsV6CIDR }}" - name: CALICO_IPV6POOL_BLOCK_SIZE - value: "120" + value: "{{ .NodeCidrMaskSizeIPv6 }}" {{- end }} {{- else }} - name: NO_DEFAULT_POOLS @@ -5072,6 +5080,9 @@ spec: - /bin/calico-node - -felix-live - -bird-live +{{- if .IPv6Support }} + - -bird6-live +{{- end }} periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 @@ -5082,6 +5093,9 @@ spec: - /bin/calico-node - -felix-ready - -bird-ready +{{- if .IPv6Support }} + - -bird6-ready +{{- end }} periodSeconds: 10 timeoutSeconds: 10 volumeMounts: @@ -5171,7 +5185,7 @@ metadata: k8s-app: calico-kube-controllers spec: # The controllers can only have a single active instance. - replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }} + replicas: {{ if .ControllerReplicas }}{{ .ControllerReplicas }}{{ else }}1{{ end }} selector: matchLabels: k8s-app: calico-kube-controllers @@ -5186,8 +5200,8 @@ spec: spec: nodeSelector: kubernetes.io/os: linux -{{ if .NodeSelector }} - {{- toYaml .NodeSelector | indent 8 }} +{{ if .ControllerNodeSelector }} + {{- toYaml .ControllerNodeSelector | indent 8 }} {{- end }} tolerations: # Mark the pod as a critical add-on for rescheduling. @@ -5197,7 +5211,7 @@ spec: effect: NoSchedule - key: node-role.kubernetes.io/control-plane effect: NoSchedule -{{ if .NodeSelector }} +{{ if .ControllerNodeSelector }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -5258,7 +5272,7 @@ spec: # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }} + replicas: {{ if .TyphaReplicas }}{{ .TyphaReplicas }}{{ else }}1{{ end }} revisionHistoryLimit: 2 selector: matchLabels: @@ -5285,14 +5299,14 @@ spec: spec: nodeSelector: kubernetes.io/os: linux -{{ if .NodeSelector }} - {{- toYaml .NodeSelector | indent 8 }} +{{ if .TyphaNodeSelector }} + {{- toYaml .TyphaNodeSelector | indent 8 }} {{- end }} hostNetwork: true # Typha supports graceful shut down, disconnecting clients slowly during the grace period. # The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value. terminationGracePeriodSeconds: 300 -{{ if .NodeSelector }} +{{ if .TyphaNodeSelector }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/docs/config-example.md b/docs/config-example.md index 9e9f9d299..dcac38177 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -148,8 +148,8 @@ spec: ipipMode: Always # IPIP Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, vxlanMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Always] vxlanMode: Never # VXLAN Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, ipipMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Never] vethMTU: 0 # The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. By default, MTU is auto-detected. [Default: 0] - kubePodsCIDR: 10.233.64.0/18,fc00::/48 - kubeServiceCIDR: 10.233.0.0/18,fd00::/108 + kubePodsCIDR: 10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112 + kubeServiceCIDR: 10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116 storage: openebs: basePath: /var/openebs/local # base path of the local PV provisioner