diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go index 29a90ccc9..86571caa3 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ b/cmd/kk/apis/kubekey/v1alpha2/default.go @@ -62,6 +62,7 @@ const ( DefaultMaxPods = 110 DefaultPodPidsLimit = 10000 DefaultNodeCidrMaskSize = 24 + DefaultNodeCidrMaskSizeIPv6 = 120 DefaultIPIPMode = "Always" DefaultVXLANMode = "Never" DefaultVethMTU = 0 @@ -138,6 +139,9 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*Kub if cfg.Kubernetes.NodeCidrMaskSize == 0 { clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize } + if cfg.Kubernetes.NodeCidrMaskSizeIPv6 == 0 { + clusterCfg.Kubernetes.NodeCidrMaskSizeIPv6 = DefaultNodeCidrMaskSizeIPv6 + } if cfg.Kubernetes.ProxyMode == "" { clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode } diff --git a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go index 9531f4817..fe15f24ac 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go @@ -32,6 +32,7 @@ type Kubernetes struct { MaxPods int `yaml:"maxPods" json:"maxPods,omitempty"` PodPidsLimit int `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"` NodeCidrMaskSize int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"` + NodeCidrMaskSizeIPv6 int `yaml:"nodeCidrMaskSizeIPv6" json:"nodeCidrMaskSizeIPv6,omitempty"` ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"` ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"` AutoRenewCerts *bool `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"` diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go index d53fb770c..856b9760a 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/network_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/network_types.go @@ -32,6 +32,7 @@ type CalicoCfg struct { VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"` Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"` + Ipv6NatOutgoing *bool `yaml:"ipv6NatOutgoing" json:"ipv6NatOutgoing,omitempty"` DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"` Typha Typha `yaml:"typha" json:"typha,omitempty"` Controller Controller `yaml:"controller" json:"controller,omitempty"` @@ -188,6 +189,14 @@ func (c *CalicoCfg) EnableIPV4POOL_NAT_OUTGOING() bool { return *c.Ipv4NatOutgoing } +// EnableIPV6POOL_NAT_OUTGOING is used to determine whether to enable CALICO_IPV6POOL_NAT_OUTGOING. +func (c *CalicoCfg) EnableIPV6POOL_NAT_OUTGOING() bool { + if c.Ipv6NatOutgoing == nil { + return false + } + return *c.Ipv6NatOutgoing +} + // EnableDefaultIPPOOL is used to determine whether to create default ippool func (c *CalicoCfg) EnableDefaultIPPOOL() bool { if c.DefaultIPPOOL == nil { diff --git a/cmd/kk/pkg/bootstrap/os/module.go b/cmd/kk/pkg/bootstrap/os/module.go index ef34ae4ef..6d41df945 100644 --- a/cmd/kk/pkg/bootstrap/os/module.go +++ b/cmd/kk/pkg/bootstrap/os/module.go @@ -64,7 +64,8 @@ func (c *ConfigureOSModule) Init() { Template: templates.InitOsScriptTmpl, Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"), Data: util.Data{ - "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), + "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), + "IPv6Support": templates.EnabledIPv6(c.KubeConf), }, }, Parallel: true, diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go index 09e14c878..9d886b77d 100644 --- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go +++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go @@ -18,6 +18,7 @@ package templates import ( "fmt" + "strings" "text/template" "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" @@ -95,11 +96,16 @@ echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf +{{- if .IPv6Support }} #add for ipv6 echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf +echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf +echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf +echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf +{{- end}} #See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf @@ -138,6 +144,18 @@ sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf. sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf +{{- if .IPv6Support }} +#add for ipv6 +sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf +sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf +{{- end}} + + tmpfile="$$.tmp" awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf mv $tmpfile /etc/sysctl.conf @@ -266,3 +284,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [ hostsList = append(hostsList, lbHost) return hostsList } + +func EnabledIPv6(kubeConf *common.KubeConf) bool { + if len(strings.Split(kubeConf.Cluster.Network.KubePodsCIDR, ",")) == 2 { + return true + } + return false +} diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go index 8bfce5abb..674da53d2 100644 --- a/cmd/kk/pkg/kubernetes/tasks.go +++ b/cmd/kk/pkg/kubernetes/tasks.go @@ -307,6 +307,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { "BootstrapToken": bootstrapToken, "CertificateKey": certificateKey, "IPv6Support": host.GetInternalIPv6Address() != "", + "NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6, }, } diff --git a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go index 6d30206a5..092d97816 100644 --- a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go +++ b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go @@ -98,7 +98,7 @@ controllerManager: extraArgs: {{- if .IPv6Support }} node-cidr-mask-size-ipv4: "{{ .NodeCidrMaskSize }}" - node-cidr-mask-size-ipv6: "64" + node-cidr-mask-size-ipv6: "{{ .NodeCidrMaskSizeIPv6 }}" {{- else }} node-cidr-mask-size: "{{ .NodeCidrMaskSize }}" {{- end }} diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go index 3bdc62b9d..e72703b05 100644 --- a/cmd/kk/pkg/plugins/network/tasks.go +++ b/cmd/kk/pkg/plugins/network/tasks.go @@ -467,8 +467,10 @@ func (g *GenerateCalicoManifests) Execute(runtime connector.Runtime) error { "VXLANMode": g.KubeConf.Cluster.Network.Calico.VXLANMode, "ConatinerManagerIsIsula": g.KubeConf.Cluster.Kubernetes.ContainerManager == "isula", "IPV4POOLNATOUTGOING": g.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(), + "IPV6POOLNATOUTGOING": g.KubeConf.Cluster.Network.Calico.EnableIPV6POOL_NAT_OUTGOING(), "DefaultIPPOOL": g.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(), "IPv6Support": IPv6Support, + "NodeCidrMaskSizeIPv6": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6, "TyphaReplicas": g.KubeConf.Cluster.Network.Calico.Typha.Replicas, "TyphaNodeSelector": g.KubeConf.Cluster.Network.Calico.Typha.NodeSelector, "ControllerReplicas": g.KubeConf.Cluster.Network.Calico.Controller.Replicas, diff --git a/cmd/kk/pkg/plugins/network/templates/calico.tmpl b/cmd/kk/pkg/plugins/network/templates/calico.tmpl index 74e0c18b7..52170c7c1 100644 --- a/cmd/kk/pkg/plugins/network/templates/calico.tmpl +++ b/cmd/kk/pkg/plugins/network/templates/calico.tmpl @@ -88,9 +88,17 @@ data: "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, +{{- if .IPv6Support }} + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "true", + "assign_ipv6": "true" + }, +{{- else }} "ipam": { "type": "calico-ipam" }, +{{- end }} "policy": { "type": "k8s" }, @@ -4981,17 +4989,16 @@ spec: value: "false" {{- end }} {{- if .IPv6Support }} + # Enable IPIP + - name: CALICO_IPV6POOL_IPIP + value: "{{ .IPIPMode }}" # Enable or Disable VXLAN on the default IPv6 IP pool. - name: CALICO_IPV6POOL_VXLAN - value: "Always" + value: "{{ .VXLANMode }}" +{{- if .IPV6POOLNATOUTGOING }} - name: CALICO_IPV6POOL_NAT_OUTGOING value: "true" -{{- else }} - # Enable or Disable VXLAN on the default IPv6 IP pool. - - name: CALICO_IPV6POOL_VXLAN - value: "Never" - - name: CALICO_IPV6POOL_NAT_OUTGOING - value: "false" +{{- end }} {{- end }} # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU @@ -5023,7 +5030,7 @@ spec: - name: CALICO_IPV6POOL_CIDR value: "{{ .KubePodsV6CIDR }}" - name: CALICO_IPV6POOL_BLOCK_SIZE - value: "120" + value: "{{ .NodeCidrMaskSizeIPv6 }}" {{- end }} {{- else }} - name: NO_DEFAULT_POOLS @@ -5072,6 +5079,9 @@ spec: - /bin/calico-node - -felix-live - -bird-live +{{- if .IPv6Support }} + - -bird6-live +{{- end }} periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 @@ -5082,6 +5092,9 @@ spec: - /bin/calico-node - -felix-ready - -bird-ready +{{- if .IPv6Support }} + - -bird6-ready +{{- end }} periodSeconds: 10 timeoutSeconds: 10 volumeMounts: diff --git a/docs/config-example.md b/docs/config-example.md index 28ecaf556..aad1a16e2 100644 --- a/docs/config-example.md +++ b/docs/config-example.md @@ -145,8 +145,8 @@ spec: ipipMode: Always # IPIP Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, vxlanMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Always] vxlanMode: Never # VXLAN Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, ipipMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Never] vethMTU: 0 # The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. By default, MTU is auto-detected. [Default: 0] - kubePodsCIDR: 10.233.64.0/18,fc00::/48 - kubeServiceCIDR: 10.233.0.0/18,fd00::/108 + kubePodsCIDR: 10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112 + kubeServiceCIDR: 10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116 storage: openebs: basePath: /var/openebs/local # base path of the local PV provisioner