Refine IPv6 Supported

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -63,6 +63,7 @@ const (
 	DefaultMaxPods                 = 110
 	DefaultPodPidsLimit            = 10000
 	DefaultNodeCidrMaskSize        = 24
+	DefaultNodeCidrMaskSizeIPv6    = 120
 	DefaultIPIPMode                = "Always"
 	DefaultVXLANMode               = "Never"
 	DefaultVethMTU                 = 0
@@ -139,6 +140,9 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*Kub
 	if cfg.Kubernetes.NodeCidrMaskSize == 0 {
 		clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize
 	}
+	if cfg.Kubernetes.NodeCidrMaskSizeIPv6 == 0 {
+		clusterCfg.Kubernetes.NodeCidrMaskSizeIPv6 = DefaultNodeCidrMaskSizeIPv6
+	}
 	if cfg.Kubernetes.ProxyMode == "" {
 		clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode
 	}

cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go

@@ -32,6 +32,7 @@ type Kubernetes struct {
 	MaxPods                int      `yaml:"maxPods" json:"maxPods,omitempty"`
 	PodPidsLimit           int      `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"`
 	NodeCidrMaskSize       int      `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"`
+	NodeCidrMaskSizeIPv6   int      `yaml:"nodeCidrMaskSizeIPv6" json:"nodeCidrMaskSizeIPv6,omitempty"`
 	ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"`
 	ProxyMode              string   `yaml:"proxyMode" json:"proxyMode,omitempty"`
 	AutoRenewCerts         *bool    `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"`

cmd/kk/apis/kubekey/v1alpha2/network_types.go

@@ -28,14 +28,25 @@ type NetworkConfig struct {
 }
 
 type CalicoCfg struct {
-	IPIPMode        string            `yaml:"ipipMode" json:"ipipMode,omitempty"`
-	VXLANMode       string            `yaml:"vxlanMode" json:"vxlanMode,omitempty"`
-	VethMTU         int               `yaml:"vethMTU" json:"vethMTU,omitempty"`
-	Ipv4NatOutgoing *bool             `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"`
-	DefaultIPPOOL   *bool             `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"`
-	EnableTypha     *bool             `yaml:"enableTypha" json:"enableTypha,omitempty"`
-	Replicas        int               `yaml:"replicas" json:"replicas,omitempty"`
-	NodeSelector    map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
+	IPIPMode        string     `yaml:"ipipMode" json:"ipipMode,omitempty"`
+	VXLANMode       string     `yaml:"vxlanMode" json:"vxlanMode,omitempty"`
+	VethMTU         int        `yaml:"vethMTU" json:"vethMTU,omitempty"`
+	Ipv4NatOutgoing *bool      `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"`
+	Ipv6NatOutgoing *bool      `yaml:"ipv6NatOutgoing" json:"ipv6NatOutgoing,omitempty"`
+	DefaultIPPOOL   *bool      `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"`
+	Typha           Typha      `yaml:"typha" json:"typha,omitempty"`
+	Controller      Controller `yaml:"controller" json:"controller,omitempty"`
+}
+
+type Typha struct {
+	Replicas     int               `yaml:"replicas" json:"replicas,omitempty"`
+	NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
+	Enabled      *bool             `yaml:"enabled" json:"enabled,omitempty"`
+}
+
+type Controller struct {
+	Replicas     int               `yaml:"replicas" json:"replicas,omitempty"`
+	NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
 }
 
 type FlannelCfg struct {
@@ -178,6 +189,14 @@ func (c *CalicoCfg) EnableIPV4POOL_NAT_OUTGOING() bool {
 	return *c.Ipv4NatOutgoing
 }
 
+// EnableIPV6POOL_NAT_OUTGOING is used to determine whether to enable CALICO_IPV6POOL_NAT_OUTGOING.
+func (c *CalicoCfg) EnableIPV6POOL_NAT_OUTGOING() bool {
+	if c.Ipv6NatOutgoing == nil {
+		return false
+	}
+	return *c.Ipv6NatOutgoing
+}
+
 // EnableDefaultIPPOOL is used to determine whether to create default ippool
 func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
 	if c.DefaultIPPOOL == nil {
@@ -186,12 +205,12 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
 	return *c.DefaultIPPOOL
 }
 
-// Typha is used to determine whether to enable calico Typha
-func (c *CalicoCfg) Typha() bool {
-	if c.EnableTypha == nil {
+// EnableTypha is used to determine whether to enable calico Typha
+func (c *CalicoCfg) EnableTypha() bool {
+	if c.Typha.Enabled == nil {
 		return false
 	}
-	return *c.EnableTypha
+	return *c.Typha.Enabled
 }
 
 // EnableInit is used to determine whether to create default network

cmd/kk/pkg/bootstrap/os/module.go

@@ -64,7 +64,8 @@ func (c *ConfigureOSModule) Init() {
 			Template: templates.InitOsScriptTmpl,
 			Dst:      filepath.Join(common.KubeScriptDir, "initOS.sh"),
 			Data: util.Data{
-				"Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf),
+				"Hosts":       templates.GenerateHosts(c.Runtime, c.KubeConf),
+				"IPv6Support": templates.EnabledIPv6(c.KubeConf),
 			},
 		},
 		Parallel: true,

cmd/kk/pkg/bootstrap/os/templates/init_script.go

@@ -99,11 +99,16 @@ echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf
 echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf
 echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf
 
+{{- if .IPv6Support }}
 #add for ipv6
 echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf
 echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf
 echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf
 echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf
+echo 'net.ipv6.conf.default.accept_dad=0' >> /etc/sysctl.conf
+echo 'net.ipv6.route.max_size=65536' >> /etc/sysctl.conf
+echo 'net.ipv6.neigh.default.retrans_time_ms=1000' >> /etc/sysctl.conf
+{{- end}}
 
 #See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf
@@ -145,6 +150,18 @@ sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.d
 sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf
 
+{{- if .IPv6Support }}
+#add for ipv6
+sed -r -i "s@#{0,}?net.ipv6.conf.all.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.all.disable_ipv6 = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.conf.default.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.default.disable_ipv6 = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.conf.lo.disable_ipv6 ?= ?([0-9]{1,})@net.ipv6.conf.lo.disable_ipv6 = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.conf.all.forwarding ?= ?([0-9]{1,})@net.ipv6.conf.all.forwarding = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.conf.default.accept_dad ?= ?([0-9]{1,})@net.ipv6.conf.default.accept_dad = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.route.max_size ?= ?([0-9]{1,})@net.ipv6.route.max_size = 65536@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv6.neigh.default.retrans_time_ms ?= ?([0-9]{1,})@net.ipv6.neigh.default.retrans_time_ms = 1000@g" /etc/sysctl.conf
+{{- end}}
+
+
 tmpfile="$$.tmp"
 awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf
 mv $tmpfile /etc/sysctl.conf
@@ -287,3 +304,10 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
 	hostsList = append(hostsList, lbHost)
 	return hostsList
 }
+
+func EnabledIPv6(kubeConf *common.KubeConf) bool {
+	if len(strings.Split(kubeConf.Cluster.Network.KubePodsCIDR, ",")) == 2 {
+		return true
+	}
+	return false
+}

cmd/kk/pkg/kubernetes/tasks.go

@@ -307,6 +307,7 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 				"BootstrapToken":         bootstrapToken,
 				"CertificateKey":         certificateKey,
 				"IPv6Support":            host.GetInternalIPv6Address() != "",
+				"NodeCidrMaskSizeIPv6":   g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6,
 			},
 		}
 

cmd/kk/pkg/kubernetes/templates/kubeadm_config.go

@@ -98,7 +98,7 @@ controllerManager:
   extraArgs:
 {{- if .IPv6Support }}
     node-cidr-mask-size-ipv4: "{{ .NodeCidrMaskSize }}"
-    node-cidr-mask-size-ipv6: "64"
+    node-cidr-mask-size-ipv6: "{{ .NodeCidrMaskSizeIPv6 }}"
 {{- else }}
     node-cidr-mask-size: "{{ .NodeCidrMaskSize }}"
 {{- end }}

cmd/kk/pkg/plugins/network/tasks.go

@@ -460,17 +460,21 @@ func (g *GenerateCalicoManifests) Execute(runtime connector.Runtime) error {
 			"CalicoFlexvolImage":      images.GetImage(runtime, g.KubeConf, "calico-flexvol").ImageName(),
 			"CalicoControllersImage":  images.GetImage(runtime, g.KubeConf, "calico-kube-controllers").ImageName(),
 			"CalicoTyphaImage":        images.GetImage(runtime, g.KubeConf, "calico-typha").ImageName(),
-			"TyphaEnabled":            len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.Typha(),
+			"TyphaEnabled":            len(runtime.GetHostsByRole(common.K8s)) > 50 || g.KubeConf.Cluster.Network.Calico.EnableTypha(),
 			"VethMTU":                 g.KubeConf.Cluster.Network.Calico.VethMTU,
 			"NodeCidrMaskSize":        g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
 			"IPIPMode":                g.KubeConf.Cluster.Network.Calico.IPIPMode,
 			"VXLANMode":               g.KubeConf.Cluster.Network.Calico.VXLANMode,
 			"ConatinerManagerIsIsula": g.KubeConf.Cluster.Kubernetes.ContainerManager == "isula",
 			"IPV4POOLNATOUTGOING":     g.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(),
+			"IPV6POOLNATOUTGOING":     g.KubeConf.Cluster.Network.Calico.EnableIPV6POOL_NAT_OUTGOING(),
 			"DefaultIPPOOL":           g.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(),
 			"IPv6Support":             IPv6Support,
-			"Replicas":                g.KubeConf.Cluster.Network.Calico.Replicas,
-			"NodeSelector":            g.KubeConf.Cluster.Network.Calico.NodeSelector,
+			"NodeCidrMaskSizeIPv6":    g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSizeIPv6,
+			"TyphaReplicas":           g.KubeConf.Cluster.Network.Calico.Typha.Replicas,
+			"TyphaNodeSelector":       g.KubeConf.Cluster.Network.Calico.Typha.NodeSelector,
+			"ControllerReplicas":      g.KubeConf.Cluster.Network.Calico.Controller.Replicas,
+			"ControllerNodeSelector":  g.KubeConf.Cluster.Network.Calico.Controller.NodeSelector,
 		},
 	}
 	templateAction.Init(nil, nil)

cmd/kk/pkg/plugins/network/templates/calico.tmpl

@@ -88,9 +88,17 @@ data:
           "datastore_type": "kubernetes",
           "nodename": "__KUBERNETES_NODE_NAME__",
           "mtu": __CNI_MTU__,
+{{- if .IPv6Support }}
+          "ipam": {
+              "type": "calico-ipam",
+              "assign_ipv4": "true",
+              "assign_ipv6": "true"
+          },
+{{- else }}
           "ipam": {
               "type": "calico-ipam"
           },
+{{- end }}
           "policy": {
               "type": "k8s"
           },
@@ -4981,17 +4989,16 @@ spec:
               value: "false"
 {{- end }}
 {{- if .IPv6Support }}
+            # Enable IPIP
+            - name: CALICO_IPV6POOL_IPIP
+              value: "{{ .IPIPMode }}"
             # Enable or Disable VXLAN on the default IPv6 IP pool.
             - name: CALICO_IPV6POOL_VXLAN
-              value: "Always"
+              value: "{{ .VXLANMode }}"
+{{- if .IPV6POOLNATOUTGOING }}
             - name: CALICO_IPV6POOL_NAT_OUTGOING
               value: "true"
-{{- else }}
-            # Enable or Disable VXLAN on the default IPv6 IP pool.
-            - name: CALICO_IPV6POOL_VXLAN
-              value: "Never"
-            - name: CALICO_IPV6POOL_NAT_OUTGOING
-              value: "false"
+{{- end }}
 {{- end }}
             # Set MTU for tunnel device used if ipip is enabled
             - name: FELIX_IPINIPMTU
@@ -5023,7 +5030,7 @@ spec:
             - name: CALICO_IPV6POOL_CIDR
               value: "{{ .KubePodsV6CIDR }}"
             - name: CALICO_IPV6POOL_BLOCK_SIZE
-              value: "120"
+              value: "{{ .NodeCidrMaskSizeIPv6 }}"
 {{- end }}
 {{- else }}
             - name: NO_DEFAULT_POOLS
@@ -5072,6 +5079,9 @@ spec:
               - /bin/calico-node
               - -felix-live
               - -bird-live
+{{- if .IPv6Support }}
+              - -bird6-live
+{{- end }}
             periodSeconds: 10
             initialDelaySeconds: 10
             failureThreshold: 6
@@ -5082,6 +5092,9 @@ spec:
               - /bin/calico-node
               - -felix-ready
               - -bird-ready
+{{- if .IPv6Support }}
+              - -bird6-ready
+{{- end }}
             periodSeconds: 10
             timeoutSeconds: 10
           volumeMounts:
@@ -5171,7 +5184,7 @@ metadata:
     k8s-app: calico-kube-controllers
 spec:
   # The controllers can only have a single active instance.
-  replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }}
+  replicas: {{ if .ControllerReplicas }}{{ .ControllerReplicas }}{{ else }}1{{ end }}
   selector:
     matchLabels:
       k8s-app: calico-kube-controllers
@@ -5186,8 +5199,8 @@ spec:
     spec:
       nodeSelector:
         kubernetes.io/os: linux
-{{ if .NodeSelector }}
-        {{- toYaml .NodeSelector | indent 8 }}
+{{ if .ControllerNodeSelector }}
+        {{- toYaml .ControllerNodeSelector | indent 8 }}
 {{- end }}
       tolerations:
         # Mark the pod as a critical add-on for rescheduling.
@@ -5197,7 +5210,7 @@ spec:
           effect: NoSchedule
         - key: node-role.kubernetes.io/control-plane
           effect: NoSchedule
-{{ if .NodeSelector }}
+{{ if .ControllerNodeSelector }}
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -5258,7 +5271,7 @@ spec:
   # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential
   # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In
   # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
-  replicas: {{ if .Replicas }}{{ .Replicas }}{{ else }}1{{ end }}
+  replicas: {{ if .TyphaReplicas }}{{ .TyphaReplicas }}{{ else }}1{{ end }}
   revisionHistoryLimit: 2
   selector:
     matchLabels:
@@ -5285,14 +5298,14 @@ spec:
     spec:
       nodeSelector:
         kubernetes.io/os: linux
-{{ if .NodeSelector }}
-        {{- toYaml .NodeSelector | indent 8 }}
+{{ if .TyphaNodeSelector }}
+        {{- toYaml .TyphaNodeSelector | indent 8 }}
 {{- end }}
       hostNetwork: true
       # Typha supports graceful shut down, disconnecting clients slowly during the grace period.
       # The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value.
       terminationGracePeriodSeconds: 300
-{{ if .NodeSelector }}
+{{ if .TyphaNodeSelector }}
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:

docs/config-example.md

@@ -148,8 +148,8 @@ spec:
       ipipMode: Always  # IPIP Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, vxlanMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Always]
       vxlanMode: Never  # VXLAN Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, ipipMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Never]
       vethMTU: 0  # The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. By default, MTU is auto-detected. [Default: 0]
-    kubePodsCIDR: 10.233.64.0/18,fc00::/48
-    kubeServiceCIDR: 10.233.0.0/18,fd00::/108
+    kubePodsCIDR: 10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112
+    kubeServiceCIDR: 10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116
   storage:
     openebs:
       basePath: /var/openebs/local # base path of the local PV provisioner