Skip to content

Commit

Permalink
Support for a common subnet for control plane and worker nodes for az…
Browse files Browse the repository at this point in the history 
…ure infra clusters.
  • Loading branch information
@LochanRn @mboersma
LochanRn authored and mboersma committed Feb 5, 2024
1 parent d28697d commit fa775ea
Show file tree
Hide file tree
Showing 15 changed files with 603 additions and 121 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ YQ_VER := v4.14.2
YQ_BIN := yq
YQ := $(TOOLS_BIN_DIR)/$(YQ_BIN)-$(YQ_VER)

KIND_VER := v0.20.0
KIND_VER := v0.21.0
KIND_BIN := kind
KIND := $(TOOLS_BIN_DIR)/$(KIND_BIN)-$(KIND_VER)

Expand Down
144 changes: 101 additions & 43 deletions api/v1beta1/azurecluster_default.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const (
DefaultControlPlaneSubnetCIDR = "10.0.0.0/16"
// DefaultNodeSubnetCIDR is the default Node Subnet CIDR.
DefaultNodeSubnetCIDR = "10.1.0.0/16"
// DefaultClusterSubnetCIDR is the default Cluster Subnet CIDR.
DefaultClusterSubnetCIDR = "10.0.0.0/16"
// DefaultNodeSubnetCIDRPattern is the pattern that will be used to generate the default subnets CIDRs.
DefaultNodeSubnetCIDRPattern = "10.%d.0.0/16"
// DefaultAzureBastionSubnetCIDR is the default Subnet CIDR for AzureBastion.
Expand Down Expand Up @@ -84,24 +86,24 @@ func (c *AzureCluster) setVnetDefaults() {
}

func (c *AzureCluster) setSubnetDefaults() {
cpSubnet, err := c.Spec.NetworkSpec.GetControlPlaneSubnet()
if err != nil {
cpSubnet = SubnetSpec{SubnetClassSpec: SubnetClassSpec{Role: SubnetControlPlane}}
c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, cpSubnet)
}

if cpSubnet.Name == "" {
cpSubnet.Name = generateControlPlaneSubnetName(c.ObjectMeta.Name)
clusterSubnet, err := c.Spec.NetworkSpec.GetSubnet(SubnetCluster)
clusterSubnetExists := err == nil
if clusterSubnetExists {
clusterSubnet.setClusterSubnetDefaults(c.ObjectMeta.Name)
c.Spec.NetworkSpec.UpdateSubnet(clusterSubnet, SubnetCluster)
}

cpSubnet.SubnetClassSpec.setDefaults(DefaultControlPlaneSubnetCIDR)

if cpSubnet.SecurityGroup.Name == "" {
cpSubnet.SecurityGroup.Name = generateControlPlaneSecurityGroupName(c.ObjectMeta.Name)
/* if there is a cp subnet set defaults
if no cp subnet and cluster subnet create a default cp subnet */
cpSubnet, errcp := c.Spec.NetworkSpec.GetSubnet(SubnetControlPlane)
if errcp == nil {
cpSubnet.setControlPlaneSubnetDefaults(c.ObjectMeta.Name)
c.Spec.NetworkSpec.UpdateSubnet(cpSubnet, SubnetControlPlane)
} else if !clusterSubnetExists {
cpSubnet = SubnetSpec{SubnetClassSpec: SubnetClassSpec{Role: SubnetControlPlane}}
cpSubnet.setControlPlaneSubnetDefaults(c.ObjectMeta.Name)
c.Spec.NetworkSpec.Subnets = append(c.Spec.NetworkSpec.Subnets, cpSubnet)
}
cpSubnet.SecurityGroup.SecurityGroupClass.setDefaults()

c.Spec.NetworkSpec.UpdateControlPlaneSubnet(cpSubnet)

var nodeSubnetFound bool
var nodeSubnetCounter int
Expand All @@ -111,36 +113,11 @@ func (c *AzureCluster) setSubnetDefaults() {
}
nodeSubnetCounter++
nodeSubnetFound = true
if subnet.Name == "" {
subnet.Name = withIndex(generateNodeSubnetName(c.ObjectMeta.Name), nodeSubnetCounter)
}
subnet.SubnetClassSpec.setDefaults(fmt.Sprintf(DefaultNodeSubnetCIDRPattern, nodeSubnetCounter))

if subnet.SecurityGroup.Name == "" {
subnet.SecurityGroup.Name = generateNodeSecurityGroupName(c.ObjectMeta.Name)
}
subnet.SecurityGroup.SecurityGroupClass.setDefaults()

if subnet.RouteTable.Name == "" {
subnet.RouteTable.Name = generateNodeRouteTableName(c.ObjectMeta.Name)
}

// NAT gateway only supports the use of IPv4 public IP addresses for outbound connectivity.
// So default use the NAT gateway for outbound traffic in IPv4 cluster instead of loadbalancer.
// We assume that if the ID is set, the subnet already exists so we shouldn't add a NAT gateway.
if !subnet.IsIPv6Enabled() && subnet.ID == "" {
if subnet.NatGateway.Name == "" {
subnet.NatGateway.Name = withIndex(generateNatGatewayName(c.ObjectMeta.Name), nodeSubnetCounter)
}
if subnet.NatGateway.NatGatewayIP.Name == "" {
subnet.NatGateway.NatGatewayIP.Name = generateNatGatewayIPName(subnet.NatGateway.Name)
}
}

subnet.setNodeSubnetDefaults(c.ObjectMeta.Name, nodeSubnetCounter)
c.Spec.NetworkSpec.Subnets[i] = subnet
}

if !nodeSubnetFound {
if !nodeSubnetFound && !clusterSubnetExists {
nodeSubnet := SubnetSpec{
SubnetClassSpec: SubnetClassSpec{
Role: SubnetNode,
Expand All @@ -163,6 +140,67 @@ func (c *AzureCluster) setSubnetDefaults() {
}
}

func (s *SubnetSpec) setNodeSubnetDefaults(clusterName string, index int) {
if s.Name == "" {
s.Name = withIndex(generateNodeSubnetName(clusterName), index)
}
s.SubnetClassSpec.setDefaults(fmt.Sprintf(DefaultNodeSubnetCIDRPattern, index))

if s.SecurityGroup.Name == "" {
s.SecurityGroup.Name = generateNodeSecurityGroupName(clusterName)
}
s.SecurityGroup.SecurityGroupClass.setDefaults()

if s.RouteTable.Name == "" {
s.RouteTable.Name = generateNodeRouteTableName(clusterName)
}

// NAT gateway only supports the use of IPv4 public IP addresses for outbound connectivity.
// So default use the NAT gateway for outbound traffic in IPv4 cluster instead of loadbalancer.
// We assume that if the ID is set, the subnet already exists so we shouldn't add a NAT gateway.
if !s.IsIPv6Enabled() && s.ID == "" {
if s.NatGateway.Name == "" {
s.NatGateway.Name = withIndex(generateNatGatewayName(clusterName), index)
}
if s.NatGateway.NatGatewayIP.Name == "" {
s.NatGateway.NatGatewayIP.Name = generateNatGatewayIPName(s.NatGateway.Name)
}
}
}

func (s *SubnetSpec) setControlPlaneSubnetDefaults(clusterName string) {
if s.Name == "" {
s.Name = generateControlPlaneSubnetName(clusterName)
}

s.SubnetClassSpec.setDefaults(DefaultControlPlaneSubnetCIDR)

if s.SecurityGroup.Name == "" {
s.SecurityGroup.Name = generateControlPlaneSecurityGroupName(clusterName)
}
s.SecurityGroup.SecurityGroupClass.setDefaults()
}

func (s *SubnetSpec) setClusterSubnetDefaults(clusterName string) {
if s.Name == "" {
s.Name = generateClusterSubnetSubnetName(clusterName)
}
if s.SecurityGroup.Name == "" {
s.SecurityGroup.Name = generateClusterSecurityGroupName(clusterName)
}
if s.RouteTable.Name == "" {
s.RouteTable.Name = generateClustereRouteTableName(clusterName)
}
if s.NatGateway.Name == "" {
s.NatGateway.Name = generateClusterNatGatewayName(clusterName)
}
if !s.IsIPv6Enabled() && s.ID == "" && s.NatGateway.NatGatewayIP.Name == "" {
s.NatGateway.NatGatewayIP.Name = generateNatGatewayIPName(s.NatGateway.Name)
}
s.setDefaults(DefaultClusterSubnetCIDR)
s.SecurityGroup.SecurityGroupClass.setDefaults()
}

func (c *AzureCluster) setVnetPeeringDefaults() {
for i, peering := range c.Spec.NetworkSpec.Vnet.Peerings {
if peering.ResourceGroup == "" {
Expand Down Expand Up @@ -217,7 +255,7 @@ func (c *AzureCluster) SetNodeOutboundLBDefaults() {

var needsOutboundLB bool
for _, subnet := range c.Spec.NetworkSpec.Subnets {
if subnet.Role == SubnetNode && subnet.IsIPv6Enabled() {
if (subnet.Role == SubnetNode || subnet.Role == SubnetCluster) && subnet.IsIPv6Enabled() {
needsOutboundLB = true
break
}
Expand Down Expand Up @@ -400,6 +438,11 @@ func generateVnetName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "vnet")
}

// generateClusterSubnetSubnetName generates a subnet name, based on the cluster name.
func generateClusterSubnetSubnetName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "subnet")
}

// generateControlPlaneSubnetName generates a node subnet name, based on the cluster name.
func generateControlPlaneSubnetName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "controlplane-subnet")
Expand All @@ -420,6 +463,11 @@ func generateAzureBastionPublicIPName(clusterName string) string {
return fmt.Sprintf("%s-azure-bastion-pip", clusterName)
}

// generateClusterSecurityGroupName generates a security group name, based on the cluster name.
func generateClusterSecurityGroupName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "nsg")
}

// generateControlPlaneSecurityGroupName generates a control plane security group name, based on the cluster name.
func generateControlPlaneSecurityGroupName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "controlplane-nsg")
Expand All @@ -430,6 +478,11 @@ func generateNodeSecurityGroupName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "node-nsg")
}

// generateClustereRouteTableName generates a route table name, based on the cluster name.
func generateClustereRouteTableName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "routetable")
}

// generateNodeRouteTableName generates a node route table name, based on the cluster name.
func generateNodeRouteTableName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "node-routetable")
Expand Down Expand Up @@ -470,6 +523,11 @@ func generateControlPlaneOutboundIPName(clusterName string) string {
return fmt.Sprintf("pip-%s-controlplane-outbound", clusterName)
}

// generateClusterNatGatewayName generates a NAT gateway name.
func generateClusterNatGatewayName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "natgw")
}

// generateNatGatewayName generates a NAT gateway name.
func generateNatGatewayName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, "node-natgw")
Expand Down
102 changes: 102 additions & 0 deletions api/v1beta1/azurecluster_default_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,108 @@ func TestSubnetDefaults(t *testing.T) {
},
},
},
{
name: "cluster subnet with custom attributes",
cluster: &AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-test",
},
Spec: AzureClusterSpec{
NetworkSpec: NetworkSpec{
Subnets: Subnets{
{
SubnetClassSpec: SubnetClassSpec{
Role: SubnetCluster,
CIDRBlocks: []string{"10.0.0.16/24"},
Name: "my-subnet",
},
NatGateway: NatGateway{
NatGatewayClassSpec: NatGatewayClassSpec{
Name: "foo-natgw",
},
},
},
},
},
},
},
output: &AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-test",
},
Spec: AzureClusterSpec{
NetworkSpec: NetworkSpec{
Subnets: Subnets{
{
SubnetClassSpec: SubnetClassSpec{
Role: SubnetCluster,
CIDRBlocks: []string{"10.0.0.16/24"},
Name: "my-subnet",
},
SecurityGroup: SecurityGroup{Name: "cluster-test-nsg"},
RouteTable: RouteTable{Name: "cluster-test-routetable"},
NatGateway: NatGateway{
NatGatewayClassSpec: NatGatewayClassSpec{
Name: "foo-natgw",
},
NatGatewayIP: PublicIPSpec{
Name: "pip-foo-natgw",
},
},
},
},
},
},
},
},
{
name: "cluster subnet with subnets specified",
cluster: &AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-test",
},
Spec: AzureClusterSpec{
NetworkSpec: NetworkSpec{
Subnets: Subnets{
{
SubnetClassSpec: SubnetClassSpec{
Role: SubnetCluster,
Name: "cluster-test-subnet",
},
},
},
},
},
},
output: &AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-test",
},
Spec: AzureClusterSpec{
NetworkSpec: NetworkSpec{
Subnets: Subnets{
{
SubnetClassSpec: SubnetClassSpec{
Role: SubnetCluster,
CIDRBlocks: []string{DefaultClusterSubnetCIDR},
Name: "cluster-test-subnet",
},
SecurityGroup: SecurityGroup{Name: "cluster-test-nsg"},
RouteTable: RouteTable{Name: "cluster-test-routetable"},
NatGateway: NatGateway{
NatGatewayClassSpec: NatGatewayClassSpec{
Name: "cluster-test-natgw",
},
NatGatewayIP: PublicIPSpec{
Name: "pip-cluster-test-natgw",
},
},
},
},
},
},
},
},
{
name: "subnets route tables specified",
cluster: &AzureCluster{
Expand Down
25 changes: 20 additions & 5 deletions api/v1beta1/azurecluster_validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func validateNetworkSpec(networkSpec NetworkSpec, old NetworkSpec, fldPath *fiel

var needOutboundLB bool
for _, subnet := range networkSpec.Subnets {
if subnet.Role == SubnetNode && subnet.IsIPv6Enabled() {
if (subnet.Role == SubnetNode || subnet.Role == SubnetCluster) && subnet.IsIPv6Enabled() {
needOutboundLB = true
break
}
Expand Down Expand Up @@ -213,14 +213,16 @@ func validateResourceGroup(resourceGroup string, fldPath *field.Path) *field.Err
}

// validateSubnets validates a list of Subnets.
// When configuring a cluster, it is essential to include either a control-plane subnet and a node subnet, or a user can configure a cluster subnet which will be used as a control-plane subnet and a node subnet.
func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
subnetNames := make(map[string]bool, len(subnets))
requiredSubnetRoles := map[string]bool{
"control-plane": false,
"node": false,
}

clusterSubnet := false
numberofClusterSubnets := 0
for i, subnet := range subnets {
if err := validateSubnetName(subnet.Name, fldPath.Index(i).Child("name")); err != nil {
allErrs = append(allErrs, err)
Expand All @@ -229,11 +231,17 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field.
allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name))
}
subnetNames[subnet.Name] = true
for role := range requiredSubnetRoles {
if role == string(subnet.Role) {
requiredSubnetRoles[role] = true
if subnet.Role == SubnetCluster {
clusterSubnet = true
numberofClusterSubnets++
} else {
for role := range requiredSubnetRoles {
if role == string(subnet.Role) {
requiredSubnetRoles[role] = true
}
}
}

for _, rule := range subnet.SecurityGroup.SecurityRules {
if err := validateSecurityRule(
rule,
Expand All @@ -252,6 +260,13 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field.
allErrs = append(allErrs, validatePrivateEndpoints(subnet.PrivateEndpoints, subnet.CIDRBlocks, fldPath.Index(i).Child("privateEndpoints"))...)
}
}

// The clusterSubnet is applicable to both the control-plane and node pools.
// Validation of requiredSubnetRoles is skipped since clusterSubnet is set to true.
if clusterSubnet {
return allErrs
}

for k, v := range requiredSubnetRoles {
if !v {
allErrs = append(allErrs, field.Required(fldPath,
Expand Down
Loading

0 comments on commit fa775ea

Please sign in to comment.