CP/DP Split: write configuration to agent

build/Dockerfile

@@ -14,7 +14,7 @@ FROM golang:1.23 AS ca-certs-provider
 FROM scratch AS common
 # CA certs are needed for telemetry report so that NGF can verify the server's certificate.
 COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-USER 102:1001
+USER 101:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}
 ENTRYPOINT [ "/usr/bin/gateway" ]

charts/nginx-gateway-fabric/README.md

@@ -268,7 +268,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
-| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -11,35 +11,21 @@ rules:
   - namespaces
   - services
   - secrets
+  - pods
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - configmaps
 {{- end }}
   verbs:
   - get
   - list
   - watch
-{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
 - apiGroups:
   - apps
   resources:
   - replicasets
   verbs:
   - get
-{{- end }}
-{{- if .Values.nginx.plus }}
-- apiGroups:
-  - apps
-  resources:
-  - replicasets
-  verbs:
   - list
-{{- end }}
 {{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
 - apiGroups:
   - ""

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -139,8 +139,9 @@ spec:
           capabilities:
             drop:
             - ALL
+          allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         {{- with .Values.nginxGateway.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -1,9 +1,10 @@
+# TODO(sberman): will need an SCC for nginx ServiceAccounts as well.
 {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
 kind: SecurityContextConstraints
 apiVersion: security.openshift.io/v1
 metadata:
   name: {{ include "nginx-gateway.scc-name" . }}
-allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
+allowPrivilegeEscalation: false
 allowHostDirVolumePlugin: false
 allowHostIPC: false
 allowHostNetwork: false
@@ -14,7 +15,7 @@ readOnlyRootFilesystem: true
 runAsUser:
   type: MustRunAsRange
   uidRangeMin: 101
-  uidRangeMax: 102
+  uidRangeMax: 101
 fsGroup:
   type: MustRunAs
   ranges:
@@ -29,16 +30,8 @@ seLinuxContext:
   type: MustRunAs
 seccompProfiles:
 - runtime/default
-volumes:
-- emptyDir
-- secret
-- configMap
-- projected
 users:
 - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }}
-allowedCapabilities:
-- NET_BIND_SERVICE
-- KILL
 requiredDropCapabilities:
 - ALL
 {{- end }}

charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml

@@ -15,5 +15,11 @@ data:
     - /var/run/nginx
     features:
     - connection
+    - configuration
+    - certificates
+    - metrics
+    {{- if .Values.nginx.plus }}
+    - api-action
+    {{- end }}
     log:
       level: debug

charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml

@@ -21,7 +21,7 @@ spec:
         command:
         - /usr/bin/gateway
         - sleep
-        - --duration=15s
+        - --duration=5s
       - name: init
         image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
         imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
@@ -49,7 +49,7 @@ spec:
             drop:
             - ALL
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         volumeMounts:
         - name: nginx-includes-bootstrap
@@ -72,13 +72,12 @@ spec:
         securityContext:
           seccompProfile:
             type: RuntimeDefault
-          allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
           capabilities:
             add:
             - NET_BIND_SERVICE
             drop:
             - ALL
-          readOnlyRootFilesystem: true
+          # readOnlyRootFilesystem: true
           runAsUser: 101
           runAsGroup: 1001
         volumeMounts:

charts/nginx-gateway-fabric/values.schema.json

@@ -259,20 +259,6 @@
           "title": "plus",
           "type": "boolean"
         },
-        "securityContext": {
-          "properties": {
-            "allowPrivilegeEscalation": {
-              "default": false,
-              "description": "Some environments may need this set to true in order for the control plane to successfully reload NGINX.",
-              "required": [],
-              "title": "allowPrivilegeEscalation",
-              "type": "boolean"
-            }
-          },
-          "required": [],
-          "title": "securityContext",
-          "type": "object"
-        },
         "usage": {
           "description": "Configuration for NGINX Plus usage reporting.",
           "properties": {

charts/nginx-gateway-fabric/values.yaml

@@ -131,10 +131,6 @@ nginx:
     # @schema
     pullPolicy: Always
 
-  securityContext:
-    # -- Some environments may need this set to true in order for the control plane to successfully reload NGINX.
-    allowPrivilegeEscalation: false
-
   # -- Is NGINX Plus image being used
   plus: false
 

cmd/gateway/initialize.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -58,7 +59,7 @@ func initialize(cfg initializeConfig) error {
 		return fmt.Errorf("failed to generate deployment context file: %w", err)
 	}
 
-	if err := file.Write(cfg.fileManager, depCtxFile); err != nil {
+	if err := file.Write(cfg.fileManager, file.Convert(depCtxFile)); err != nil {
 		return fmt.Errorf("failed to write deployment context file: %w", err)
 	}
 
@@ -84,5 +85,9 @@ func copyFile(osFileManager file.OSFileManager, src, dest string) error {
 		return fmt.Errorf("error copying file contents: %w", err)
 	}
 
+	if err := osFileManager.Chmod(destFile, os.FileMode(file.RegularFileModeInt)); err != nil {
+		return fmt.Errorf("error setting file permissions: %w", err)
+	}
+
 	return nil
 }

cmd/gateway/initialize_test.go

@@ -133,7 +133,7 @@ func TestInitialize_Plus(t *testing.T) {
 			g.Expect(fakeGenerator.GenerateDeploymentContextArgsForCall(0)).To(Equal(test.depCtx))
 			g.Expect(fakeCollector.CollectCallCount()).To(Equal(1))
 			g.Expect(fakeFileMgr.WriteCallCount()).To(Equal(1))
-			g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(1))
+			g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(3))
 		})
 	}
 }
@@ -161,6 +161,7 @@ func TestCopyFileErrors(t *testing.T) {
 	openErr := errors.New("open error")
 	createErr := errors.New("create error")
 	copyErr := errors.New("copy error")
+	chmodErr := errors.New("chmod error")
 
 	tests := []struct {
 		fileMgr *filefakes.FakeOSFileManager
@@ -194,6 +195,15 @@ func TestCopyFileErrors(t *testing.T) {
 			},
 			expErr: copyErr,
 		},
+		{
+			name: "can't set permissions",
+			fileMgr: &filefakes.FakeOSFileManager{
+				ChmodStub: func(_ *os.File, _ os.FileMode) error {
+					return chmodErr
+				},
+			},
+			expErr: chmodErr,
+		},
 	}
 
 	for _, test := range tests {

config/tests/static-deployment.yaml

@@ -69,8 +69,9 @@ spec:
           capabilities:
             drop:
             - ALL
+          allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
       terminationGracePeriodSeconds: 30
       serviceAccountName: nginx-gateway

deploy/aws-nlb/deploy.yaml

@@ -28,22 +28,18 @@ rules:
   - namespaces
   - services
   - secrets
+  - pods
   verbs:
   - get
   - list
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
 - apiGroups:
   - apps
   resources:
   - replicasets
   verbs:
   - get
+  - list
 - apiGroups:
   - ""
   resources:
@@ -157,6 +153,9 @@ data:
     - /var/run/nginx
     features:
     - connection
+    - configuration
+    - certificates
+    - metrics
     log:
       level: debug
 kind: ConfigMap
@@ -293,12 +292,13 @@ spec:
           initialDelaySeconds: 3
           periodSeconds: 1
         securityContext:
+          allowPrivilegeEscalation: false
           capabilities:
             drop:
             - ALL
           readOnlyRootFilesystem: true
           runAsGroup: 1001
-          runAsUser: 102
+          runAsUser: 101
           seccompProfile:
             type: RuntimeDefault
       securityContext:
@@ -333,13 +333,11 @@ spec:
         - containerPort: 443
           name: https
         securityContext:
-          allowPrivilegeEscalation: false
           capabilities:
             add:
             - NET_BIND_SERVICE
             drop:
             - ALL
-          readOnlyRootFilesystem: true
           runAsGroup: 1001
           runAsUser: 101
           seccompProfile:
@@ -365,7 +363,7 @@ spec:
       - command:
         - /usr/bin/gateway
         - sleep
-        - --duration=15s
+        - --duration=5s
         image: ghcr.io/nginxinc/nginx-gateway-fabric:edge
         imagePullPolicy: Always
         name: sleep
@@ -390,7 +388,7 @@ spec:
             - ALL
           readOnlyRootFilesystem: true
           runAsGroup: 1001
-          runAsUser: 102
+          runAsUser: 101
           seccompProfile:
             type: RuntimeDefault
         volumeMounts: