diff --git a/build/Dockerfile b/build/Dockerfile index c1b7a6a29..103e49189 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -14,7 +14,7 @@ FROM golang:1.23 AS ca-certs-provider FROM scratch AS common # CA certs are needed for telemetry report so that NGF can verify the server's certificate. COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -USER 102:1001 +USER 101:1001 ARG BUILD_AGENT ENV BUILD_AGENT=${BUILD_AGENT} ENTRYPOINT [ "/usr/bin/gateway" ] diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 9fdd20bca..4ff5cade5 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -268,7 +268,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.image.tag` | | string | `"edge"` | | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` | | `nginx.plus` | Is NGINX Plus image being used | bool | `false` | -| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` | | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 9ee1be425..830fe1b39 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -11,6 +11,7 @@ rules: - namespaces - services - secrets + - pods {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} - configmaps {{- end }} @@ -18,28 +19,13 @@ rules: - get - list - watch -{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }} -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -{{- end }} -{{- if .Values.nginx.plus }} -- apiGroups: - - apps - resources: - - replicasets - verbs: - list -{{- end }} {{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }} - apiGroups: - "" diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 35468e682..7a254a3bf 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -139,8 +139,9 @@ spec: capabilities: drop: - ALL + allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 {{- with .Values.nginxGateway.extraVolumeMounts -}} {{ toYaml . | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/templates/scc.yaml b/charts/nginx-gateway-fabric/templates/scc.yaml index e58389a8e..6ab7dc92c 100644 --- a/charts/nginx-gateway-fabric/templates/scc.yaml +++ b/charts/nginx-gateway-fabric/templates/scc.yaml @@ -1,9 +1,10 @@ +# TODO(sberman): will need an SCC for nginx ServiceAccounts as well. {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }} kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: name: {{ include "nginx-gateway.scc-name" . }} -allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }} +allowPrivilegeEscalation: false allowHostDirVolumePlugin: false allowHostIPC: false allowHostNetwork: false @@ -14,7 +15,7 @@ readOnlyRootFilesystem: true runAsUser: type: MustRunAsRange uidRangeMin: 101 - uidRangeMax: 102 + uidRangeMax: 101 fsGroup: type: MustRunAs ranges: @@ -29,16 +30,8 @@ seLinuxContext: type: MustRunAs seccompProfiles: - runtime/default -volumes: -- emptyDir -- secret -- configMap -- projected users: - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }} -allowedCapabilities: -- NET_BIND_SERVICE -- KILL requiredDropCapabilities: - ALL {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml index 80aba1c86..e2126ee9e 100644 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml +++ b/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml @@ -15,5 +15,11 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + {{- if .Values.nginx.plus }} + - api-action + {{- end }} log: level: debug diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml index 55c9ee597..df4a0a962 100644 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml @@ -21,7 +21,7 @@ spec: command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s - name: init image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} @@ -49,7 +49,7 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 volumeMounts: - name: nginx-includes-bootstrap @@ -72,13 +72,12 @@ spec: securityContext: seccompProfile: type: RuntimeDefault - allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }} capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true + # readOnlyRootFilesystem: true runAsUser: 101 runAsGroup: 1001 volumeMounts: diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 651fea311..9a3a7d9a8 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -259,20 +259,6 @@ "title": "plus", "type": "boolean" }, - "securityContext": { - "properties": { - "allowPrivilegeEscalation": { - "default": false, - "description": "Some environments may need this set to true in order for the control plane to successfully reload NGINX.", - "required": [], - "title": "allowPrivilegeEscalation", - "type": "boolean" - } - }, - "required": [], - "title": "securityContext", - "type": "object" - }, "usage": { "description": "Configuration for NGINX Plus usage reporting.", "properties": { diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 4168c3d66..e71cbb572 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -131,10 +131,6 @@ nginx: # @schema pullPolicy: Always - securityContext: - # -- Some environments may need this set to true in order for the control plane to successfully reload NGINX. - allowPrivilegeEscalation: false - # -- Is NGINX Plus image being used plus: false diff --git a/cmd/gateway/initialize.go b/cmd/gateway/initialize.go index 4ce6fc049..d8080168f 100644 --- a/cmd/gateway/initialize.go +++ b/cmd/gateway/initialize.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "path/filepath" "time" @@ -58,7 +59,7 @@ func initialize(cfg initializeConfig) error { return fmt.Errorf("failed to generate deployment context file: %w", err) } - if err := file.Write(cfg.fileManager, depCtxFile); err != nil { + if err := file.Write(cfg.fileManager, file.Convert(depCtxFile)); err != nil { return fmt.Errorf("failed to write deployment context file: %w", err) } @@ -84,5 +85,9 @@ func copyFile(osFileManager file.OSFileManager, src, dest string) error { return fmt.Errorf("error copying file contents: %w", err) } + if err := osFileManager.Chmod(destFile, os.FileMode(file.RegularFileModeInt)); err != nil { + return fmt.Errorf("error setting file permissions: %w", err) + } + return nil } diff --git a/cmd/gateway/initialize_test.go b/cmd/gateway/initialize_test.go index 4276ff9a2..16ef8dc3b 100644 --- a/cmd/gateway/initialize_test.go +++ b/cmd/gateway/initialize_test.go @@ -133,7 +133,7 @@ func TestInitialize_Plus(t *testing.T) { g.Expect(fakeGenerator.GenerateDeploymentContextArgsForCall(0)).To(Equal(test.depCtx)) g.Expect(fakeCollector.CollectCallCount()).To(Equal(1)) g.Expect(fakeFileMgr.WriteCallCount()).To(Equal(1)) - g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(1)) + g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(3)) }) } } @@ -161,6 +161,7 @@ func TestCopyFileErrors(t *testing.T) { openErr := errors.New("open error") createErr := errors.New("create error") copyErr := errors.New("copy error") + chmodErr := errors.New("chmod error") tests := []struct { fileMgr *filefakes.FakeOSFileManager @@ -194,6 +195,15 @@ func TestCopyFileErrors(t *testing.T) { }, expErr: copyErr, }, + { + name: "can't set permissions", + fileMgr: &filefakes.FakeOSFileManager{ + ChmodStub: func(_ *os.File, _ os.FileMode) error { + return chmodErr + }, + }, + expErr: chmodErr, + }, } for _, test := range tests { diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml index 8e581ff56..35fb3d8ad 100644 --- a/config/tests/static-deployment.yaml +++ b/config/tests/static-deployment.yaml @@ -69,8 +69,9 @@ spec: capabilities: drop: - ALL + allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsUser: 102 + runAsUser: 101 runAsGroup: 1001 terminationGracePeriodSeconds: 30 serviceAccountName: nginx-gateway diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index 295cb42d0..dba4bc53d 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,6 +153,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -293,12 +292,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -333,13 +333,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -365,7 +363,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -390,7 +388,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 45121bf50..c5557b11b 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,6 +153,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -290,12 +289,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault nodeSelector: @@ -332,13 +332,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -364,7 +362,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -389,7 +387,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 2e1a53f3e..6b90c6669 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,6 +153,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -290,12 +289,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -330,13 +330,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -362,7 +360,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -387,7 +385,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index f846d0ca7..54bb27fc9 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -30,28 +30,18 @@ rules: - namespaces - services - secrets + - pods - configmaps verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -170,6 +160,10 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug kind: ConfigMap @@ -311,12 +305,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -351,13 +346,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -388,7 +381,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -416,7 +409,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 68bd273be..6116b0991 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -28,23 +28,19 @@ rules: - namespaces - services - secrets + - pods - configmaps verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -162,6 +158,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -296,12 +295,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -336,13 +336,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -368,7 +366,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -393,7 +391,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index adb6593de..7d545dce9 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -30,27 +30,17 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -165,6 +155,10 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug kind: ConfigMap @@ -305,12 +299,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -345,13 +340,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -382,7 +375,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -410,7 +403,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index af66c8faf..e5cdc5997 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -157,6 +153,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -290,12 +289,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -330,13 +330,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -362,7 +360,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -387,7 +385,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index a3bc1b01e..f32825a7b 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -165,6 +161,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -298,12 +297,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -338,13 +338,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -370,7 +368,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -395,7 +393,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: @@ -461,9 +459,6 @@ allowHostPID: false allowHostPorts: false allowPrivilegeEscalation: false allowPrivilegedContainer: false -allowedCapabilities: -- NET_BIND_SERVICE -- KILL apiVersion: security.openshift.io/v1 fsGroup: ranges: @@ -478,7 +473,7 @@ requiredDropCapabilities: - ALL runAsUser: type: MustRunAsRange - uidRangeMax: 102 + uidRangeMax: 101 uidRangeMin: 101 seLinuxContext: type: MustRunAs @@ -491,8 +486,3 @@ supplementalGroups: type: MustRunAs users: - system:serviceaccount:nginx-gateway:nginx-gateway -volumes: -- emptyDir -- secret -- configMap -- projected diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 6278c799f..6a7a6640c 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -30,27 +30,17 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get -- apiGroups: - - apps - resources: - - replicasets - verbs: - list - apiGroups: - "" @@ -167,6 +157,10 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics + - api-action log: level: debug kind: ConfigMap @@ -308,12 +302,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -348,13 +343,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -385,7 +378,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -413,7 +406,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index b4d01ca6f..910256fb7 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -28,22 +28,18 @@ rules: - namespaces - services - secrets + - pods verbs: - get - list - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - apiGroups: - apps resources: - replicasets verbs: - get + - list - apiGroups: - "" resources: @@ -159,6 +155,9 @@ data: - /var/run/nginx features: - connection + - configuration + - certificates + - metrics log: level: debug kind: ConfigMap @@ -293,12 +292,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 1 securityContext: + allowPrivilegeEscalation: false capabilities: drop: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault securityContext: @@ -333,13 +333,11 @@ spec: - containerPort: 443 name: https securityContext: - allowPrivilegeEscalation: false capabilities: add: - NET_BIND_SERVICE drop: - ALL - readOnlyRootFilesystem: true runAsGroup: 1001 runAsUser: 101 seccompProfile: @@ -365,7 +363,7 @@ spec: - command: - /usr/bin/gateway - sleep - - --duration=15s + - --duration=5s image: ghcr.io/nginxinc/nginx-gateway-fabric:edge imagePullPolicy: Always name: sleep @@ -390,7 +388,7 @@ spec: - ALL readOnlyRootFilesystem: true runAsGroup: 1001 - runAsUser: 102 + runAsUser: 101 seccompProfile: type: RuntimeDefault volumeMounts: diff --git a/go.mod b/go.mod index 01b9604df..4dcf235cd 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-kit/log v0.2.1 github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4 github.com/nginxinc/telemetry-exporter v0.1.2 @@ -19,6 +20,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.69.2 + google.golang.org/protobuf v1.35.2 k8s.io/api v0.32.0 k8s.io/apiextensions-apiserver v0.32.0 k8s.io/apimachinery v0.32.0 @@ -49,7 +51,6 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect - github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -83,7 +84,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/protobuf v1.35.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b88cbe43f..4aa869186 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,46 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -30,6 +54,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -58,6 +84,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -74,30 +102,60 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4 h1:Tn0SOlxq9uaJuqc6DUGZGYszrtHAHOaLnhbBWMzK1Bs= github.com/nginx/agent/v3 v3.0.0-20241220140549-28adb688a8b4/go.mod h1:HDi/Je5AKCe5by/hWs2jbzUqi3BN4K32hMD2/hWN5G8= +github.com/nginxinc/nginx-plus-go-client/v2 v2.0.1 h1:5VVK38bnELMDWnwfF6dSv57ResXh9AUzeDa72ENj94o= +github.com/nginxinc/nginx-plus-go-client/v2 v2.0.1/go.mod h1:He+1izxYxVVO5/C9ZTukwOpvkAx5eS19nRQgKXDhX5I= github.com/nginxinc/telemetry-exporter v0.1.2 h1:97vUGhQYgQ2KEsXKCBmr5gqfuujJCKPHwdg5HKoANUs= github.com/nginxinc/telemetry-exporter v0.1.2/go.mod h1:eKa/Ceh9irmyZ1xV2QxBIxduIyVC5RlmtiWwcTlHuMg= github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -109,22 +167,52 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= +github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= @@ -150,6 +238,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -211,6 +301,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/framework/file/file.go b/internal/framework/file/file.go index 0a6fb491a..82ecea73c 100644 --- a/internal/framework/file/file.go +++ b/internal/framework/file/file.go @@ -5,15 +5,21 @@ import ( "fmt" "io" "os" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate const ( - // regularFileMode defines the default file mode for regular files. - regularFileMode = 0o644 - // secretFileMode defines the default file mode for files with secrets. - secretFileMode = 0o640 + // RegularFileModeInt defines the default file mode for regular files as an integer. + RegularFileModeInt = 0o644 + // RegularFileMode defines the default file mode for regular files. + RegularFileMode = "0644" + // secretFileMode defines the default file mode for files with secrets as an integer. + secretFileModeInt = 0o640 + // SecretFileMode defines the default file mode for files with secrets. + SecretFileMode = "0640" ) // Type is the type of File. @@ -78,14 +84,14 @@ func Write(fileMgr OSFileManager, file File) error { switch file.Type { case TypeRegular: - if err := fileMgr.Chmod(f, regularFileMode); err != nil { + if err := fileMgr.Chmod(f, RegularFileModeInt); err != nil { resultErr = fmt.Errorf( - "failed to set file mode to %#o for %q: %w", regularFileMode, file.Path, err) + "failed to set file mode to %#o for %q: %w", RegularFileModeInt, file.Path, err) return resultErr } case TypeSecret: - if err := fileMgr.Chmod(f, secretFileMode); err != nil { - resultErr = fmt.Errorf("failed to set file mode to %#o for %q: %w", secretFileMode, file.Path, err) + if err := fileMgr.Chmod(f, secretFileModeInt); err != nil { + resultErr = fmt.Errorf("failed to set file mode to %#o for %q: %w", secretFileModeInt, file.Path, err) return resultErr } default: @@ -105,3 +111,24 @@ func ensureType(fileType Type) { panic(fmt.Sprintf("unknown file type %d", fileType)) } } + +// Convert an agent File to an internal File type. +func Convert(agentFile agent.File) File { + if agentFile.Meta == nil { + return File{} + } + + var t Type + switch agentFile.Meta.Permissions { + case RegularFileMode: + t = TypeRegular + case SecretFileMode: + t = TypeSecret + } + + return File{ + Content: agentFile.Contents, + Path: agentFile.Meta.Name, + Type: t, + } +} diff --git a/internal/framework/file/file_test.go b/internal/framework/file/file_test.go index d5b52b48a..00f58f480 100644 --- a/internal/framework/file/file_test.go +++ b/internal/framework/file/file_test.go @@ -5,11 +5,13 @@ import ( "os" "path/filepath" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file/filefakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" ) var _ = Describe("Write files", Ordered, func() { @@ -152,4 +154,35 @@ var _ = Describe("Write files", Ordered, func() { ), ) }) + + It("converts agent files to internal files", func() { + agentFile := agent.File{ + Contents: []byte("file contents"), + Meta: &pb.FileMeta{ + Name: "regular-file", + Permissions: file.RegularFileMode, + }, + } + expFile := file.File{ + Path: "regular-file", + Content: []byte("file contents"), + Type: file.TypeRegular, + } + + secretAgentFile := agent.File{ + Contents: []byte("secret contents"), + Meta: &pb.FileMeta{ + Name: "secret-file", + Permissions: file.SecretFileMode, + }, + } + expSecretFile := file.File{ + Path: "secret-file", + Content: []byte("secret contents"), + Type: file.TypeSecret, + } + + Expect(file.Convert(agentFile)).To(Equal(expFile)) + Expect(file.Convert(secretAgentFile)).To(Equal(expSecretFile)) + }) }) diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index a07688481..6fb0a3c50 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -164,7 +164,10 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.cfg.graphBuiltHealthChecker.setAsReady() } + // TODO(sberman): if nginx Deployment is scaled down, we should remove the pod from the ConnectionsTracker + // If fully deleted, then delete the deployment from the Store var err error + var configApplied bool switch changeType { case state.NoChange: logger.Info("Handling events didn't result into NGINX configuration changes") @@ -181,9 +184,14 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.setLatestConfiguration(&cfg) if h.cfg.plus { - h.cfg.nginxUpdater.UpdateUpstreamServers() + // TODO(sberman): hardcode this deployment name until we support provisioning data planes + deployment := types.NamespacedName{ + Name: "tmp-nginx-deployment", + Namespace: h.cfg.gatewayPodConfig.Namespace, + } + configApplied, err = h.cfg.nginxUpdater.UpdateUpstreamServers(ctx, deployment, cfg) } else { - err = h.updateNginxConf(cfg) + configApplied, err = h.updateNginxConf(ctx, cfg) } case state.ClusterStateChange: h.version++ @@ -196,20 +204,25 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.setLatestConfiguration(&cfg) - err = h.updateNginxConf(cfg) + configApplied, err = h.updateNginxConf(ctx, cfg) } var nginxReloadRes status.NginxReloadResult - if err != nil { + switch { + case err != nil: logger.Error(err, "Failed to update NGINX configuration") nginxReloadRes.Error = err - } else { + case configApplied: logger.Info("NGINX configuration was successfully updated") + default: + logger.Info("No NGINX instances to configure") } h.latestReloadResult = nginxReloadRes - h.updateStatuses(ctx, logger, gr) + if configApplied || err != nil { + h.updateStatuses(ctx, logger, gr) + } } func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logger, gr *graph.Graph) { @@ -295,19 +308,30 @@ func (h *eventHandlerImpl) parseAndCaptureEvent(ctx context.Context, logger logr } // updateNginxConf updates nginx conf files and reloads nginx. -// -//nolint:unparam // temporarily returning only nil -func (h *eventHandlerImpl) updateNginxConf(conf dataplane.Configuration) error { +func (h *eventHandlerImpl) updateNginxConf(ctx context.Context, conf dataplane.Configuration) (bool, error) { files := h.cfg.generator.Generate(conf) - h.cfg.nginxUpdater.UpdateConfig(len(files)) + // TODO(sberman): hardcode this deployment name until we support provisioning data planes + deployment := types.NamespacedName{ + Name: "tmp-nginx-deployment", + Namespace: h.cfg.gatewayPodConfig.Namespace, + } + + applied, err := h.cfg.nginxUpdater.UpdateConfig(ctx, deployment, files) + if err != nil { + return false, err + } // If using NGINX Plus, update upstream servers using the API. - if h.cfg.plus { - h.cfg.nginxUpdater.UpdateUpstreamServers() + var plusApplied bool + if h.cfg.plus && applied { + plusApplied, err = h.cfg.nginxUpdater.UpdateUpstreamServers(ctx, deployment, conf) + if err != nil { + return false, err + } } - return nil + return applied || plusApplied, nil } // updateControlPlaneAndSetStatus updates the control plane configuration and then sets the status @@ -423,6 +447,8 @@ func (h *eventHandlerImpl) GetLatestConfiguration() *dataplane.Configuration { } // setLatestConfiguration sets the latest configuration. +// TODO(sberman): once we support multiple Gateways, this will likely have to be a map +// of all configurations. func (h *eventHandlerImpl) setLatestConfiguration(cfg *dataplane.Configuration) { h.lock.Lock() defer h.lock.Unlock() diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index ce185565c..bc9530625 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/zap" @@ -19,12 +20,12 @@ import ( ngfAPI "github.com/nginxinc/nginx-gateway-fabric/apis/v1alpha1" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/events" - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/status/statusfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/licensing/licensingfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/metrics/collectors" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/agentfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/configfakes" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state" @@ -58,15 +59,15 @@ var _ = Describe("eventHandler", func() { } } - expectReconfig := func(expectedConf dataplane.Configuration, expectedFiles []file.File) { + expectReconfig := func(expectedConf dataplane.Configuration, expectedFiles []agent.File) { Expect(fakeProcessor.ProcessCallCount()).Should(Equal(1)) Expect(fakeGenerator.GenerateCallCount()).Should(Equal(1)) Expect(fakeGenerator.GenerateArgsForCall(0)).Should(Equal(expectedConf)) Expect(fakeNginxUpdater.UpdateConfigCallCount()).Should(Equal(1)) - lenFiles := fakeNginxUpdater.UpdateConfigArgsForCall(0) - Expect(expectedFiles).To(HaveLen(lenFiles)) + _, _, files := fakeNginxUpdater.UpdateConfigArgsForCall(0) + Expect(expectedFiles).To(Equal(files)) Expect(fakeStatusUpdater.UpdateGroupCallCount()).Should(Equal(2)) _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) @@ -83,6 +84,7 @@ var _ = Describe("eventHandler", func() { fakeProcessor.ProcessReturns(state.NoChange, &graph.Graph{}) fakeGenerator = &configfakes.FakeGenerator{} fakeNginxUpdater = &agentfakes.FakeNginxUpdater{} + fakeNginxUpdater.UpdateConfigReturns(true, nil) fakeStatusUpdater = &statusfakes.FakeGroupUpdater{} fakeEventRecorder = record.NewFakeRecorder(1) zapLogLevelSetter = newZapLogLevelSetter(zap.NewAtomicLevel()) @@ -113,10 +115,11 @@ var _ = Describe("eventHandler", func() { }) Describe("Process the Gateway API resources events", func() { - fakeCfgFiles := []file.File{ + fakeCfgFiles := []agent.File{ { - Type: file.TypeRegular, - Path: "test.conf", + Meta: &pb.FileMeta{ + Name: "test.conf", + }, }, } diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index b6da949c5..70ba17405 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -179,7 +179,7 @@ func StartManager(cfg config.Config) error { Logger: cfg.Logger.WithName("deployCtxCollector"), }) - nginxUpdater := agent.NewNginxUpdater(cfg.Logger.WithName("nginxUpdater"), cfg.Plus) + nginxUpdater := agent.NewNginxUpdater(cfg.Logger.WithName("nginxUpdater"), mgr.GetAPIReader(), cfg.Plus) grpcServer := agentgrpc.NewServer( cfg.Logger.WithName("agentGRPCServer"), @@ -505,6 +505,7 @@ func registerControllers( objectType: &ngfAPI.NginxGateway{}, options: []controller.Option{ controller.WithNamespacedNameFilter(filter.CreateSingleResourceFilter(controlConfigNSName)), + controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}), }, }) if err := setInitialConfig( diff --git a/internal/mode/static/nginx/agent/agent.go b/internal/mode/static/nginx/agent/agent.go index 1ce5d21b0..7e95bd10f 100644 --- a/internal/mode/static/nginx/agent/agent.go +++ b/internal/mode/static/nginx/agent/agent.go @@ -1,7 +1,20 @@ package agent import ( + "context" + "errors" + "fmt" + "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "google.golang.org/protobuf/types/known/structpb" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -10,39 +23,155 @@ import ( // NginxUpdater is an interface for updating NGINX using the NGINX agent. type NginxUpdater interface { - UpdateConfig(int) - UpdateUpstreamServers() + UpdateConfig( + ctx context.Context, + deploymentNsName types.NamespacedName, + files []File, + ) (bool, error) + UpdateUpstreamServers( + ctx context.Context, + deploymentNsName types.NamespacedName, + conf dataplane.Configuration, + ) (bool, error) } // NginxUpdaterImpl implements the NginxUpdater interface. type NginxUpdaterImpl struct { - CommandService *commandService - FileService *fileService - logger logr.Logger - plus bool + CommandService *commandService + FileService *fileService + nginxDeployments *DeploymentStore + logger logr.Logger + plus bool } // NewNginxUpdater returns a new NginxUpdaterImpl instance. -func NewNginxUpdater(logger logr.Logger, plus bool) *NginxUpdaterImpl { +func NewNginxUpdater( + logger logr.Logger, + reader client.Reader, + plus bool, +) *NginxUpdaterImpl { + connTracker := agentgrpc.NewConnectionsTracker() + nginxDeployments := NewDeploymentStore(connTracker) + + commandService := newCommandService(logger.WithName("commandService"), reader, nginxDeployments, connTracker) + fileService := newFileService(logger.WithName("fileService"), nginxDeployments, connTracker) + return &NginxUpdaterImpl{ - logger: logger, - plus: plus, - CommandService: newCommandService(logger.WithName("commandService")), - FileService: newFileService(logger.WithName("fileService")), + logger: logger, + plus: plus, + nginxDeployments: nginxDeployments, + CommandService: commandService, + FileService: fileService, } } // UpdateConfig sends the nginx configuration to the agent. -func (n *NginxUpdaterImpl) UpdateConfig(files int) { - n.logger.Info("Sending nginx configuration to agent", "numFiles", files) +// Returns whether configuration was applied or not, and any error that occurred. +func (n *NginxUpdaterImpl) UpdateConfig(ctx context.Context, nsName types.NamespacedName, files []File) (bool, error) { + n.logger.Info("Sending nginx configuration to agent") + + deployment := n.nginxDeployments.GetOrStore(ctx, nsName) + if deployment == nil { + return false, fmt.Errorf("failed to register nginx deployment %q", nsName.Name) + } + + // TODO(sberman): wait to send config until Deployment pods have all connected. + // If an nginx Pod creation event triggered this update, then we should include that + // pod name in the call to this function. Then we can wait for the DeploymentStore + // to show that this Pod has connected, and proceed with sending the config. + + msg := deployment.SetFiles(files) + + applied, err := deployment.GetBroadcaster().Send(msg) + if err != nil { + return false, fmt.Errorf("could not set nginx files: %w", err) + } + + return applied, nil } // UpdateUpstreamServers sends an APIRequest to the agent to update upstream servers using the NGINX Plus API. // Only applicable when using NGINX Plus. -func (n *NginxUpdaterImpl) UpdateUpstreamServers() { +// Returns whether configuration was applied or not, and any error that occurred. +func (n *NginxUpdaterImpl) UpdateUpstreamServers( + ctx context.Context, + nsName types.NamespacedName, + conf dataplane.Configuration, +) (bool, error) { if !n.plus { - return + return false, nil } n.logger.Info("Updating upstream servers using NGINX Plus API") + + deployment := n.nginxDeployments.GetOrStore(ctx, nsName) + if deployment == nil { + return false, fmt.Errorf("failed to register nginx deployment %q", nsName.Name) + } + broadcaster := deployment.GetBroadcaster() + + var updateErr error + var applied bool + actions := make([]*pb.NGINXPlusAction, 0, len(conf.Upstreams)) + for _, upstream := range conf.Upstreams { + action := &pb.NGINXPlusAction{ + Action: &pb.NGINXPlusAction_UpdateHttpUpstreamServers{ + UpdateHttpUpstreamServers: buildUpstreamServers(upstream), + }, + } + actions = append(actions, action) + + msg := broadcast.NginxAgentMessage{ + Type: broadcast.APIRequest, + NGINXPlusAction: action, + } + + var err error + applied, err = broadcaster.Send(msg) + if err != nil { + updateErr = errors.Join(updateErr, fmt.Errorf( + "couldn't update upstream %q via the API: %w", upstream.Name, err)) + } + } + // Store the most recent actions on the deployment so any new subscribers can apply them when first connecting. + deployment.SetNGINXPlusActions(actions) + + return applied, updateErr +} + +func buildUpstreamServers(upstream dataplane.Upstream) *pb.UpdateHTTPUpstreamServers { + servers := make([]*structpb.Struct, 0, len(upstream.Endpoints)) + + for _, endpoint := range upstream.Endpoints { + port, format := getPortAndIPFormat(endpoint) + value := fmt.Sprintf(format, endpoint.Address, port) + + server := &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "server": structpb.NewStringValue(value), + }, + } + + servers = append(servers, server) + } + + return &pb.UpdateHTTPUpstreamServers{ + HttpUpstreamName: upstream.Name, + Servers: servers, + } +} + +func getPortAndIPFormat(ep resolver.Endpoint) (string, string) { + var port string + + if ep.Port != 0 { + port = fmt.Sprintf(":%d", ep.Port) + } + + format := "%s%s" + if ep.IPv6 { + format = "[%s]%s" + } + + return port, format } diff --git a/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go b/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go index 013381415..ee228a361 100644 --- a/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go +++ b/internal/mode/static/nginx/agent/agentfakes/fake_nginx_updater.go @@ -2,36 +2,73 @@ package agentfakes import ( + "context" "sync" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" + "k8s.io/apimachinery/pkg/types" ) type FakeNginxUpdater struct { - UpdateConfigStub func(int) + UpdateConfigStub func(context.Context, types.NamespacedName, []agent.File) (bool, error) updateConfigMutex sync.RWMutex updateConfigArgsForCall []struct { - arg1 int + arg1 context.Context + arg2 types.NamespacedName + arg3 []agent.File } - UpdateUpstreamServersStub func() + updateConfigReturns struct { + result1 bool + result2 error + } + updateConfigReturnsOnCall map[int]struct { + result1 bool + result2 error + } + UpdateUpstreamServersStub func(context.Context, types.NamespacedName, dataplane.Configuration) (bool, error) updateUpstreamServersMutex sync.RWMutex updateUpstreamServersArgsForCall []struct { + arg1 context.Context + arg2 types.NamespacedName + arg3 dataplane.Configuration + } + updateUpstreamServersReturns struct { + result1 bool + result2 error + } + updateUpstreamServersReturnsOnCall map[int]struct { + result1 bool + result2 error } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeNginxUpdater) UpdateConfig(arg1 int) { +func (fake *FakeNginxUpdater) UpdateConfig(arg1 context.Context, arg2 types.NamespacedName, arg3 []agent.File) (bool, error) { + var arg3Copy []agent.File + if arg3 != nil { + arg3Copy = make([]agent.File, len(arg3)) + copy(arg3Copy, arg3) + } fake.updateConfigMutex.Lock() + ret, specificReturn := fake.updateConfigReturnsOnCall[len(fake.updateConfigArgsForCall)] fake.updateConfigArgsForCall = append(fake.updateConfigArgsForCall, struct { - arg1 int - }{arg1}) + arg1 context.Context + arg2 types.NamespacedName + arg3 []agent.File + }{arg1, arg2, arg3Copy}) stub := fake.UpdateConfigStub - fake.recordInvocation("UpdateConfig", []interface{}{arg1}) + fakeReturns := fake.updateConfigReturns + fake.recordInvocation("UpdateConfig", []interface{}{arg1, arg2, arg3Copy}) fake.updateConfigMutex.Unlock() if stub != nil { - fake.UpdateConfigStub(arg1) + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 } + return fakeReturns.result1, fakeReturns.result2 } func (fake *FakeNginxUpdater) UpdateConfigCallCount() int { @@ -40,29 +77,64 @@ func (fake *FakeNginxUpdater) UpdateConfigCallCount() int { return len(fake.updateConfigArgsForCall) } -func (fake *FakeNginxUpdater) UpdateConfigCalls(stub func(int)) { +func (fake *FakeNginxUpdater) UpdateConfigCalls(stub func(context.Context, types.NamespacedName, []agent.File) (bool, error)) { fake.updateConfigMutex.Lock() defer fake.updateConfigMutex.Unlock() fake.UpdateConfigStub = stub } -func (fake *FakeNginxUpdater) UpdateConfigArgsForCall(i int) int { +func (fake *FakeNginxUpdater) UpdateConfigArgsForCall(i int) (context.Context, types.NamespacedName, []agent.File) { fake.updateConfigMutex.RLock() defer fake.updateConfigMutex.RUnlock() argsForCall := fake.updateConfigArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeNginxUpdater) UpdateConfigReturns(result1 bool, result2 error) { + fake.updateConfigMutex.Lock() + defer fake.updateConfigMutex.Unlock() + fake.UpdateConfigStub = nil + fake.updateConfigReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeNginxUpdater) UpdateConfigReturnsOnCall(i int, result1 bool, result2 error) { + fake.updateConfigMutex.Lock() + defer fake.updateConfigMutex.Unlock() + fake.UpdateConfigStub = nil + if fake.updateConfigReturnsOnCall == nil { + fake.updateConfigReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.updateConfigReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} } -func (fake *FakeNginxUpdater) UpdateUpstreamServers() { +func (fake *FakeNginxUpdater) UpdateUpstreamServers(arg1 context.Context, arg2 types.NamespacedName, arg3 dataplane.Configuration) (bool, error) { fake.updateUpstreamServersMutex.Lock() + ret, specificReturn := fake.updateUpstreamServersReturnsOnCall[len(fake.updateUpstreamServersArgsForCall)] fake.updateUpstreamServersArgsForCall = append(fake.updateUpstreamServersArgsForCall, struct { - }{}) + arg1 context.Context + arg2 types.NamespacedName + arg3 dataplane.Configuration + }{arg1, arg2, arg3}) stub := fake.UpdateUpstreamServersStub - fake.recordInvocation("UpdateUpstreamServers", []interface{}{}) + fakeReturns := fake.updateUpstreamServersReturns + fake.recordInvocation("UpdateUpstreamServers", []interface{}{arg1, arg2, arg3}) fake.updateUpstreamServersMutex.Unlock() if stub != nil { - fake.UpdateUpstreamServersStub() + return stub(arg1, arg2, arg3) } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 } func (fake *FakeNginxUpdater) UpdateUpstreamServersCallCount() int { @@ -71,12 +143,45 @@ func (fake *FakeNginxUpdater) UpdateUpstreamServersCallCount() int { return len(fake.updateUpstreamServersArgsForCall) } -func (fake *FakeNginxUpdater) UpdateUpstreamServersCalls(stub func()) { +func (fake *FakeNginxUpdater) UpdateUpstreamServersCalls(stub func(context.Context, types.NamespacedName, dataplane.Configuration) (bool, error)) { fake.updateUpstreamServersMutex.Lock() defer fake.updateUpstreamServersMutex.Unlock() fake.UpdateUpstreamServersStub = stub } +func (fake *FakeNginxUpdater) UpdateUpstreamServersArgsForCall(i int) (context.Context, types.NamespacedName, dataplane.Configuration) { + fake.updateUpstreamServersMutex.RLock() + defer fake.updateUpstreamServersMutex.RUnlock() + argsForCall := fake.updateUpstreamServersArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeNginxUpdater) UpdateUpstreamServersReturns(result1 bool, result2 error) { + fake.updateUpstreamServersMutex.Lock() + defer fake.updateUpstreamServersMutex.Unlock() + fake.UpdateUpstreamServersStub = nil + fake.updateUpstreamServersReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeNginxUpdater) UpdateUpstreamServersReturnsOnCall(i int, result1 bool, result2 error) { + fake.updateUpstreamServersMutex.Lock() + defer fake.updateUpstreamServersMutex.Unlock() + fake.UpdateUpstreamServersStub = nil + if fake.updateUpstreamServersReturnsOnCall == nil { + fake.updateUpstreamServersReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.updateUpstreamServersReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeNginxUpdater) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() diff --git a/internal/mode/static/nginx/agent/broadcast/broadcast.go b/internal/mode/static/nginx/agent/broadcast/broadcast.go new file mode 100644 index 000000000..ded3e005e --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/broadcast.go @@ -0,0 +1,160 @@ +package broadcast + +import ( + "context" + "errors" + "sync" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . Broadcaster + +// Broadcaster defines an interface for consumers to subscribe to File updates. +type Broadcaster interface { + Subscribe() SubscriberChannels + Send(NginxAgentMessage) (bool, error) + CancelSubscription(string) +} + +type SubscriberChannels struct { + ListenCh <-chan NginxAgentMessage + ResponseCh chan<- error + ID string +} + +type storedChannels struct { + listenCh chan<- NginxAgentMessage + responseCh <-chan error + id string +} + +// DeploymentBroadcaster sends out a signal when an nginx Deployment has updated +// configuration files. The signal is received by any agent Subscription that cares +// about this Deployment. The agent Subscription will then send a response of whether or not +// the configuration was successfully applied. +type DeploymentBroadcaster struct { + publishCh chan NginxAgentMessage + subCh chan storedChannels + unsubCh chan string + listeners map[string]storedChannels + errorCh chan error +} + +// NewDeploymentBroadcaster returns a new instance of a DeploymentBroadcaster. +func NewDeploymentBroadcaster(ctx context.Context) *DeploymentBroadcaster { + broadcaster := &DeploymentBroadcaster{ + listeners: make(map[string]storedChannels), + publishCh: make(chan NginxAgentMessage), + subCh: make(chan storedChannels), + unsubCh: make(chan string), + errorCh: make(chan error), + } + go broadcaster.run(ctx) + + return broadcaster +} + +// Subscribe allows a listener to subscribe to broadcast messages. It returns the channel +// to listen on for messages, as well as a channel to respond on. +func (b *DeploymentBroadcaster) Subscribe() SubscriberChannels { + listenCh := make(chan NginxAgentMessage) + responseCh := make(chan error) + id := string(uuid.NewUUID()) + + subscriberChans := SubscriberChannels{ + ID: id, + ListenCh: listenCh, + ResponseCh: responseCh, + } + storedChans := storedChannels{ + id: id, + listenCh: listenCh, + responseCh: responseCh, + } + + b.subCh <- storedChans + return subscriberChans +} + +// Send the message to all listeners. Wait for all listeners to respond. +// Returns true if there were listeners that received the message, and returns any +// responses (nil for success, error for failure). +func (b *DeploymentBroadcaster) Send(message NginxAgentMessage) (bool, error) { + b.publishCh <- message + + return len(b.listeners) > 0, <-b.errorCh +} + +// CancelSubscription removes a Subscriber from the channel list. +func (b *DeploymentBroadcaster) CancelSubscription(id string) { + b.unsubCh <- id +} + +// run starts the broadcaster loop. It handles the following events: +// - if context is canceled, return. +// - if receiving a new subscriber, add it to the subscriber list. +// - if receiving a canceled subscription, remove it from the subscriber list. +// - if receiving a message to publish, send it to all subscribers. +func (b *DeploymentBroadcaster) run(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case channels := <-b.subCh: + b.listeners[channels.id] = channels + case id := <-b.unsubCh: + delete(b.listeners, id) + case msg := <-b.publishCh: + var wg sync.WaitGroup + wg.Add(len(b.listeners)) + + responses := make(chan error, len(b.listeners)) + for _, channels := range b.listeners { + go func() { + defer wg.Done() + + // send message and wait for it to be read + channels.listenCh <- msg + // wait for response + res := <-channels.responseCh + // add response to the list of responses + responses <- res + }() + } + wg.Wait() + + var err error + for range len(b.listeners) { + err = errors.Join(err, <-responses) + } + b.errorCh <- err + } + } +} + +// MessageType is the type of message to be sent. +type MessageType int + +const ( + // ConfigApplyRequest sends files to update nginx configuration. + ConfigApplyRequest MessageType = iota + // APIRequest sends an NGINX Plus API request to update configuration. + APIRequest +) + +// NginxAgentMessage is sent to all subscribers to send to the nginx agents for either a ConfigApplyRequest +// or an APIActionRequest. +type NginxAgentMessage struct { + // ConfigVersion is the hashed configuration version of the included files. + ConfigVersion string + // NGINXPlusAction is an NGINX Plus API action to be sent. + NGINXPlusAction *pb.NGINXPlusAction + // FileOverviews contain the overviews of all files to be sent. + FileOverviews []*pb.File + // Type defines the type of message to be sent. + Type MessageType +} diff --git a/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go b/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go new file mode 100644 index 000000000..f9fc63aa0 --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/broadcastfakes/fake_broadcaster.go @@ -0,0 +1,220 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package broadcastfakes + +import ( + "sync" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" +) + +type FakeBroadcaster struct { + CancelSubscriptionStub func(string) + cancelSubscriptionMutex sync.RWMutex + cancelSubscriptionArgsForCall []struct { + arg1 string + } + SendStub func(broadcast.NginxAgentMessage) (bool, error) + sendMutex sync.RWMutex + sendArgsForCall []struct { + arg1 broadcast.NginxAgentMessage + } + sendReturns struct { + result1 bool + result2 error + } + sendReturnsOnCall map[int]struct { + result1 bool + result2 error + } + SubscribeStub func() broadcast.SubscriberChannels + subscribeMutex sync.RWMutex + subscribeArgsForCall []struct { + } + subscribeReturns struct { + result1 broadcast.SubscriberChannels + } + subscribeReturnsOnCall map[int]struct { + result1 broadcast.SubscriberChannels + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeBroadcaster) CancelSubscription(arg1 string) { + fake.cancelSubscriptionMutex.Lock() + fake.cancelSubscriptionArgsForCall = append(fake.cancelSubscriptionArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.CancelSubscriptionStub + fake.recordInvocation("CancelSubscription", []interface{}{arg1}) + fake.cancelSubscriptionMutex.Unlock() + if stub != nil { + fake.CancelSubscriptionStub(arg1) + } +} + +func (fake *FakeBroadcaster) CancelSubscriptionCallCount() int { + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + return len(fake.cancelSubscriptionArgsForCall) +} + +func (fake *FakeBroadcaster) CancelSubscriptionCalls(stub func(string)) { + fake.cancelSubscriptionMutex.Lock() + defer fake.cancelSubscriptionMutex.Unlock() + fake.CancelSubscriptionStub = stub +} + +func (fake *FakeBroadcaster) CancelSubscriptionArgsForCall(i int) string { + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + argsForCall := fake.cancelSubscriptionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBroadcaster) Send(arg1 broadcast.NginxAgentMessage) (bool, error) { + fake.sendMutex.Lock() + ret, specificReturn := fake.sendReturnsOnCall[len(fake.sendArgsForCall)] + fake.sendArgsForCall = append(fake.sendArgsForCall, struct { + arg1 broadcast.NginxAgentMessage + }{arg1}) + stub := fake.SendStub + fakeReturns := fake.sendReturns + fake.recordInvocation("Send", []interface{}{arg1}) + fake.sendMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBroadcaster) SendCallCount() int { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + return len(fake.sendArgsForCall) +} + +func (fake *FakeBroadcaster) SendCalls(stub func(broadcast.NginxAgentMessage) (bool, error)) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = stub +} + +func (fake *FakeBroadcaster) SendArgsForCall(i int) broadcast.NginxAgentMessage { + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + argsForCall := fake.sendArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBroadcaster) SendReturns(result1 bool, result2 error) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + fake.sendReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeBroadcaster) SendReturnsOnCall(i int, result1 bool, result2 error) { + fake.sendMutex.Lock() + defer fake.sendMutex.Unlock() + fake.SendStub = nil + if fake.sendReturnsOnCall == nil { + fake.sendReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.sendReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeBroadcaster) Subscribe() broadcast.SubscriberChannels { + fake.subscribeMutex.Lock() + ret, specificReturn := fake.subscribeReturnsOnCall[len(fake.subscribeArgsForCall)] + fake.subscribeArgsForCall = append(fake.subscribeArgsForCall, struct { + }{}) + stub := fake.SubscribeStub + fakeReturns := fake.subscribeReturns + fake.recordInvocation("Subscribe", []interface{}{}) + fake.subscribeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeBroadcaster) SubscribeCallCount() int { + fake.subscribeMutex.RLock() + defer fake.subscribeMutex.RUnlock() + return len(fake.subscribeArgsForCall) +} + +func (fake *FakeBroadcaster) SubscribeCalls(stub func() broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = stub +} + +func (fake *FakeBroadcaster) SubscribeReturns(result1 broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = nil + fake.subscribeReturns = struct { + result1 broadcast.SubscriberChannels + }{result1} +} + +func (fake *FakeBroadcaster) SubscribeReturnsOnCall(i int, result1 broadcast.SubscriberChannels) { + fake.subscribeMutex.Lock() + defer fake.subscribeMutex.Unlock() + fake.SubscribeStub = nil + if fake.subscribeReturnsOnCall == nil { + fake.subscribeReturnsOnCall = make(map[int]struct { + result1 broadcast.SubscriberChannels + }) + } + fake.subscribeReturnsOnCall[i] = struct { + result1 broadcast.SubscriberChannels + }{result1} +} + +func (fake *FakeBroadcaster) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.cancelSubscriptionMutex.RLock() + defer fake.cancelSubscriptionMutex.RUnlock() + fake.sendMutex.RLock() + defer fake.sendMutex.RUnlock() + fake.subscribeMutex.RLock() + defer fake.subscribeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeBroadcaster) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ broadcast.Broadcaster = new(FakeBroadcaster) diff --git a/internal/mode/static/nginx/agent/broadcast/doc.go b/internal/mode/static/nginx/agent/broadcast/doc.go new file mode 100644 index 000000000..3640dcfa5 --- /dev/null +++ b/internal/mode/static/nginx/agent/broadcast/doc.go @@ -0,0 +1,5 @@ +/* +Package broadcast contains the functions for creating a broadcaster to send updates to consumers. +It is used to send nginx configuration for an nginx Deployment to all pod subscribers for that Deployment. +*/ +package broadcast diff --git a/internal/mode/static/nginx/agent/command.go b/internal/mode/static/nginx/agent/command.go index 9eabd8680..b9e051c31 100644 --- a/internal/mode/static/nginx/agent/command.go +++ b/internal/mode/static/nginx/agent/command.go @@ -4,28 +4,48 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/go-logr/logr" pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/meta" ) // commandService handles the connection and subscription to the data plane agent. type commandService struct { pb.CommandServiceServer - connTracker *agentgrpc.ConnectionsTracker + nginxDeployments *DeploymentStore + connTracker *agentgrpc.ConnectionsTracker + k8sReader client.Reader // TODO(sberman): all logs are at Info level right now. Adjust appropriately. logger logr.Logger } -func newCommandService(logger logr.Logger) *commandService { +func newCommandService( + logger logr.Logger, + reader client.Reader, + depStore *DeploymentStore, + connTracker *agentgrpc.ConnectionsTracker, +) *commandService { return &commandService{ - logger: logger, - connTracker: agentgrpc.NewConnectionsTracker(), + k8sReader: reader, + logger: logger, + connTracker: connTracker, + nginxDeployments: depStore, } } @@ -34,6 +54,8 @@ func (cs *commandService) Register(server *grpc.Server) { } // CreateConnection registers a data plane agent with the control plane. +// The nginx InstanceID could be empty if the agent hasn't discovered its nginx instance yet. +// Once discovered, the agent will send an UpdateDataPlaneStatus request with the nginx InstanceID set. func (cs *commandService) CreateConnection( ctx context.Context, req *pb.CreateConnectionRequest, @@ -47,10 +69,28 @@ func (cs *commandService) CreateConnection( return nil, agentgrpc.ErrStatusInvalidConnection } - podName := req.GetResource().GetContainerInfo().GetHostname() - + resource := req.GetResource() + podName := resource.GetContainerInfo().GetHostname() cs.logger.Info(fmt.Sprintf("Creating connection for nginx pod: %s", podName)) - cs.connTracker.Track(gi.IPAddress, podName) + + owner, err := cs.getPodOwner(podName) + if err != nil { + response := &pb.CreateConnectionResponse{ + Response: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_ERROR, + Message: "error getting pod owner", + Error: err.Error(), + }, + } + return response, status.Errorf(codes.Internal, "error getting pod owner %s", err.Error()) + } + + conn := agentgrpc.Connection{ + Parent: owner, + PodName: podName, + InstanceID: getNginxInstanceID(resource.GetInstances()), + } + cs.connTracker.Track(gi.IPAddress, conn) return &pb.CreateConnectionResponse{ Response: &pb.CommandResponse{ @@ -70,73 +110,215 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error cs.logger.Info(fmt.Sprintf("Received subscribe request from %q", gi.IPAddress)) - go cs.listenForDataPlaneResponse(ctx, in) - - // wait for the agent to report itself - podName, err := cs.waitForConnection(ctx, gi) + // wait for the agent to report itself and nginx + conn, deployment, err := cs.waitForConnection(ctx, gi) if err != nil { cs.logger.Error(err, "error waiting for connection") return err } - cs.logger.Info(fmt.Sprintf("Handling subscription for %s/%s", podName, gi.IPAddress)) + // apply current config before starting listen loop + deployment.RLock() + fileOverviews, configVersion := deployment.GetFileOverviews() + if err = in.Send(buildRequest(fileOverviews, conn.InstanceID, configVersion)); err != nil { + fmt.Printf("ERROR applying initial config: %v\n", err) + // TODO(sberman): how do we write this status? + } + + for _, action := range deployment.GetNGINXPlusActions() { + if err := in.Send(buildPlusAPIRequest(action, conn.InstanceID)); err != nil { + fmt.Printf("ERROR applying initial API config: %v\n", err) + // TODO(sberman): how do we write this status? + } + } + deployment.RUnlock() + + if err == nil { + cs.logger.Info(fmt.Sprintf("Successfully configured nginx for new subscription %q", conn.PodName)) + } + + // subscribe to the deployment broadcaster to get file updates + broadcaster := deployment.GetBroadcaster() + channels := broadcaster.Subscribe() + defer broadcaster.CancelSubscription(channels.ID) + + go cs.listenForDataPlaneResponse(ctx, in, channels.ResponseCh) + for { select { case <-ctx.Done(): return ctx.Err() - case <-time.After(1 * time.Minute): - dummyRequest := &pb.ManagementPlaneRequest{ - Request: &pb.ManagementPlaneRequest_HealthRequest{ - HealthRequest: &pb.HealthRequest{}, - }, + case msg := <-channels.ListenCh: + deployment.RLock() + var req *pb.ManagementPlaneRequest + switch msg.Type { + case broadcast.ConfigApplyRequest: + req = buildRequest(msg.FileOverviews, conn.InstanceID, msg.ConfigVersion) + case broadcast.APIRequest: + req = buildPlusAPIRequest(msg.NGINXPlusAction, conn.InstanceID) + default: + panic(fmt.Sprintf("unknown request type %d", msg.Type)) } - if err := in.Send(dummyRequest); err != nil { // TODO(sberman): will likely need retry logic + + if err := in.Send(req); err != nil { + deployment.RUnlock() cs.logger.Error(err, "error sending request to agent") + channels.ResponseCh <- err + + return err } + deployment.RUnlock() } } } // TODO(sberman): current issue: when control plane restarts, agent doesn't re-establish a CreateConnection call, // so this fails. -func (cs *commandService) waitForConnection(ctx context.Context, gi grpcContext.GrpcInfo) (string, error) { - var podName string +func (cs *commandService) waitForConnection( + ctx context.Context, + gi grpcContext.GrpcInfo, +) (*agentgrpc.Connection, *Deployment, error) { ticker := time.NewTicker(time.Second) defer ticker.Stop() timer := time.NewTimer(30 * time.Second) defer timer.Stop() + agentConnectErr := errors.New("timed out waiting for agent to register nginx") + deploymentStoreErr := errors.New("timed out waiting for nginx deployment to be added to store") + + var err error for { select { case <-ctx.Done(): - return "", ctx.Err() + return nil, nil, ctx.Err() case <-timer.C: - return "", errors.New("timed out waiting for agent connection") + return nil, nil, err case <-ticker.C: - if podName = cs.connTracker.GetConnection(gi.IPAddress); podName != "" { - return podName, nil + if conn, ok := cs.connTracker.ConnectionIsReady(gi.IPAddress); ok { + // connection has been established, now ensure that the deployment exists in the store + if deployment, ok := cs.nginxDeployments.Get(conn.Parent); ok { + return &conn, deployment, nil + } + err = deploymentStoreErr + continue } + err = agentConnectErr } } } -func (cs *commandService) listenForDataPlaneResponse(ctx context.Context, in pb.CommandService_SubscribeServer) { +func (cs *commandService) listenForDataPlaneResponse( + ctx context.Context, + in pb.CommandService_SubscribeServer, + responseCh chan<- error, +) { for { select { case <-ctx.Done(): return default: dataPlaneResponse, err := in.Recv() - cs.logger.Info(fmt.Sprintf("Received data plane response: %v", dataPlaneResponse)) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "context canceled") { cs.logger.Error(err, "failed to receive data plane response") return } + + res := dataPlaneResponse.GetCommandResponse() + if res.GetStatus() != pb.CommandResponse_COMMAND_STATUS_OK { + err := fmt.Errorf("bad response from agent: %s; error: %s", res.GetMessage(), res.GetError()) + responseCh <- err + } else { + responseCh <- nil + } } } } +func buildRequest(fileOverviews []*pb.File, instanceID, version string) *pb.ManagementPlaneRequest { + return &pb.ManagementPlaneRequest{ + MessageMeta: &pb.MessageMeta{ + MessageId: meta.GenerateMessageID(), + CorrelationId: meta.GenerateMessageID(), + Timestamp: timestamppb.Now(), + }, + Request: &pb.ManagementPlaneRequest_ConfigApplyRequest{ + ConfigApplyRequest: &pb.ConfigApplyRequest{ + Overview: &pb.FileOverview{ + Files: fileOverviews, + ConfigVersion: &pb.ConfigVersion{ + InstanceId: instanceID, + Version: version, + }, + }, + }, + }, + } +} + +func buildPlusAPIRequest(action *pb.NGINXPlusAction, instanceID string) *pb.ManagementPlaneRequest { + return &pb.ManagementPlaneRequest{ + MessageMeta: &pb.MessageMeta{ + MessageId: meta.GenerateMessageID(), + CorrelationId: meta.GenerateMessageID(), + Timestamp: timestamppb.Now(), + }, + Request: &pb.ManagementPlaneRequest_ActionRequest{ + ActionRequest: &pb.APIActionRequest{ + InstanceId: instanceID, + Action: &pb.APIActionRequest_NginxPlusAction{ + NginxPlusAction: action, + }, + }, + }, + } +} + +func (cs *commandService) getPodOwner(podName string) (types.NamespacedName, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var pods v1.PodList + listOpts := &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": podName}), + } + if err := cs.k8sReader.List(ctx, &pods, listOpts); err != nil { + return types.NamespacedName{}, fmt.Errorf("error listing pods: %w", err) + } + + if len(pods.Items) > 1 { + return types.NamespacedName{}, fmt.Errorf("should only be one pod with name %q", podName) + } + pod := pods.Items[0] + + podOwnerRefs := pod.GetOwnerReferences() + if len(podOwnerRefs) != 1 { + return types.NamespacedName{}, fmt.Errorf("expected one owner reference of the NGF Pod, got %d", len(podOwnerRefs)) + } + + if podOwnerRefs[0].Kind != "ReplicaSet" { + err := fmt.Errorf("expected pod owner reference to be ReplicaSet, got %s", podOwnerRefs[0].Kind) + return types.NamespacedName{}, err + } + + var replicaSet appsv1.ReplicaSet + if err := cs.k8sReader.Get( + ctx, + types.NamespacedName{Namespace: pod.Namespace, Name: podOwnerRefs[0].Name}, + &replicaSet, + ); err != nil { + return types.NamespacedName{}, fmt.Errorf("failed to get NGF Pod's ReplicaSet: %w", err) + } + + replicaOwnerRefs := replicaSet.GetOwnerReferences() + if len(replicaOwnerRefs) != 1 { + err := fmt.Errorf("expected one owner reference of the NGF ReplicaSet, got %d", len(replicaOwnerRefs)) + return types.NamespacedName{}, err + } + + return types.NamespacedName{Namespace: pod.Namespace, Name: replicaOwnerRefs[0].Name}, nil +} + // UpdateDataPlaneHealth includes full health information about the data plane as reported by the agent. // TODO(sberman): Is health monitoring the data planes something useful for us to do? func (cs *commandService) UpdateDataPlaneHealth( @@ -147,11 +329,39 @@ func (cs *commandService) UpdateDataPlaneHealth( } // UpdateDataPlaneStatus is called by agent on startup and upon any change in agent metadata, -// instance metadata, or configurations. Since directly changing nginx configuration on the instance -// is not supported, this is a no-op for NGF. +// instance metadata, or configurations. InstanceID may not be set on an initial CreateConnection, +// and will instead be set on a call to UpdateDataPlaneStatus once the agent discovers its nginx instance. func (cs *commandService) UpdateDataPlaneStatus( - _ context.Context, - _ *pb.UpdateDataPlaneStatusRequest, + ctx context.Context, + req *pb.UpdateDataPlaneStatusRequest, ) (*pb.UpdateDataPlaneStatusResponse, error) { + if req == nil { + return nil, errors.New("empty UpdateDataPlaneStatus request") + } + + gi, ok := grpcContext.GrpcInfoFromContext(ctx) + if !ok { + return nil, agentgrpc.ErrStatusInvalidConnection + } + + instanceID := getNginxInstanceID(req.GetResource().GetInstances()) + if instanceID == "" { + return nil, status.Errorf(codes.InvalidArgument, "request does not contain nginx instanceID") + } + + cs.connTracker.SetInstanceID(gi.IPAddress, instanceID) + return &pb.UpdateDataPlaneStatusResponse{}, nil } + +func getNginxInstanceID(instances []*pb.Instance) string { + for _, instance := range instances { + instanceType := instance.GetInstanceMeta().GetInstanceType() + if instanceType == pb.InstanceMeta_INSTANCE_TYPE_NGINX || + instanceType == pb.InstanceMeta_INSTANCE_TYPE_NGINX_PLUS { + return instance.GetInstanceMeta().GetInstanceId() + } + } + + return "" +} diff --git a/internal/mode/static/nginx/agent/deployment.go b/internal/mode/static/nginx/agent/deployment.go new file mode 100644 index 000000000..b4ecb3013 --- /dev/null +++ b/internal/mode/static/nginx/agent/deployment.go @@ -0,0 +1,177 @@ +package agent + +import ( + "context" + "fmt" + "sync" + + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" + "k8s.io/apimachinery/pkg/types" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/broadcast" + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/hack" +) + +// Deployment represents an nginx Deployment. It contains its own nginx configuration files, +// and a broadcaster for sending those files to all of its pods that are subscribed. +type Deployment struct { + broadcaster broadcast.Broadcaster + + configVersion string + nginxPlusActions []*pb.NGINXPlusAction + fileOverviews []*pb.File + files []File + + lock sync.RWMutex +} + +// newDeployment returns a new deployment object. +func newDeployment(ctx context.Context) *Deployment { + return &Deployment{ + broadcaster: broadcast.NewDeploymentBroadcaster(ctx), + } +} + +// RLock locks the deployment for reading. Used by the Subscriber to lock the deployment from any file +// changes while updating agent. +func (d *Deployment) RLock() { + d.lock.RLock() +} + +// RUnlock unlocks the deployment from reading. +func (d *Deployment) RUnlock() { + d.lock.RUnlock() +} + +// GetBroadcaster returns the deployment's broadcaster. +func (d *Deployment) GetBroadcaster() broadcast.Broadcaster { + return d.broadcaster +} + +// GetFileOverviews returns the current list of fileOverviews and configVersion for the deployment. +func (d *Deployment) GetFileOverviews() ([]*pb.File, string) { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.fileOverviews, d.configVersion +} + +// GetNGINXPlusActions returns the current NGINX Plus API Actions for the deployment. +func (d *Deployment) GetNGINXPlusActions() []*pb.NGINXPlusAction { + d.lock.RLock() + defer d.lock.RUnlock() + + return d.nginxPlusActions +} + +// GetFile gets the requested file for the deployment and returns its contents. +// This function MUST only be called after Deployment.Lock() has been called. +// This function is called by the agent during a ConfigApplyRequest transaction. +// Since the Deployment must be locked for the duration of the transaction, +// the Subscriber Locks and Unlocks the Deployment. +func (d *Deployment) GetFile(name, hash string) []byte { + for _, file := range d.files { + if name == file.Meta.GetName() && hash == file.Meta.GetHash() { + return file.Contents + } + } + + return nil +} + +// SetFiles updates the nginx files and fileOverviews for the deployment and returns the message to send. +func (d *Deployment) SetFiles(files []File) broadcast.NginxAgentMessage { + d.lock.Lock() + defer d.lock.Unlock() + + d.files = files + + fileOverviews := make([]*pb.File, 0, len(files)) + for _, file := range files { + fileOverviews = append(fileOverviews, &pb.File{FileMeta: file.Meta}) + } + + // hack to include unchanging static files in the payload so they don't get deleted + staticFiles := hack.GetStaticFiles() + for _, file := range staticFiles { + meta := &pb.FileMeta{ + Name: file.Name, + Hash: filesHelper.GenerateHash(file.Contents), + Permissions: file.Permissions, + } + + fileOverviews = append(fileOverviews, &pb.File{ + FileMeta: meta, + }) + + d.files = append(d.files, File{Meta: meta, Contents: file.Contents}) + } + + d.configVersion = filesHelper.GenerateConfigVersion(fileOverviews) + d.fileOverviews = fileOverviews + + return broadcast.NginxAgentMessage{ + Type: broadcast.ConfigApplyRequest, + FileOverviews: fileOverviews, + ConfigVersion: d.configVersion, + } +} + +// SetNGINXPlusActions updates the deployment's latest NGINX Plus Actions to perform if using NGINX Plus. +// Used by a Subscriber when it first connects. +func (d *Deployment) SetNGINXPlusActions(actions []*pb.NGINXPlusAction) { + d.lock.Lock() + defer d.lock.Unlock() + + d.nginxPlusActions = actions +} + +// DeploymentStore holds a map of all Deployments. +type DeploymentStore struct { + connTracker *agentgrpc.ConnectionsTracker + deployments sync.Map +} + +// NewDeploymentStore returns a new instance of a DeploymentStore. +func NewDeploymentStore(connTracker *agentgrpc.ConnectionsTracker) *DeploymentStore { + return &DeploymentStore{ + connTracker: connTracker, + } +} + +// Get returns the desired deployment from the store. +func (d *DeploymentStore) Get(nsName types.NamespacedName) (*Deployment, bool) { + val, ok := d.deployments.Load(nsName) + if !ok { + return nil, false + } + + deployment, ok := val.(*Deployment) + if !ok { + panic(fmt.Sprintf("expected Deployment, got type %T", val)) + } + + return deployment, true +} + +// GetOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +func (d *DeploymentStore) GetOrStore(ctx context.Context, nsName types.NamespacedName) *Deployment { + if deployment, ok := d.Get(nsName); ok { + return deployment + } + + deployment := newDeployment(ctx) + d.deployments.Store(nsName, deployment) + + return deployment +} + +// Remove cleans up any connections that are tracked for this deployment, and then removes +// the deployment from the store. +func (d *DeploymentStore) Remove(nsName types.NamespacedName) { + d.connTracker.UntrackConnectionsForParent(nsName) + d.deployments.Delete(nsName) +} diff --git a/internal/mode/static/nginx/agent/file.go b/internal/mode/static/nginx/agent/file.go index 296e1705e..d2b1b390f 100644 --- a/internal/mode/static/nginx/agent/file.go +++ b/internal/mode/static/nginx/agent/file.go @@ -2,22 +2,43 @@ package agent import ( "context" - "fmt" "github.com/go-logr/logr" pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + agentgrpc "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" + grpcContext "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" ) +// File is an nginx configuration file that the nginx agent gets from the control plane +// after a ConfigApplyRequest. +type File struct { + Meta *pb.FileMeta + Contents []byte +} + // fileService handles file management between the control plane and the agent. type fileService struct { pb.FileServiceServer + nginxDeployments *DeploymentStore + connTracker *agentgrpc.ConnectionsTracker // TODO(sberman): all logs are at Info level right now. Adjust appropriately. logger logr.Logger } -func newFileService(logger logr.Logger) *fileService { - return &fileService{logger: logger} +func newFileService( + logger logr.Logger, + depStore *DeploymentStore, + connTracker *agentgrpc.ConnectionsTracker, +) *fileService { + return &fileService{ + logger: logger, + nginxDeployments: depStore, + connTracker: connTracker, + } } func (fs *fileService) Register(server *grpc.Server) { @@ -25,28 +46,47 @@ func (fs *fileService) Register(server *grpc.Server) { } // GetOverview gets the overview of files for a particular configuration version of an instance. -// Agent calls this if it's missing an overview when a ConfigApplyRequest is called by the control plane. +// At the moment it doesn't appear to be used by the agent. func (fs *fileService) GetOverview( _ context.Context, _ *pb.GetOverviewRequest, ) (*pb.GetOverviewResponse, error) { - fs.logger.Info("Get overview request") - - return &pb.GetOverviewResponse{ - Overview: &pb.FileOverview{}, - }, nil + return &pb.GetOverviewResponse{}, nil } // GetFile is called by the agent when it needs to download a file for a ConfigApplyRequest. func (fs *fileService) GetFile( - _ context.Context, + ctx context.Context, req *pb.GetFileRequest, ) (*pb.GetFileResponse, error) { filename := req.GetFileMeta().GetName() hash := req.GetFileMeta().GetHash() - fs.logger.Info(fmt.Sprintf("Getting file: %s, %s", filename, hash)) - return &pb.GetFileResponse{}, nil + gi, ok := grpcContext.GrpcInfoFromContext(ctx) + if !ok { + return nil, agentgrpc.ErrStatusInvalidConnection + } + + conn := fs.connTracker.GetConnection(gi.IPAddress) + if conn.PodName == "" { + return nil, status.Errorf(codes.NotFound, "connection not found") + } + + deployment, ok := fs.nginxDeployments.Get(conn.Parent) + if !ok { + return nil, status.Errorf(codes.NotFound, "deployment not found in store") + } + + contents := deployment.GetFile(filename, hash) + if len(contents) == 0 { + return nil, status.Errorf(codes.NotFound, "File not found") + } + + return &pb.GetFileResponse{ + Contents: &pb.FileContents{ + Contents: contents, + }, + }, nil } // UpdateOverview is called by agent on startup and whenever any files change on the instance. diff --git a/internal/mode/static/nginx/agent/grpc/connections.go b/internal/mode/static/nginx/agent/grpc/connections.go index af99b8400..324288382 100644 --- a/internal/mode/static/nginx/agent/grpc/connections.go +++ b/internal/mode/static/nginx/agent/grpc/connections.go @@ -2,23 +2,28 @@ package grpc import ( "sync" + + "k8s.io/apimachinery/pkg/types" ) +type Connection struct { + PodName string + InstanceID string + Parent types.NamespacedName +} + // ConnectionsTracker keeps track of all connections between the control plane and nginx agents. type ConnectionsTracker struct { - // connections contains a map of all IP addresses that have connected and their associated pod names. - // TODO(sberman): we'll likely need to create a channel for each connection that can be stored in this map. - // Then the Subscription listens on the channel for its connection, while the nginxUpdater sends the config - // for the pod over that channel. - connections map[string]string + // connections contains a map of all IP addresses that have connected and their connection info. + connections map[string]Connection - lock sync.Mutex + lock sync.RWMutex } // NewConnectionsTracker returns a new ConnectionsTracker instance. func NewConnectionsTracker() *ConnectionsTracker { return &ConnectionsTracker{ - connections: make(map[string]string), + connections: make(map[string]Connection), } } @@ -26,25 +31,50 @@ func NewConnectionsTracker() *ConnectionsTracker { // TODO(sberman): we need to handle the case when the token expires (once we support the token). // This likely involves setting a callback to cancel a context when the token expires, which triggers // the connection to be removed from the tracking list. -func (c *ConnectionsTracker) Track(address, hostname string) { +func (c *ConnectionsTracker) Track(key string, conn Connection) { c.lock.Lock() defer c.lock.Unlock() - c.connections[address] = hostname + c.connections[key] = conn +} + +// GetConnection returns the requested connection. +func (c *ConnectionsTracker) GetConnection(key string) Connection { + c.lock.RLock() + defer c.lock.RUnlock() + + return c.connections[key] } -// GetConnections returns all connections that are currently tracked. -func (c *ConnectionsTracker) GetConnections() map[string]string { +// ConnectionIsReady returns if the connection is ready to be used. In other words, agent +// has registered itself and an nginx instance with the control plane. +func (c *ConnectionsTracker) ConnectionIsReady(key string) (Connection, bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + conn, ok := c.connections[key] + return conn, ok && conn.InstanceID != "" +} + +// SetInstanceID sets the nginx instanceID for a connection. +func (c *ConnectionsTracker) SetInstanceID(key, id string) { c.lock.Lock() defer c.lock.Unlock() - return c.connections + if conn, ok := c.connections[key]; ok { + conn.InstanceID = id + c.connections[key] = conn + } } -// GetConnection returns the hostname of the requested connection. -func (c *ConnectionsTracker) GetConnection(address string) string { +// UntrackConnectionsForParent removes all Connections that reference the specified parent. +func (c *ConnectionsTracker) UntrackConnectionsForParent(parent types.NamespacedName) { c.lock.Lock() defer c.lock.Unlock() - return c.connections[address] + for key, conn := range c.connections { + if conn.Parent == parent { + delete(c.connections, key) + } + } } diff --git a/internal/mode/static/nginx/agent/hack/hack.go b/internal/mode/static/nginx/agent/hack/hack.go new file mode 100644 index 000000000..7f158f570 --- /dev/null +++ b/internal/mode/static/nginx/agent/hack/hack.go @@ -0,0 +1,182 @@ +package hack + +// This is here for now to send our base files to agent that NGF normally doesn't send. +// Otherwise, agent will attempt to delete them since they aren't included in the payload. + +type File struct { + Name string + Permissions string + Contents []byte +} + +func GetStaticFiles() []File { + return []File{ + { + Name: "/etc/nginx/nginx.conf", + Permissions: "0644", + Contents: nginxConf, + }, + { + Name: "/etc/nginx/mime.types", + Permissions: "0644", + Contents: mimeTypes, + }, + } +} + +var nginxConf = []byte(` +load_module /usr/lib/nginx/modules/ngx_http_js_module.so; +include /etc/nginx/main-includes/*.conf; + +worker_processes auto; + +pid /var/run/nginx/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/mime.types; + js_import /usr/lib/nginx/modules/njs/httpmatches.js; + + default_type application/octet-stream; + + proxy_headers_hash_bucket_size 512; + proxy_headers_hash_max_size 1024; + server_names_hash_bucket_size 256; + server_names_hash_max_size 1024; + variables_hash_bucket_size 512; + variables_hash_max_size 1024; + + sendfile on; + tcp_nopush on; + + server_tokens off; + + server { + listen unix:/var/run/nginx/nginx-status.sock; + access_log off; + + location /stub_status { + stub_status; + } + } +} + +stream { + variables_hash_bucket_size 512; + variables_hash_max_size 1024; + + map_hash_max_size 2048; + map_hash_bucket_size 256; + + log_format stream-main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$ssl_preread_server_name"'; + access_log /dev/stdout stream-main; + include /etc/nginx/stream-conf.d/*.conf; +} + `) + +var mimeTypes = []byte(` +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/avif avif; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + + font/woff woff; + font/woff2 woff2; + + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; + application/vnd.openxmlformats-officedocument.presentationml.presentation + pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document + docx; + application/vnd.wap.wmlc wmlc; + application/wasm wasm; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} +`) diff --git a/internal/mode/static/nginx/agent/meta/doc.go b/internal/mode/static/nginx/agent/meta/doc.go new file mode 100644 index 000000000..166cc412c --- /dev/null +++ b/internal/mode/static/nginx/agent/meta/doc.go @@ -0,0 +1,4 @@ +/* +Package meta contains the functions for creating MessageMeta that is included in requests to the nginx agent. +*/ +package meta diff --git a/internal/mode/static/nginx/agent/meta/meta.go b/internal/mode/static/nginx/agent/meta/meta.go new file mode 100644 index 000000000..5621f9604 --- /dev/null +++ b/internal/mode/static/nginx/agent/meta/meta.go @@ -0,0 +1,20 @@ +package meta + +import ( + "time" + + "github.com/google/uuid" + agentUuid "github.com/nginx/agent/v3/pkg/uuid" +) + +const CorrelationIDKey = "correlation_id" + +// GenerateMessageID generates a unique message ID, falling back to sha256 and timestamp if UUID generation fails. +func GenerateMessageID() string { + uuidv7, err := uuid.NewUUID() + if err != nil { + return agentUuid.Generate("%s", time.Now().String()) + } + + return uuidv7.String() +} diff --git a/internal/mode/static/nginx/config/configfakes/fake_generator.go b/internal/mode/static/nginx/config/configfakes/fake_generator.go index 9746df517..f5475a16b 100644 --- a/internal/mode/static/nginx/config/configfakes/fake_generator.go +++ b/internal/mode/static/nginx/config/configfakes/fake_generator.go @@ -4,41 +4,41 @@ package configfakes import ( "sync" - "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" ) type FakeGenerator struct { - GenerateStub func(dataplane.Configuration) []file.File + GenerateStub func(dataplane.Configuration) []agent.File generateMutex sync.RWMutex generateArgsForCall []struct { arg1 dataplane.Configuration } generateReturns struct { - result1 []file.File + result1 []agent.File } generateReturnsOnCall map[int]struct { - result1 []file.File + result1 []agent.File } - GenerateDeploymentContextStub func(dataplane.DeploymentContext) (file.File, error) + GenerateDeploymentContextStub func(dataplane.DeploymentContext) (agent.File, error) generateDeploymentContextMutex sync.RWMutex generateDeploymentContextArgsForCall []struct { arg1 dataplane.DeploymentContext } generateDeploymentContextReturns struct { - result1 file.File + result1 agent.File result2 error } generateDeploymentContextReturnsOnCall map[int]struct { - result1 file.File + result1 agent.File result2 error } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeGenerator) Generate(arg1 dataplane.Configuration) []file.File { +func (fake *FakeGenerator) Generate(arg1 dataplane.Configuration) []agent.File { fake.generateMutex.Lock() ret, specificReturn := fake.generateReturnsOnCall[len(fake.generateArgsForCall)] fake.generateArgsForCall = append(fake.generateArgsForCall, struct { @@ -63,7 +63,7 @@ func (fake *FakeGenerator) GenerateCallCount() int { return len(fake.generateArgsForCall) } -func (fake *FakeGenerator) GenerateCalls(stub func(dataplane.Configuration) []file.File) { +func (fake *FakeGenerator) GenerateCalls(stub func(dataplane.Configuration) []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = stub @@ -76,30 +76,30 @@ func (fake *FakeGenerator) GenerateArgsForCall(i int) dataplane.Configuration { return argsForCall.arg1 } -func (fake *FakeGenerator) GenerateReturns(result1 []file.File) { +func (fake *FakeGenerator) GenerateReturns(result1 []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = nil fake.generateReturns = struct { - result1 []file.File + result1 []agent.File }{result1} } -func (fake *FakeGenerator) GenerateReturnsOnCall(i int, result1 []file.File) { +func (fake *FakeGenerator) GenerateReturnsOnCall(i int, result1 []agent.File) { fake.generateMutex.Lock() defer fake.generateMutex.Unlock() fake.GenerateStub = nil if fake.generateReturnsOnCall == nil { fake.generateReturnsOnCall = make(map[int]struct { - result1 []file.File + result1 []agent.File }) } fake.generateReturnsOnCall[i] = struct { - result1 []file.File + result1 []agent.File }{result1} } -func (fake *FakeGenerator) GenerateDeploymentContext(arg1 dataplane.DeploymentContext) (file.File, error) { +func (fake *FakeGenerator) GenerateDeploymentContext(arg1 dataplane.DeploymentContext) (agent.File, error) { fake.generateDeploymentContextMutex.Lock() ret, specificReturn := fake.generateDeploymentContextReturnsOnCall[len(fake.generateDeploymentContextArgsForCall)] fake.generateDeploymentContextArgsForCall = append(fake.generateDeploymentContextArgsForCall, struct { @@ -124,7 +124,7 @@ func (fake *FakeGenerator) GenerateDeploymentContextCallCount() int { return len(fake.generateDeploymentContextArgsForCall) } -func (fake *FakeGenerator) GenerateDeploymentContextCalls(stub func(dataplane.DeploymentContext) (file.File, error)) { +func (fake *FakeGenerator) GenerateDeploymentContextCalls(stub func(dataplane.DeploymentContext) (agent.File, error)) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = stub @@ -137,28 +137,28 @@ func (fake *FakeGenerator) GenerateDeploymentContextArgsForCall(i int) dataplane return argsForCall.arg1 } -func (fake *FakeGenerator) GenerateDeploymentContextReturns(result1 file.File, result2 error) { +func (fake *FakeGenerator) GenerateDeploymentContextReturns(result1 agent.File, result2 error) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = nil fake.generateDeploymentContextReturns = struct { - result1 file.File + result1 agent.File result2 error }{result1, result2} } -func (fake *FakeGenerator) GenerateDeploymentContextReturnsOnCall(i int, result1 file.File, result2 error) { +func (fake *FakeGenerator) GenerateDeploymentContextReturnsOnCall(i int, result1 agent.File, result2 error) { fake.generateDeploymentContextMutex.Lock() defer fake.generateDeploymentContextMutex.Unlock() fake.GenerateDeploymentContextStub = nil if fake.generateDeploymentContextReturnsOnCall == nil { fake.generateDeploymentContextReturnsOnCall = make(map[int]struct { - result1 file.File + result1 agent.File result2 error }) } fake.generateDeploymentContextReturnsOnCall[i] = struct { - result1 file.File + result1 agent.File result2 error }{result1, result2} } diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index df43d0fb5..e55f40b06 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -6,9 +6,12 @@ import ( "path/filepath" "github.com/go-logr/logr" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/http" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/clientsettings" @@ -61,9 +64,9 @@ const ( // This interface is used for testing purposes only. type Generator interface { // Generate generates NGINX configuration files from internal representation. - Generate(configuration dataplane.Configuration) []file.File + Generate(configuration dataplane.Configuration) []agent.File // GenerateDeploymentContext generates the deployment context used for N+ licensing. - GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (file.File, error) + GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (agent.File, error) } // GeneratorImpl is an implementation of Generator. @@ -103,8 +106,8 @@ type executeFunc func(configuration dataplane.Configuration) []executeResult // It is the responsibility of the caller to validate the configuration before calling this function. // In case of invalid configuration, NGINX will fail to reload or could be configured with malicious configuration. // To validate, use the validators from the validation package. -func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { - files := make([]file.File, 0) +func (g GeneratorImpl) Generate(conf dataplane.Configuration) []agent.File { + files := make([]agent.File, 0) for id, pair := range conf.SSLKeyPairs { files = append(files, generatePEM(id, pair.Cert, pair.Key)) @@ -126,16 +129,19 @@ func (g GeneratorImpl) Generate(conf dataplane.Configuration) []file.File { // GenerateDeploymentContext generates the deployment_ctx.json file needed for N+ licensing. // It's exported since it's used by the init container process. -func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (file.File, error) { +func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentContext) (agent.File, error) { depCtxBytes, err := json.Marshal(depCtx) if err != nil { - return file.File{}, fmt.Errorf("error building deployment context for mgmt block: %w", err) + return agent.File{}, fmt.Errorf("error building deployment context for mgmt block: %w", err) } - deploymentCtxFile := file.File{ - Content: depCtxBytes, - Path: mainIncludesFolder + "/deployment_ctx.json", - Type: file.TypeRegular, + deploymentCtxFile := agent.File{ + Meta: &pb.FileMeta{ + Name: mainIncludesFolder + "/deployment_ctx.json", + Hash: filesHelper.GenerateHash(depCtxBytes), + Permissions: file.RegularFileMode, + }, + Contents: depCtxBytes, } return deploymentCtxFile, nil @@ -144,7 +150,7 @@ func (g GeneratorImpl) GenerateDeploymentContext(depCtx dataplane.DeploymentCont func (g GeneratorImpl) executeConfigTemplates( conf dataplane.Configuration, generator policies.Generator, -) []file.File { +) []agent.File { fileBytes := make(map[string][]byte) httpUpstreams := g.createUpstreams(conf.Upstreams, upstreamsettings.NewProcessor()) @@ -157,17 +163,20 @@ func (g GeneratorImpl) executeConfigTemplates( } } - var mgmtFiles []file.File + var mgmtFiles []agent.File if g.plus { mgmtFiles = g.generateMgmtFiles(conf) } - files := make([]file.File, 0, len(fileBytes)+len(mgmtFiles)) + files := make([]agent.File, 0, len(fileBytes)+len(mgmtFiles)) for fp, bytes := range fileBytes { - files = append(files, file.File{ - Path: fp, - Content: bytes, - Type: file.TypeRegular, + files = append(files, agent.File{ + Meta: &pb.FileMeta{ + Name: fp, + Hash: filesHelper.GenerateHash(bytes), + Permissions: file.RegularFileMode, + }, + Contents: bytes, }) } files = append(files, mgmtFiles...) @@ -194,16 +203,19 @@ func (g GeneratorImpl) getExecuteFuncs( } } -func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) file.File { +func generatePEM(id dataplane.SSLKeyPairID, cert []byte, key []byte) agent.File { c := make([]byte, 0, len(cert)+len(key)+1) c = append(c, cert...) c = append(c, '\n') c = append(c, key...) - return file.File{ - Content: c, - Path: generatePEMFileName(id), - Type: file.TypeSecret, + return agent.File{ + Meta: &pb.FileMeta{ + Name: generatePEMFileName(id), + Hash: filesHelper.GenerateHash(c), + Permissions: file.SecretFileMode, + }, + Contents: c, } } @@ -211,11 +223,14 @@ func generatePEMFileName(id dataplane.SSLKeyPairID) string { return filepath.Join(secretsFolder, string(id)+".pem") } -func generateCertBundle(id dataplane.CertBundleID, cert []byte) file.File { - return file.File{ - Content: cert, - Path: generateCertBundleFileName(id), - Type: file.TypeRegular, +func generateCertBundle(id dataplane.CertBundleID, cert []byte) agent.File { + return agent.File{ + Meta: &pb.FileMeta{ + Name: generateCertBundleFileName(id), + Hash: filesHelper.GenerateHash(cert), + Permissions: file.SecretFileMode, + }, + Contents: cert, } } diff --git a/internal/mode/static/nginx/config/generator_test.go b/internal/mode/static/nginx/config/generator_test.go index a6f4540ad..14e9171db 100644 --- a/internal/mode/static/nginx/config/generator_test.go +++ b/internal/mode/static/nginx/config/generator_test.go @@ -4,6 +4,8 @@ import ( "sort" "testing" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -11,6 +13,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" ngfConfig "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -145,7 +148,7 @@ func TestGenerate(t *testing.T) { g.Expect(files).To(HaveLen(16)) arrange := func(i, j int) bool { - return files[i].Path < files[j].Path + return files[i].Meta.Name < files[j].Meta.Name } sort.Slice(files, arrange) @@ -169,9 +172,9 @@ func TestGenerate(t *testing.T) { /etc/nginx/stream-conf.d/stream.conf */ - g.Expect(files[0].Type).To(Equal(file.TypeRegular)) - g.Expect(files[0].Path).To(Equal("/etc/nginx/conf.d/http.conf")) - httpCfg := string(files[0].Content) // converting to string so that on failure gomega prints strings not byte arrays + g.Expect(files[0].Meta.Permissions).To(Equal(file.RegularFileMode)) + g.Expect(files[0].Meta.Name).To(Equal("/etc/nginx/conf.d/http.conf")) + httpCfg := string(files[0].Contents) // converting to string so that on failure gomega prints strings not byte arrays // Note: this only verifies that Generate() returns a byte array with upstream, server, and split_client blocks. // It does not test the correctness of those blocks. That functionality is covered by other tests in this package. g.Expect(httpCfg).To(ContainSubstring("listen 80")) @@ -188,34 +191,34 @@ func TestGenerate(t *testing.T) { g.Expect(httpCfg).To(ContainSubstring("include /etc/nginx/includes/http_snippet1.conf;")) g.Expect(httpCfg).To(ContainSubstring("include /etc/nginx/includes/http_snippet2.conf;")) - g.Expect(files[1].Path).To(Equal("/etc/nginx/conf.d/matches.json")) + g.Expect(files[1].Meta.Name).To(Equal("/etc/nginx/conf.d/matches.json")) - g.Expect(files[1].Type).To(Equal(file.TypeRegular)) + g.Expect(files[1].Meta.Permissions).To(Equal(file.RegularFileMode)) expString := "{}" - g.Expect(string(files[1].Content)).To(Equal(expString)) + g.Expect(string(files[1].Contents)).To(Equal(expString)) // snippet include files // content is not checked in this test. - g.Expect(files[2].Path).To(Equal("/etc/nginx/includes/http_snippet1.conf")) - g.Expect(files[3].Path).To(Equal("/etc/nginx/includes/http_snippet2.conf")) - g.Expect(files[4].Path).To(Equal("/etc/nginx/includes/main_snippet1.conf")) - g.Expect(files[5].Path).To(Equal("/etc/nginx/includes/main_snippet2.conf")) + g.Expect(files[2].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet1.conf")) + g.Expect(files[3].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet2.conf")) + g.Expect(files[4].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet1.conf")) + g.Expect(files[5].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet2.conf")) - g.Expect(files[6].Path).To(Equal("/etc/nginx/main-includes/deployment_ctx.json")) - deploymentCtx := string(files[6].Content) + g.Expect(files[6].Meta.Name).To(Equal("/etc/nginx/main-includes/deployment_ctx.json")) + deploymentCtx := string(files[6].Contents) g.Expect(deploymentCtx).To(ContainSubstring("\"integration\":\"ngf\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_id\":\"test-uid\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"installation_id\":\"test-uid-replicaSet\"")) g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_node_count\":1")) - g.Expect(files[7].Path).To(Equal("/etc/nginx/main-includes/main.conf")) - mainConfStr := string(files[7].Content) + g.Expect(files[7].Meta.Name).To(Equal("/etc/nginx/main-includes/main.conf")) + mainConfStr := string(files[7].Contents) g.Expect(mainConfStr).To(ContainSubstring("load_module modules/ngx_otel_module.so;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet1.conf;")) g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet2.conf;")) - g.Expect(files[8].Path).To(Equal("/etc/nginx/main-includes/mgmt.conf")) - mgmtConf := string(files[8].Content) + g.Expect(files[8].Meta.Name).To(Equal("/etc/nginx/main-includes/mgmt.conf")) + mgmtConf := string(files[8].Contents) g.Expect(mgmtConf).To(ContainSubstring("usage_report endpoint=test-endpoint")) g.Expect(mgmtConf).To(ContainSubstring("license_token /etc/nginx/secrets/license.jwt")) g.Expect(mgmtConf).To(ContainSubstring("deployment_context /etc/nginx/main-includes/deployment_ctx.json")) @@ -223,31 +226,34 @@ func TestGenerate(t *testing.T) { g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate /etc/nginx/secrets/mgmt-tls.crt")) g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate_key /etc/nginx/secrets/mgmt-tls.key")) - g.Expect(files[9].Path).To(Equal("/etc/nginx/secrets/license.jwt")) - g.Expect(string(files[9].Content)).To(Equal("license")) + g.Expect(files[9].Meta.Name).To(Equal("/etc/nginx/secrets/license.jwt")) + g.Expect(string(files[9].Contents)).To(Equal("license")) - g.Expect(files[10].Path).To(Equal("/etc/nginx/secrets/mgmt-ca.crt")) - g.Expect(string(files[10].Content)).To(Equal("ca")) + g.Expect(files[10].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-ca.crt")) + g.Expect(string(files[10].Contents)).To(Equal("ca")) - g.Expect(files[11].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.crt")) - g.Expect(string(files[11].Content)).To(Equal("cert")) + g.Expect(files[11].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.crt")) + g.Expect(string(files[11].Contents)).To(Equal("cert")) - g.Expect(files[12].Path).To(Equal("/etc/nginx/secrets/mgmt-tls.key")) - g.Expect(string(files[12].Content)).To(Equal("key")) + g.Expect(files[12].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.key")) + g.Expect(string(files[12].Contents)).To(Equal("key")) - g.Expect(files[13].Path).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) - certBundle := string(files[13].Content) + g.Expect(files[13].Meta.Name).To(Equal("/etc/nginx/secrets/test-certbundle.crt")) + certBundle := string(files[13].Contents) g.Expect(certBundle).To(Equal("test-cert")) - g.Expect(files[14]).To(Equal(file.File{ - Type: file.TypeSecret, - Path: "/etc/nginx/secrets/test-keypair.pem", - Content: []byte("test-cert\ntest-key"), + g.Expect(files[14]).To(Equal(agent.File{ + Meta: &pb.FileMeta{ + Name: "/etc/nginx/secrets/test-keypair.pem", + Hash: filesHelper.GenerateHash([]byte("test-cert\ntest-key")), + Permissions: file.SecretFileMode, + }, + Contents: []byte("test-cert\ntest-key"), })) - g.Expect(files[15].Path).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) - g.Expect(files[15].Type).To(Equal(file.TypeRegular)) - streamCfg := string(files[15].Content) + g.Expect(files[15].Meta.Name).To(Equal("/etc/nginx/stream-conf.d/stream.conf")) + g.Expect(files[15].Meta.Permissions).To(Equal(file.RegularFileMode)) + streamCfg := string(files[15].Contents) g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock")) g.Expect(streamCfg).To(ContainSubstring("listen 443")) g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock")) diff --git a/internal/mode/static/nginx/config/main_config.go b/internal/mode/static/nginx/config/main_config.go index bd6f2256f..0d75cea1f 100644 --- a/internal/mode/static/nginx/config/main_config.go +++ b/internal/mode/static/nginx/config/main_config.go @@ -3,8 +3,12 @@ package config import ( gotemplate "text/template" + pb "github.com/nginx/agent/v3/api/grpc/mpi/v1" + filesHelper "github.com/nginx/agent/v3/pkg/files" + "github.com/nginxinc/nginx-gateway-fabric/internal/framework/file" "github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/agent" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/shared" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -50,7 +54,7 @@ type mgmtConf struct { // generateMgmtFiles generates the NGINX Plus configuration file for the mgmt block. As part of this, // it writes the secret and deployment context files that are referenced in the mgmt block. -func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.File { +func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []agent.File { if !g.plus { return nil } @@ -60,47 +64,59 @@ func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.Fi panic("nginx plus token not set in expected map") } - tokenFile := file.File{ - Content: tokenContent, - Path: secretsFolder + "/license.jwt", - Type: file.TypeSecret, + tokenFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/license.jwt", + Hash: filesHelper.GenerateHash(tokenContent), + Permissions: file.SecretFileMode, + }, + Contents: tokenContent, } - files := []file.File{tokenFile} + files := []agent.File{tokenFile} cfg := mgmtConf{ Endpoint: g.usageReportConfig.Endpoint, Resolver: g.usageReportConfig.Resolver, - LicenseTokenFile: tokenFile.Path, + LicenseTokenFile: tokenFile.Meta.Name, SkipVerify: g.usageReportConfig.SkipVerify, } if content, ok := conf.AuxiliarySecrets[graph.PlusReportCACertificate]; ok { - caFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-ca.crt", - Type: file.TypeSecret, + caFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-ca.crt", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.CACertFile = caFile.Path + cfg.CACertFile = caFile.Meta.Name files = append(files, caFile) } if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLCertificate]; ok { - certFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-tls.crt", - Type: file.TypeSecret, + certFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-tls.crt", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.ClientSSLCertFile = certFile.Path + cfg.ClientSSLCertFile = certFile.Meta.Name files = append(files, certFile) } if content, ok := conf.AuxiliarySecrets[graph.PlusReportClientSSLKey]; ok { - keyFile := file.File{ - Content: content, - Path: secretsFolder + "/mgmt-tls.key", - Type: file.TypeSecret, + keyFile := agent.File{ + Meta: &pb.FileMeta{ + Name: secretsFolder + "/mgmt-tls.key", + Hash: filesHelper.GenerateHash(content), + Permissions: file.SecretFileMode, + }, + Contents: content, } - cfg.ClientSSLKeyFile = keyFile.Path + cfg.ClientSSLKeyFile = keyFile.Meta.Name files = append(files, keyFile) } @@ -111,10 +127,14 @@ func (g GeneratorImpl) generateMgmtFiles(conf dataplane.Configuration) []file.Fi files = append(files, deploymentCtxFile) } - mgmtBlockFile := file.File{ - Content: helpers.MustExecuteTemplate(mgmtConfigTemplate, cfg), - Path: mgmtIncludesFile, - Type: file.TypeRegular, + mgmtContents := helpers.MustExecuteTemplate(mgmtConfigTemplate, cfg) + mgmtBlockFile := agent.File{ + Meta: &pb.FileMeta{ + Name: mgmtIncludesFile, + Hash: filesHelper.GenerateHash(mgmtContents), + Permissions: file.RegularFileMode, + }, + Contents: mgmtContents, } return append(files, mgmtBlockFile) diff --git a/internal/mode/static/state/conditions/conditions.go b/internal/mode/static/state/conditions/conditions.go index c97f9efa2..1d79ed348 100644 --- a/internal/mode/static/state/conditions/conditions.go +++ b/internal/mode/static/state/conditions/conditions.go @@ -19,7 +19,7 @@ const ( // ListenerMessageFailedNginxReload is a message used with ListenerConditionProgrammed (false) // when nginx fails to reload. ListenerMessageFailedNginxReload = "The Listener is not programmed due to a failure to " + - "reload nginx with the configuration. Please see the nginx container logs for any possible configuration issues." + "reload nginx with the configuration" // RouteReasonBackendRefUnsupportedValue is used with the "ResolvedRefs" condition when one of the // Route rules has a backendRef with an unsupported value. @@ -68,7 +68,7 @@ const ( // GatewayMessageFailedNginxReload is a message used with GatewayConditionProgrammed (false) // when nginx fails to reload. GatewayMessageFailedNginxReload = "The Gateway is not programmed due to a failure to " + - "reload nginx with the configuration. Please see the nginx container logs for any possible configuration issues" + "reload nginx with the configuration" // RouteMessageFailedNginxReload is a message used with RouteReasonGatewayNotProgrammed // when nginx fails to reload. diff --git a/internal/mode/static/status/prepare_requests.go b/internal/mode/static/status/prepare_requests.go index e0add956a..8bcf48cb9 100644 --- a/internal/mode/static/status/prepare_requests.go +++ b/internal/mode/static/status/prepare_requests.go @@ -272,9 +272,10 @@ func prepareGatewayRequest( } if nginxReloadRes.Error != nil { + msg := fmt.Sprintf("%s: %s", staticConds.ListenerMessageFailedNginxReload, nginxReloadRes.Error.Error()) conds = append( conds, - staticConds.NewListenerNotProgrammedInvalid(staticConds.ListenerMessageFailedNginxReload), + staticConds.NewListenerNotProgrammedInvalid(msg), ) } @@ -300,9 +301,10 @@ func prepareGatewayRequest( } if nginxReloadRes.Error != nil { + msg := fmt.Sprintf("%s: %s", staticConds.GatewayMessageFailedNginxReload, nginxReloadRes.Error.Error()) gwConds = append( gwConds, - staticConds.NewGatewayNotProgrammedInvalid(staticConds.GatewayMessageFailedNginxReload), + staticConds.NewGatewayNotProgrammedInvalid(msg), ) } diff --git a/internal/mode/static/status/prepare_requests_test.go b/internal/mode/static/status/prepare_requests_test.go index d52b43e7a..77261d1ef 100644 --- a/internal/mode/static/status/prepare_requests_test.go +++ b/internal/mode/static/status/prepare_requests_test.go @@ -3,6 +3,7 @@ package status import ( "context" "errors" + "fmt" "testing" . "github.com/onsi/gomega" @@ -1087,7 +1088,7 @@ func TestBuildGatewayStatuses(t *testing.T) { ObservedGeneration: 2, LastTransitionTime: transitionTime, Reason: string(v1.GatewayReasonInvalid), - Message: staticConds.GatewayMessageFailedNginxReload, + Message: fmt.Sprintf("%s: test error", staticConds.GatewayMessageFailedNginxReload), }, }, Listeners: []v1.ListenerStatus{ @@ -1125,7 +1126,7 @@ func TestBuildGatewayStatuses(t *testing.T) { ObservedGeneration: 2, LastTransitionTime: transitionTime, Reason: string(v1.ListenerReasonInvalid), - Message: staticConds.ListenerMessageFailedNginxReload, + Message: fmt.Sprintf("%s: test error", staticConds.ListenerMessageFailedNginxReload), }, }, }, diff --git a/site/content/how-to/monitoring/troubleshooting.md b/site/content/how-to/monitoring/troubleshooting.md index 5c8cc89e4..67a698187 100644 --- a/site/content/how-to/monitoring/troubleshooting.md +++ b/site/content/how-to/monitoring/troubleshooting.md @@ -354,17 +354,6 @@ Events: Normal Started 39s kubelet Started container nginx ``` -##### Insufficient Privileges errors - -Depending on your environment's configuration, the control plane may not have the proper permissions to reload NGINX. The NGINX configuration will not be applied and you will see the following error in the _nginx-gateway_ logs: - -`failed to reload NGINX: failed to send the HUP signal to NGINX main: operation not permitted` - -To **resolve** this issue you will need to set `allowPrivilegeEscalation` to `true`. - -- If using Helm, you can set the `nginxGateway.securityContext.allowPrivilegeEscalation` value. -- If using the manifests directly, you can update this field under the `nginx-gateway` container's `securityContext`. - ##### NGINX Plus failure to start or traffic interruptions Beginning with NGINX Gateway Fabric 1.5.0, NGINX Plus requires a valid JSON Web Token (JWT) to run. If this is not set up properly, or your JWT token has expired, you may see errors in the NGINX logs that look like the following: