diff --git a/Makefile b/Makefile index 591cb15..9e4bbb4 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ OPERATOR_SDK_VERSION ?= v1.38.0 # Image URL to use all building/pushing image targets DOCKER_HUB_NAME ?= $(shell docker info | sed '/Username:/!d;s/.* //') IMG_NAME ?= typesense-operator -IMG_TAG ?= 0.2.0-rc.0 +IMG_TAG ?= 0.2.1 IMG ?= $(DOCKER_HUB_NAME)/$(IMG_NAME):$(IMG_TAG) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. @@ -200,11 +200,11 @@ undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/. .PHONY: deploy-with-samples deploy-with-samples: kustomize generate manifests install ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/samples | $(KUBECTL) apply -f - + $(KUSTOMIZE) build config/samples | $(KUBECTL) apply -f config/samples/ts_v1alpha1_typesensecluster_kind.yaml .PHONY: samples samples: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/samples | $(KUBECTL) apply -f - + $(KUSTOMIZE) build config/samples | $(KUBECTL) apply -f config/samples/ts_v1alpha1_typesensecluster_kind.yaml ##@ Dependencies diff --git a/README.md b/README.md index 6fb0d40..0dafc9f 100644 --- a/README.md +++ b/README.md @@ -19,62 +19,33 @@ Key features of Typesense Kubernetes Operator include: - Continuous active (re)discovery of the quorum configuration reacting to changes in `ReplicaSet` **without the need of an additional sidecar container**, - Automatic recovery of a cluster that has lost quorum **without the need of manual intervention**. -### Custom Resource Definition +## Background -Typesense Kubernetes Operator is controlling the lifecycle of multiple Typesense instances in the same Kubernetes cluster by -introducing `TypesenseCluster`, a new Custom Resource Definition: - -![image](https://github.com/user-attachments/assets/23e40781-ca21-4297-93bf-2b5dbebc7e0e) - -The _specification_ of the CRD includes the following properties: - -- `image`: the Typesense docker image to use, required -- `replicas`: the size of the cluster, defaults to `1` -- `apiPort`: the REST/API port, defaults to `8108` -- `peeringPort`: the peering port, defaults to `8107` -- `resetPeersOnError`: whether to reset nodes in error state or not, defaults to `true` -- `corsDomains`: domains that would be allowed for CORS calls, optional. -- `storage.size`: the size of the underlying `PersistentVolume`, defaults to `100Mi` -- `storage.storageClassName`: the storage class to use, defaults to `standard` - -The _status_ of the CRD includes a single property, `condition`, of type `[]metav1.Condition`. There is actually only one -condition, `ConditionReady`, which steers the whole reconciliation process and results to `true` or `false` by evaluating -the aggregated health of the cluster. There are 5 different condition reasons that can lead to a ready or not ready condition of -the CRD: - -- `QuorumReady`: the Typesense cluster is provisioned and fully **operational** -- `QuorumNotReady`: the Typesense cluster is provisioned but **not operational** -- `QuorumDegraded`: the Typesense cluster is provisioned but **not operational**, and scheduled for downgrade to single instance -- `QuorumUpgraded`: the Typesense cluster is provisioned and fully **operational** as a single instance and scheduled for an upgrade to desired replicas -- `QuorumNeedsIntervention`: the Typesense cluster is provisioned but **not operational**, the allocated memory or disk are not sufficient and administrative intervention is required - -### Background - -Typesense is using raft in the background to establish its clusters. Raft is a consensus algorithm based on the +Typesense is using raft in the background to establish its clusters. Raft is a consensus algorithm based on the paper "[Raft: In Search of an Understandable Consensus Algorithm](https://raft.github.io/raft.pdf)". -Raft nodes operate in one of three possible states: _follower_, _candidate_, or _leader_. Every new node always joins the -quorum as a follower. Followers can receive log entries from the leader and participate in voting for electing a leader. If no -log entries are received for a specified period of time, a follower transitions to the candidate state. As a candidate, the node -can accept votes from its peers nodes. Upon receiving a majority of votes, the candidate is becoming the leader of the quorum. -The leader’s responsibilities include handling new log entries and replicating them to other nodes. +Raft nodes operate in one of three possible states: _follower_, _candidate_, or _leader_. Every new node always joins the +quorum as a follower. Followers can receive log entries from the leader and participate in voting for electing a leader. If no +log entries are received for a specified period of time, a follower transitions to the candidate state. As a candidate, the node +can accept votes from its peers nodes. Upon receiving a majority of votes, the candidate is becoming the leader of the quorum. +The leader’s responsibilities include handling new log entries and replicating them to other nodes. -Another thing to consider is what happens when the node set changes, when nodes join or leave the cluster. -If a quorum of nodes is **available**, raft can dynamically modify the node set without any issue (this happens every 30sec). -But if the cluster cannot form a quorum, then problems start to appear or better to pile up. A cluster with `N` nodes can tolerate +Another thing to consider is what happens when the node set changes, when nodes join or leave the cluster. +If a quorum of nodes is **available**, raft can dynamically modify the node set without any issue (this happens every 30sec). +But if the cluster cannot form a quorum, then problems start to appear or better to pile up. A cluster with `N` nodes can tolerate a failure of at most `(N-1)/2` nodes without losing its quorum. If the available nodes go below this threshold then two events are taking place: - raft declares the whole cluster as **unavailable** (no leader can be elected, no more log entries can be processed) - the remaining nodes are restarted in bootstrap mode -In a Kubernetes environment, the nodes are actually `Pods` which are rather volatile by nature and their lifetime is quite ephemeral and subjects -to potential restarts, and that puts the whole concept of raft protocol consensus under a tough spot. As we can read in the official +In a Kubernetes environment, the nodes are actually `Pods` which are rather volatile by nature and their lifetime is quite ephemeral and subjects +to potential restarts, and that puts the whole concept of raft protocol consensus under a tough spot. As we can read in the official documentation of Typesense when it comes to [recovering a cluster that has lost quorum](https://typesense.org/docs/guide/high-availability.html#recovering-a-cluster-that-has-lost-quorum), it is explicitly stated: -> If a Typesense cluster loses more than `(N-1)/2` nodes at the same time, the cluster becomes unstable because it loses quorum -and the remaining node(s) cannot safely build consensus on which node is the leader. To avoid a potential split brain issue, +> If a Typesense cluster loses more than `(N-1)/2` nodes at the same time, the cluster becomes unstable because it loses quorum +and the remaining node(s) cannot safely build consensus on which node is the leader. To avoid a potential split brain issue, Typesense then stops accepting writes and reads **until some manual verification and intervention is done**. ![image](https://github.com/user-attachments/assets/a28357bf-199f-45e7-9ce4-9557043bfc20) @@ -82,7 +53,7 @@ Typesense then stops accepting writes and reads **until some manual verification > [!NOTE] > Illustration's been taken from [Free Gophers Pack](https://github.com/MariaLetta/free-gophers-pack) -In production environments, manual intervention is sometimes impossible or undesirable, and downtime for a service like +In production environments, manual intervention is sometimes impossible or undesirable, and downtime for a service like Typesense may be unacceptable. The Typesense Kubernetes Operator addresses both of these challenges. ### Problem 1: Quorum reconfiguration @@ -90,37 +61,37 @@ Typesense may be unacceptable. The Typesense Kubernetes Operator addresses both The Typesense Kubernetes Operator manages the entire lifecycle of Typesense Clusters within Kubernetes: 1. A random token is generated and stored as a base64-encoded value in a new `Secret`. This token serves as the Admin API key for bootstrapping the Typesense cluster. -2. A `ConfigMap` is created, containing the endpoints of the cluster nodes as a single concatenated string in its `data` field. -During each reconciliation loop, the operator identifies any changes in endpoints and updates the `ConfigMap`. This `ConfigMap` -is mounted in every `Pod` at the path where raft expects the quorum configuration, ensuring quorum configuration stays always updated. -The Fully Qualified Domain Name (FQDN) for each endpoint of the headless service adheres to the following naming convention: +2. A `ConfigMap` is created, containing the endpoints of the cluster nodes as a single concatenated string in its `data` field. + During each reconciliation loop, the operator identifies any changes in endpoints and updates the `ConfigMap`. This `ConfigMap` + is mounted in every `Pod` at the path where raft expects the quorum configuration, ensuring quorum configuration stays always updated. + The Fully Qualified Domain Name (FQDN) for each endpoint of the headless service adheres to the following naming convention: `{cluster-name}-sts-{pod-index}.{cluster-name}-sts-svc.{namespace}.svc.cluster.local:{peering-port}:{api-port}` > [!IMPORTANT] -> **This completely eliminates the need for a sidecar** to translate the endpoints of the headless `Service` into `Pod` IP addresses. -> The FQDN of the endpoints automatically resolves to the new IP addresses, and raft will begin contacting these endpoints +> **This completely eliminates the need for a sidecar** to translate the endpoints of the headless `Service` into `Pod` IP addresses. +> The FQDN of the endpoints automatically resolves to the new IP addresses, and raft will begin contacting these endpoints > within its 30-second polling interval. -3. Next, the reconciler creates a headless `Service` required for the `StatefulSet`, along with a standard Kubernetes -Service of type `ClusterIP`. The latter exposes the REST/API endpoints of the Typesense cluster to external systems. +3. Next, the reconciler creates a headless `Service` required for the `StatefulSet`, along with a standard Kubernetes + Service of type `ClusterIP`. The latter exposes the REST/API endpoints of the Typesense cluster to external systems. 4. A `StatefulSet` is then created. The quorum configuration stored in the `ConfigMap` is mounted as a volume in each `Pod` -under `/usr/share/typesense/nodelist`. No `Pod` restart is necessary when the `ConfigMap` changes, as raft automatically -detects and applies the updates. + under `/usr/share/typesense/nodelist`. No `Pod` restart is necessary when the `ConfigMap` changes, as raft automatically + detects and applies the updates. ![image](https://github.com/user-attachments/assets/30b6989c-c872-46ef-8ece-86c5d4911667) > [!NOTE] -> The interval between reconciliation loops depends on the number of nodes. This approach ensures raft has sufficient -> "breathing room" to carry out its operations—such as leader election, log replication, and bootstrapping—before the +> The interval between reconciliation loops depends on the number of nodes. This approach ensures raft has sufficient +> "breathing room" to carry out its operations—such as leader election, log replication, and bootstrapping—before the > next quorum health reconciliation begins. 5. The controller assesses the quorum's health by probing each node at `http://{nodeUrl}:{api-port}/health`. Based on the -results, it formulates an action plan for the next reconciliation loop. This process is detailed in the following section: + results, it formulates an action plan for the next reconciliation loop. This process is detailed in the following section: ### Problem 2: Recovering a cluster that has lost quorum -During configuration changes, we cannot switch directly from the old configuration to the next, because conflicting +During configuration changes, we cannot switch directly from the old configuration to the next, because conflicting majorities could arise. When that happens, no leader can be elected and eventually raft declares the whole cluster as unavailable which leaves it in a hot loop. One way to solve it, is to force the cluster downgrade to a single instance cluster and then gradually introduce new nodes (by scaling up the `StatefulSet`). With that approach we avoid the need @@ -129,7 +100,7 @@ of manual intervention in order to recover a cluster that has lost quorum. ![image](https://github.com/user-attachments/assets/007852ba-e173-43a4-babf-d250f8a34ad1) > [!IMPORTANT] -> Scaling the `StatefulSet` down and subsequently up, would typically be the manual intervention needed to recover a cluster that has lost its quorum. +> Scaling the `StatefulSet` down and subsequently up, would typically be the manual intervention needed to recover a cluster that has lost its quorum. > **However**, the controller automates this process, as long as is not a memory or disk capacity issue, ensuring no service > interruption and **eliminating the need for any administration action**. @@ -137,37 +108,94 @@ of manual intervention in order to recover a cluster that has lost quorum. **Left Path:** -1. The quorum reconciler probes each cluster node at `http://{nodeUrl}:{api-port}/health`. If every node responds with `{ ok: true }`, - the `ConditionReady` status of the `TypesenseCluster` custom resource is updated to `QuorumReady`, indicating that the cluster is fully healthy and operational. +1. The quorum reconciler probes each cluster node at `http://{nodeUrl}:{api-port}/health`. If every node responds with `{ ok: true }`, + the `ConditionReady` status of the `TypesenseCluster` custom resource is updated to `QuorumReady`, indicating that the cluster is fully healthy and operational. 2. - - If the cluster size matches the desired size defined in the `TypesenseCluster` custom resource (and was not downgraded - during a previous loop—this scenario will be discussed later), the quorum reconciliation loop sets the `ConditionReady` - status of the `TypesenseCluster` custom resource to `QuorumReady`, exits, and hands control back to the main controller loop. - - If the cluster was downgraded to a single instance during a previous reconciliation loop, the quorum reconciliation loop - sets the `ConditionReady` status of the `TypesenseCluster` custom resource to `QuorumUpgraded`. It then returns control - to the main controller loop, which will attempt to restore the cluster to the desired size defined in the `TypesenseCluster` - custom resource during the next reconciliation loop. Raft will then identify the new quorum configuration and elect a new leader. - - If a node runs out of memory or disk, the health endpoint response will include an additional `resource_error` field, - set to either `OUT_OF_MEMORY` or `OUT_OF_DISK`, depending on the issue. In this case, the quorum reconciler marks the - `ConditionReady` status of the `TypesenseCluster` as `QuorumNeedsIntervention`, triggers a Kubernetes `Event`, and - returns control to the main controller loop. **In this scenario, manual intervention is required**. You must adjust the - resources in the `PodSpec` or the storage in the `PersistentVolumeClaim` of the `StatefulSet` to provide new memory limits - or increased storage size. This can be done by modifying and re-applying the corresponding `TypesenseCluster` manifest. + - If the cluster size matches the desired size defined in the `TypesenseCluster` custom resource (and was not downgraded + during a previous loop—this scenario will be discussed later), the quorum reconciliation loop sets the `ConditionReady` + status of the `TypesenseCluster` custom resource to `QuorumReady`, exits, and hands control back to the main controller loop. + - If the cluster was downgraded to a single instance during a previous reconciliation loop, the quorum reconciliation loop + sets the `ConditionReady` status of the `TypesenseCluster` custom resource to `QuorumUpgraded`. It then returns control + to the main controller loop, which will attempt to restore the cluster to the desired size defined in the `TypesenseCluster` + custom resource during the next reconciliation loop. Raft will then identify the new quorum configuration and elect a new leader. + - If a node runs out of memory or disk, the health endpoint response will include an additional `resource_error` field, + set to either `OUT_OF_MEMORY` or `OUT_OF_DISK`, depending on the issue. In this case, the quorum reconciler marks the + `ConditionReady` status of the `TypesenseCluster` as `QuorumNeedsIntervention`, triggers a Kubernetes `Event`, and + returns control to the main controller loop. **In this scenario, manual intervention is required**. You must adjust the + resources in the `PodSpec` or the storage in the `PersistentVolumeClaim` of the `StatefulSet` to provide new memory limits + or increased storage size. This can be done by modifying and re-applying the corresponding `TypesenseCluster` manifest. **Right Path:** -1. The quorum reconciler probes each node of the cluster at http://{nodeUrl}:{api-port}/health. - - If the required number of nodes (at least `(N-1)/2`) return `{ ok: true }`, the `ConditionReady` status of the - `TypesenseCluster` custom resource is set to `QuorumReady`, indicating that the cluster is healthy and operational, - **even if** some nodes are unavailable. Control is then returned to the main controller loop. - - If the required number of nodes (at least `(N-1)/2`) return `{ ok: false }`, the `ConditionReady` status of the - `TypesenseCluster` custom resource is set to `QuorumDowngrade`, marking the cluster as unhealthy. As part of the - mitigation plan, the cluster is scheduled for a downgrade to a single instance, with the intent to allow raft to automatically recover the quorum. - The quorum reconciliation loop then returns control to the main controller loop. - - In the next quorum reconciliation, the process will take the **Left Path**, that will eventually discover a healthy quorum, - nevertheless with the wrong amount of nodes; thing that will lead to setting the `ConditionReady` condition of the `TypesenseCluster` as `QuorumUpgraded`. - What happens next is already described in the **Left Path**. - +1. The quorum reconciler probes each node of the cluster at http://{nodeUrl}:{api-port}/health. + - If the required number of nodes (at least `(N-1)/2`) return `{ ok: true }`, the `ConditionReady` status of the + `TypesenseCluster` custom resource is set to `QuorumReady`, indicating that the cluster is healthy and operational, + **even if** some nodes are unavailable. Control is then returned to the main controller loop. + - If the required number of nodes (at least `(N-1)/2`) return `{ ok: false }`, the `ConditionReady` status of the + `TypesenseCluster` custom resource is set to `QuorumDowngrade`, marking the cluster as unhealthy. As part of the + mitigation plan, the cluster is scheduled for a downgrade to a single instance, with the intent to allow raft to automatically recover the quorum. + The quorum reconciliation loop then returns control to the main controller loop. + - In the next quorum reconciliation, the process will take the **Left Path**, that will eventually discover a healthy quorum, + nevertheless with the wrong amount of nodes; thing that will lead to setting the `ConditionReady` condition of the `TypesenseCluster` as `QuorumUpgraded`. + What happens next is already described in the **Left Path**. + +## Custom Resource Definitions + +### TypesenseCluster + +Typesense Kubernetes Operator is controlling the lifecycle of multiple Typesense instances in the same Kubernetes cluster by +introducing `TypesenseCluster`, a new Custom Resource Definition: + +![image](https://github.com/user-attachments/assets/23e40781-ca21-4297-93bf-2b5dbebc7e0e) + +**Spec** + +| Name | Description | Optional | Default | +|-------------------|----------------------------------------------|----------|---------| +| image | Typesense image | | | +| replicas | Size of the cluster | | 1 | +| apiPort | REST/API port | | 8108 | +| peeringPort | Peering port | | 8107 | +| resetPeersOnError | automatic reset of peers on error | | true | +| corsDomains | domains that would be allowed for CORS calls | X | | +| storage | check StorageSpec below | | | +| ingress | check IngressSpec below | X | | + +**StorageSpec** (optional) + +| Name | Description | Optional | Default | +|------------------|---------------------------|----------|----------| +| size | Size of the underlying PV | X | 100Mi | +| storageClassName | Storage Class to use | | standard | + +**IngressSpec** (optional) + +| Name | Description | Optional | Default | +|------------------|--------------------------------------|----------|---------| +| referer | FQDN allowed to access reverse proxy | X | | +| host | Ingress Host | | | +| clusterIssuer | cert-manager ClusterIssuer | | | +| ingressClassName | Ingress to use | | | +| annotations | User-Defined annotations | X | | + +> [!CAUTION] +> Although in Typesense documentation under _Production Best Practices_ -> _Configuration_ is stated: +> "_Typesense comes built-in with a high performance HTTP server (opens new window)that is used by likes of Fastly (opens new window)in +> their edge servers at scale. So Typesense can be directly exposed to incoming public-facing internet traffic, +> without the need to place it behind another web server like Nginx / Apache or your backend API._" it is highly recommended +> , from this operator's perspective, to always expose Typesense behind a reverse proxy (using the `referer` option). + + +**Status** + +| Condition | Value | Reason | Description | +|----------------|-------|-------------------------|------------------------------------------------------------| +| ConditionReady | true | QuorumReady | Cluster is Operational | +| | false | QuorumNotReady | Cluster is not Operational | +| | false | QuorumDegraded | Cluster is not Operational; Scheduled to Single-Instance | +| | false | QuorumUpgraded | Cluster is Operational; Scheduled to Original Size | +| | false | QuorumNeedsIntervention | Cluster is not Operational; Administrative Action Required | + ## Getting Started You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). diff --git a/api/v1alpha1/typesensecluster_types.go b/api/v1alpha1/typesensecluster_types.go index dfa3383..4c3ec57 100644 --- a/api/v1alpha1/typesensecluster_types.go +++ b/api/v1alpha1/typesensecluster_types.go @@ -55,6 +55,8 @@ type TypesenseClusterSpec struct { CorsDomains *string `json:"corsDomains,omitempty"` Storage *StorageSpec `json:"storage"` + + Ingress *IngressSpec `json:"ingress,omitempty"` } type StorageSpec struct { @@ -66,12 +68,20 @@ type StorageSpec struct { StorageClassName string `json:"storageClassName"` } -type CorsSpec struct { - +type IngressSpec struct { // +optional - // +kubebuilder:default=true - // +kubebuilder:validation:Type=boolean - Enabled bool `json:"enabled,omitempty"` + // +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$` + Referer *string `json:"referer,omitempty"` + + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$` + Host string `json:"host"` + + ClusterIssuer string `json:"clusterIssuer"` + + IngressClassName string `json:"ingressClassName"` + + Annotations map[string]string `json:"annotations,omitempty"` } // TypesenseClusterStatus defines the observed state of TypesenseCluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 40d7730..930cc31 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,16 +26,28 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CorsSpec) DeepCopyInto(out *CorsSpec) { +func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { *out = *in + if in.Referer != nil { + in, out := &in.Referer, &out.Referer + *out = new(string) + **out = **in + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CorsSpec. -func (in *CorsSpec) DeepCopy() *CorsSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec. +func (in *IngressSpec) DeepCopy() *IngressSpec { if in == nil { return nil } - out := new(CorsSpec) + out := new(IngressSpec) in.DeepCopyInto(out) return out } @@ -128,6 +140,11 @@ func (in *TypesenseClusterSpec) DeepCopyInto(out *TypesenseClusterSpec) { *out = new(StorageSpec) (*in).DeepCopyInto(*out) } + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TypesenseClusterSpec. diff --git a/charts/typesense-operator/Chart.yaml b/charts/typesense-operator/Chart.yaml index c7f6473..6eaaffd 100644 --- a/charts/typesense-operator/Chart.yaml +++ b/charts/typesense-operator/Chart.yaml @@ -13,9 +13,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.2.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.2.0-rc.0" +appVersion: "0.2.1" diff --git a/charts/typesense-operator/templates/deployment.yaml b/charts/typesense-operator/templates/deployment.yaml index b243f3b..30d1cac 100644 --- a/charts/typesense-operator/templates/deployment.yaml +++ b/charts/typesense-operator/templates/deployment.yaml @@ -20,9 +20,7 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} - - --zap-log-level={{ .Values.controllerManager.manager.logLevel }} + - args: {{- toYaml .Values.controllerManager.manager.args | nindent 8 }} command: - /manager env: diff --git a/charts/typesense-operator/templates/manager-rbac.yaml b/charts/typesense-operator/templates/manager-rbac.yaml index e48f4d2..a6c9c1a 100644 --- a/charts/typesense-operator/templates/manager-rbac.yaml +++ b/charts/typesense-operator/templates/manager-rbac.yaml @@ -11,6 +11,7 @@ rules: - configmaps verbs: - create + - delete - get - list - patch @@ -30,8 +31,11 @@ rules: - secrets verbs: - create + - delete - get - list + - patch + - update - watch - apiGroups: - "" @@ -39,8 +43,23 @@ rules: - services verbs: - create + - delete - get - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update - watch - apiGroups: - apps @@ -48,6 +67,7 @@ rules: - statefulsets verbs: - create + - delete - get - list - patch @@ -60,6 +80,18 @@ rules: verbs: - create - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - ts.opentelekomcloud.com resources: diff --git a/charts/typesense-operator/templates/typesensecluster-crd.yaml b/charts/typesense-operator/templates/typesensecluster-crd.yaml index b6e706e..6604c92 100644 --- a/charts/typesense-operator/templates/typesensecluster-crd.yaml +++ b/charts/typesense-operator/templates/typesensecluster-crd.yaml @@ -63,6 +63,27 @@ spec: type: string image: type: string + ingress: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterIssuer: + type: string + host: + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + ingressClassName: + type: string + referer: + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + required: + - clusterIssuer + - host + - ingressClassName + type: object peeringPort: default: 8107 type: integer diff --git a/charts/typesense-operator/values.yaml b/charts/typesense-operator/values.yaml index 030818e..0a22f0d 100644 --- a/charts/typesense-operator/values.yaml +++ b/charts/typesense-operator/values.yaml @@ -4,7 +4,7 @@ controllerManager: - --metrics-bind-address=:8443 - --leader-elect - --health-probe-bind-address=:8081 - logLevel: debug + - --zap-log-level=info containerSecurityContext: allowPrivilegeEscalation: false capabilities: @@ -12,7 +12,7 @@ controllerManager: - ALL image: repository: akyriako78/typesense-operator - tag: 0.2.0-rc.0 + tag: 0.2.1 resources: limits: cpu: 500m diff --git a/config/crd/bases/ts.opentelekomcloud.com_typesenseclusters.yaml b/config/crd/bases/ts.opentelekomcloud.com_typesenseclusters.yaml index 0c91d66..313ecfe 100644 --- a/config/crd/bases/ts.opentelekomcloud.com_typesenseclusters.yaml +++ b/config/crd/bases/ts.opentelekomcloud.com_typesenseclusters.yaml @@ -62,6 +62,27 @@ spec: type: string image: type: string + ingress: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterIssuer: + type: string + host: + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + ingressClassName: + type: string + referer: + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string + required: + - clusterIssuer + - host + - ingressClassName + type: object peeringPort: default: 8107 type: integer diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a8c80f5..f41e811 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,6 +10,7 @@ rules: - configmaps verbs: - create + - delete - get - list - patch @@ -29,8 +30,11 @@ rules: - secrets verbs: - create + - delete - get - list + - patch + - update - watch - apiGroups: - "" @@ -38,8 +42,23 @@ rules: - services verbs: - create + - delete - get - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update - watch - apiGroups: - apps @@ -47,6 +66,7 @@ rules: - statefulsets verbs: - create + - delete - get - list - patch @@ -59,6 +79,18 @@ rules: verbs: - create - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - ts.opentelekomcloud.com resources: diff --git a/config/samples/ts_v1alpha1_typesensecluster_aws.yaml b/config/samples/ts_v1alpha1_typesensecluster_aws.yaml index 3920d84..f894d6b 100644 --- a/config/samples/ts_v1alpha1_typesensecluster_aws.yaml +++ b/config/samples/ts_v1alpha1_typesensecluster_aws.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-1 + name: c-aws-1 spec: image: typesense/typesense:27.1 replicas: 3 @@ -18,7 +18,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-2 + name: c-aws-2 spec: image: typesense/typesense:27.1 replicas: 1 diff --git a/config/samples/ts_v1alpha1_typesensecluster_azure.yaml b/config/samples/ts_v1alpha1_typesensecluster_azure.yaml index 04d9020..b699514 100644 --- a/config/samples/ts_v1alpha1_typesensecluster_azure.yaml +++ b/config/samples/ts_v1alpha1_typesensecluster_azure.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-1 + name: c-az-1 spec: image: typesense/typesense:27.1 replicas: 3 @@ -18,7 +18,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-2 + name: c-az-2 spec: image: typesense/typesense:26.0 replicas: 1 diff --git a/config/samples/ts_v1alpha1_typesensecluster_bm.yaml b/config/samples/ts_v1alpha1_typesensecluster_bm.yaml index 634a315..34543f5 100644 --- a/config/samples/ts_v1alpha1_typesensecluster_bm.yaml +++ b/config/samples/ts_v1alpha1_typesensecluster_bm.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-1 + name: c-bm-1 spec: image: typesense/typesense:27.1 replicas: 3 @@ -18,7 +18,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-2 + name: c-bm-2 spec: image: typesense/typesense:27.1 replicas: 1 diff --git a/config/samples/ts_v1alpha1_typesensecluster_kind.yaml b/config/samples/ts_v1alpha1_typesensecluster_kind.yaml index 04d5002..b28fdd4 100644 --- a/config/samples/ts_v1alpha1_typesensecluster_kind.yaml +++ b/config/samples/ts_v1alpha1_typesensecluster_kind.yaml @@ -13,13 +13,18 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-1 + name: c-kind-1 spec: image: typesense/typesense:27.1 replicas: 3 storage: size: 10Mi storageClassName: typesense-local-path + ingress: + referer: referer.example.com + host: host.example.com + ingressClassName: nginx + clusterIssuer: lets-encrypt-prod --- apiVersion: ts.opentelekomcloud.com/v1alpha1 kind: TypesenseCluster @@ -27,9 +32,13 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-2 + name: c-kind-2 spec: image: typesense/typesense:26.0 replicas: 1 storage: - storageClassName: typesense-local-path \ No newline at end of file + storageClassName: typesense-local-path + ingress: + host: host.example.com + ingressClassName: nginx + clusterIssuer: lets-encrypt-prod \ No newline at end of file diff --git a/config/samples/ts_v1alpha1_typesensecluster_opentelekomcloud.yaml b/config/samples/ts_v1alpha1_typesensecluster_opentelekomcloud.yaml index 74bb1fd..f8fa31d 100644 --- a/config/samples/ts_v1alpha1_typesensecluster_opentelekomcloud.yaml +++ b/config/samples/ts_v1alpha1_typesensecluster_opentelekomcloud.yaml @@ -4,13 +4,18 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-1 + name: c-otc-1 spec: image: typesense/typesense:27.1 replicas: 3 storage: size: 10Mi storageClassName: csi-disk + ingress: + referer: ts.sydpaa.de + host: ts-test.sydpaa.de + ingressClassName: cce + clusterIssuer: opentelekomcloud-letsencrypt --- apiVersion: ts.opentelekomcloud.com/v1alpha1 kind: TypesenseCluster @@ -18,7 +23,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-2 + name: c-otc-2 spec: image: typesense/typesense:26.0 replicas: 1 @@ -31,7 +36,7 @@ metadata: labels: app.kubernetes.io/name: typesense-operator app.kubernetes.io/managed-by: kustomize - name: cluster-3 + name: c-otc-3 spec: image: typesense/typesense:26.0 replicas: 3 diff --git a/internal/controller/typesensecluster_condition_types.go b/internal/controller/typesensecluster_condition_types.go index 4707d0b..e88456b 100644 --- a/internal/controller/typesensecluster_condition_types.go +++ b/internal/controller/typesensecluster_condition_types.go @@ -17,6 +17,7 @@ const ( ConditionReasonSecretNotReady = "SecretNotReady" ConditionReasonConfigMapNotReady = "ConfigMapNotReady" ConditionReasonServicesNotReady = "ServicesNotReady" + ConditionReasonIngressNotReady = "IngressNotReady" ConditionReasonQuorumReady ConditionQuorum = "QuorumReady" ConditionReasonQuorumNotReady ConditionQuorum = "QuorumNotReady" ConditionReasonQuorumDowngraded ConditionQuorum = "QuorumDowngraded" diff --git a/internal/controller/typesensecluster_configmap.go b/internal/controller/typesensecluster_configmap.go index a7bda91..a84bd47 100644 --- a/internal/controller/typesensecluster_configmap.go +++ b/internal/controller/typesensecluster_configmap.go @@ -13,6 +13,8 @@ import ( ) func (r *TypesenseClusterReconciler) ReconcileConfigMap(ctx context.Context, ts tsv1alpha1.TypesenseCluster) (updated *bool, err error) { + r.logger.V(debugLevel).Info("reconciling config map") + configMapName := fmt.Sprintf("%s-nodeslist", ts.Name) configMapExists := true configMapObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: configMapName} @@ -59,7 +61,7 @@ func (r *TypesenseClusterReconciler) createConfigMap(ctx context.Context, key cl } cm := &v1.ConfigMap{ - ObjectMeta: getObjectMeta(ts, &key.Name), + ObjectMeta: getObjectMeta(ts, &key.Name, nil), Data: map[string]string{ "nodes": strings.Join(nodes, ","), }, diff --git a/internal/controller/typesensecluster_controller.go b/internal/controller/typesensecluster_controller.go index efd1867..328a50f 100644 --- a/internal/controller/typesensecluster_controller.go +++ b/internal/controller/typesensecluster_controller.go @@ -69,12 +69,14 @@ var ( // +kubebuilder:rbac:groups=ts.opentelekomcloud.com,resources=typesenseclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=ts.opentelekomcloud.com,resources=typesenseclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ts.opentelekomcloud.com,resources=typesenseclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create -// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch -// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -126,6 +128,15 @@ func (r *TypesenseClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, err } + err = r.ReconcileIngress(ctx, ts) + if err != nil { + cerr := r.setConditionNotReady(ctx, &ts, ConditionReasonIngressNotReady, err) + if cerr != nil { + err = errors.Wrap(err, cerr.Error()) + } + return ctrl.Result{}, err + } + sts, err := r.ReconcileStatefulSet(ctx, ts) if err != nil { cerr := r.setConditionNotReady(ctx, &ts, ConditionReasonStatefulSetNotReady, err) diff --git a/internal/controller/typesensecluster_ingress.go b/internal/controller/typesensecluster_ingress.go new file mode 100644 index 0000000..fb488a3 --- /dev/null +++ b/internal/controller/typesensecluster_ingress.go @@ -0,0 +1,353 @@ +package controller + +import ( + "context" + "fmt" + tsv1alpha1 "github.com/akyriako/typesense-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "maps" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + conf = `events {} + http { + server { + listen 80; + + %s + + location / { + proxy_pass http://%s-svc:8108/; + proxy_pass_request_headers on; + } + } + }` + + referer = `valid_referers server_names %s; + if ($invalid_referer) { + return 403; + }` +) + +func (r *TypesenseClusterReconciler) ReconcileIngress(ctx context.Context, ts tsv1alpha1.TypesenseCluster) (err error) { + r.logger.V(debugLevel).Info("reconciling ingress") + + ingressName := fmt.Sprintf("%s-reverse-proxy", ts.Name) + ingressExists := true + ingressObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: ingressName} + + var ig = &networkingv1.Ingress{} + if err := r.Get(ctx, ingressObjectKey, ig); err != nil { + if apierrors.IsNotFound(err) { + ingressExists = false + } else { + r.logger.Error(err, fmt.Sprintf("unable to fetch ingress: %s", ingressName)) + } + } + + if ingressExists && ts.Spec.Ingress == nil { + return r.deleteIngress(ctx, ig) + } else if !ingressExists && ts.Spec.Ingress == nil { + return nil + } + + if !ingressExists { + r.logger.V(debugLevel).Info("creating ingress", "ingress", ingressObjectKey.Name) + + ig, err = r.createIngress(ctx, ingressObjectKey, &ts) + if err != nil { + r.logger.Error(err, "creating ingress failed", "ingress", ingressObjectKey.Name) + return err + } + } + + configMapName := fmt.Sprintf("%s-reverse-proxy-config", ts.Name) + configMapExists := true + configMapObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: configMapName} + + var cm = &v1.ConfigMap{} + if err := r.Get(ctx, configMapObjectKey, cm); err != nil { + if apierrors.IsNotFound(err) { + configMapExists = false + } else { + r.logger.Error(err, fmt.Sprintf("unable to fetch ingress config map: %s", configMapName)) + } + } + + if !configMapExists { + r.logger.V(debugLevel).Info("creating ingress config map", "configmap", configMapObjectKey.Name) + + _, err = r.createIngressConfigMap(ctx, configMapObjectKey, &ts, ig) + if err != nil { + r.logger.Error(err, "creating ingress config map failed", "configmap", configMapObjectKey.Name) + return err + } + } else { + r.logger.V(debugLevel).Info("updating ingress config map", "configmap", configMapObjectKey.Name) + + _, err = r.updateIngressConfigMap(ctx, cm, &ts) + if err != nil { + return err + } + } + + deploymentName := fmt.Sprintf("%s-reverse-proxy", ts.Name) + deploymentExists := true + deploymentObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: deploymentName} + + var deployment = &appsv1.Deployment{} + if err := r.Get(ctx, deploymentObjectKey, deployment); err != nil { + if apierrors.IsNotFound(err) { + deploymentExists = false + } else { + r.logger.Error(err, fmt.Sprintf("unable to fetch ingress reverse proxy deployment: %s", deploymentName)) + } + } + + if !deploymentExists { + r.logger.V(debugLevel).Info("creating ingress reverse proxy deployment", "deployment", deploymentObjectKey.Name) + + _, err = r.createIngressDeployment(ctx, deploymentObjectKey, &ts, ig) + if err != nil { + r.logger.Error(err, "creating ingress reverse proxy deployment failed", "deployment", deploymentObjectKey.Name) + return err + } + } + + serviceName := fmt.Sprintf("%s-reverse-proxy-svc", ts.Name) + serviceExists := true + serviceNameObjectKey := client.ObjectKey{Namespace: ts.Namespace, Name: serviceName} + + var service = &v1.Service{} + if err := r.Get(ctx, serviceNameObjectKey, service); err != nil { + if apierrors.IsNotFound(err) { + serviceExists = false + } else { + r.logger.Error(err, fmt.Sprintf("unable to fetch ingress reverse proxy service: %s", serviceName)) + } + } + + if !serviceExists { + r.logger.V(debugLevel).Info("creating ingress reverse proxy service", "service", serviceNameObjectKey.Name) + + _, err = r.createIngressService(ctx, serviceNameObjectKey, &ts, ig) + if err != nil { + r.logger.Error(err, "creating ingress reverse proxy service failed", "service", serviceNameObjectKey.Name) + return err + } + } + + return nil +} + +func (r *TypesenseClusterReconciler) createIngress(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster) (*networkingv1.Ingress, error) { + annotations := map[string]string{} + annotations["cert-manager.io/cluster-issuer"] = ts.Spec.Ingress.ClusterIssuer + + if ts.Spec.Ingress.Annotations != nil { + maps.Copy(annotations, ts.Spec.Ingress.Annotations) + } + + ingress := &networkingv1.Ingress{ + ObjectMeta: getObjectMeta(ts, &key.Name, annotations), + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To(ts.Spec.Ingress.IngressClassName), + TLS: []networkingv1.IngressTLS{ + { + Hosts: []string{ts.Spec.Ingress.Host}, + SecretName: fmt.Sprintf("%s-reverse-proxy-%s-certificate-tls", ts.Name, ts.Spec.Ingress.ClusterIssuer), + }, + }, + Rules: []networkingv1.IngressRule{ + { + Host: ts.Spec.Ingress.Host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: ptr.To[networkingv1.PathType](networkingv1.PathTypeImplementationSpecific), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: fmt.Sprintf("%s-reverse-proxy-svc", ts.Name), + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + err := ctrl.SetControllerReference(ts, ingress, r.Scheme) + if err != nil { + return nil, err + } + + err = r.Create(ctx, ingress) + if err != nil { + return nil, err + } + + return ingress, nil +} + +func (r *TypesenseClusterReconciler) deleteIngress(ctx context.Context, ig *networkingv1.Ingress) error { + err := r.Delete(ctx, ig) + if err != nil { + return err + } + + return nil +} + +func (r *TypesenseClusterReconciler) createIngressConfigMap(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster, ig *networkingv1.Ingress) (*v1.ConfigMap, error) { + icm := &v1.ConfigMap{ + ObjectMeta: getObjectMeta(ts, &key.Name, nil), + Data: map[string]string{ + "nginx.conf": r.getIngressNginxConf(ts), + }, + } + + err := ctrl.SetControllerReference(ig, icm, r.Scheme) + if err != nil { + return nil, err + } + + err = r.Create(ctx, icm) + if err != nil { + return nil, err + } + + return icm, nil +} + +func (r *TypesenseClusterReconciler) updateIngressConfigMap(ctx context.Context, cm *v1.ConfigMap, ts *tsv1alpha1.TypesenseCluster) (*v1.ConfigMap, error) { + desired := cm.DeepCopy() + desired.Data = map[string]string{ + "nginx.conf": r.getIngressNginxConf(ts), + } + + if cm.Data["nginx.conf"] != desired.Data["nginx.conf"] { + err := r.Update(ctx, desired) + if err != nil { + r.logger.Error(err, "updating ingress config map failed") + return nil, err + } + } + + return desired, nil +} + +func (r *TypesenseClusterReconciler) getIngressNginxConf(ts *tsv1alpha1.TypesenseCluster) string { + ref := "" + if ts.Spec.Ingress != nil && ts.Spec.Ingress.Referer != nil { + ref = fmt.Sprintf(referer, *ts.Spec.Ingress.Referer) + } + + return fmt.Sprintf(conf, ref, ts.Name) +} + +func (r *TypesenseClusterReconciler) createIngressDeployment(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster, ig *networkingv1.Ingress) (*appsv1.Deployment, error) { + deployment := &appsv1.Deployment{ + ObjectMeta: getObjectMeta(ts, &key.Name, nil), + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](3), + Selector: &metav1.LabelSelector{ + MatchLabels: getReverseProxyLabels(ts), + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: getReverseProxyLabels(ts), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: fmt.Sprintf("%s-reverse-proxy", ts.Name), + Image: "nginx:alpine", + Ports: []v1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "nginx-config", + MountPath: "/etc/nginx/nginx.conf", + SubPath: "nginx.conf", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "nginx-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: fmt.Sprintf("%s-reverse-proxy-config", ts.Name), + }, + }, + }, + }, + }, + }, + }, + }, + } + + err := ctrl.SetControllerReference(ig, deployment, r.Scheme) + if err != nil { + return nil, err + } + + err = r.Create(ctx, deployment) + if err != nil { + return nil, err + } + + return deployment, nil +} + +func (r *TypesenseClusterReconciler) createIngressService(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster, ig *networkingv1.Ingress) (*v1.Service, error) { + service := &v1.Service{ + ObjectMeta: getObjectMeta(ts, &key.Name, nil), + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + Selector: getReverseProxyLabels(ts), + Ports: []v1.ServicePort{ + { + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: int32(80)}, + Name: "http", + }, + }, + }, + } + + err := ctrl.SetControllerReference(ig, service, r.Scheme) + if err != nil { + return nil, err + } + + err = r.Create(ctx, service) + if err != nil { + return nil, err + } + + return service, nil +} diff --git a/internal/controller/typesensecluster_quorum.go b/internal/controller/typesensecluster_quorum.go index 1018e6f..1fcca90 100644 --- a/internal/controller/typesensecluster_quorum.go +++ b/internal/controller/typesensecluster_quorum.go @@ -104,7 +104,7 @@ func (r *TypesenseClusterReconciler) getQuorumHealth(ctx context.Context, ts *ts return ConditionReasonQuorumNotReady, 0, err } - err = r.scaleStatefulSet(ctx, sts, 1) + err = r.ScaleStatefulSet(ctx, sts, 1) if err != nil { return ConditionReasonQuorumNotReady, 0, err } @@ -122,7 +122,7 @@ func (r *TypesenseClusterReconciler) getQuorumHealth(ctx context.Context, ts *ts return ConditionReasonQuorumNotReady, 0, err } - err = r.scaleStatefulSet(ctx, sts, ts.Spec.Replicas) + err = r.ScaleStatefulSet(ctx, sts, ts.Spec.Replicas) if err != nil { return ConditionReasonQuorumNotReady, 0, err } @@ -133,19 +133,3 @@ func (r *TypesenseClusterReconciler) getQuorumHealth(ctx context.Context, ts *ts return ConditionReasonQuorumReady, healthyNodes, nil } - -func (r *TypesenseClusterReconciler) scaleStatefulSet(ctx context.Context, sts *appsv1.StatefulSet, desiredReplicas int32) error { - if sts.Spec.Replicas != nil && *sts.Spec.Replicas == desiredReplicas { - r.logger.V(debugLevel).Info("statefulset already scaled to desired replicas", "name", sts.Name, "replicas", desiredReplicas) - return nil - } - - desired := sts.DeepCopy() - desired.Spec.Replicas = &desiredReplicas - if err := r.Client.Update(ctx, desired); err != nil { - r.logger.Error(err, "updating stateful replicas failed", "name", desired.Name) - return err - } - - return nil -} diff --git a/internal/controller/typesensecluster_secret.go b/internal/controller/typesensecluster_secret.go index dd4a94d..3d1f635 100644 --- a/internal/controller/typesensecluster_secret.go +++ b/internal/controller/typesensecluster_secret.go @@ -13,9 +13,10 @@ import ( const adminApiKeyName = "typesense-api-key" func (r *TypesenseClusterReconciler) ReconcileSecret(ctx context.Context, ts tsv1alpha1.TypesenseCluster) error { + r.logger.V(debugLevel).Info("reconciling secret") + secretName := fmt.Sprintf("%s-admin-key", ts.Name) secretExists := true - r.logger.V(debugLevel).Info("reconciling api key") secretObjectKey := client.ObjectKey{ Namespace: ts.Namespace, @@ -55,7 +56,7 @@ func (r *TypesenseClusterReconciler) createAdminApiKey( } secret := &v1.Secret{ - ObjectMeta: getObjectMeta(ts, &secretObjectKey.Name), + ObjectMeta: getObjectMeta(ts, &secretObjectKey.Name, nil), Type: v1.SecretTypeOpaque, Data: map[string][]byte{ adminApiKeyName: []byte(token), diff --git a/internal/controller/typesensecluster_services.go b/internal/controller/typesensecluster_services.go index a5b22d1..a560262 100644 --- a/internal/controller/typesensecluster_services.go +++ b/internal/controller/typesensecluster_services.go @@ -65,7 +65,7 @@ func (r *TypesenseClusterReconciler) ReconcileServices(ctx context.Context, ts t func (r *TypesenseClusterReconciler) createHeadlessService(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster) (*v1.Service, error) { svc := &v1.Service{ - ObjectMeta: getObjectMeta(ts, &key.Name), + ObjectMeta: getObjectMeta(ts, &key.Name, nil), Spec: v1.ServiceSpec{ ClusterIP: v1.ClusterIPNone, PublishNotReadyAddresses: true, @@ -95,7 +95,7 @@ func (r *TypesenseClusterReconciler) createHeadlessService(ctx context.Context, func (r *TypesenseClusterReconciler) createService(ctx context.Context, key client.ObjectKey, ts *tsv1alpha1.TypesenseCluster) (*v1.Service, error) { svc := &v1.Service{ - ObjectMeta: getObjectMeta(ts, &key.Name), + ObjectMeta: getObjectMeta(ts, &key.Name, nil), Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, Selector: getLabels(ts), diff --git a/internal/controller/typesensecluster_statefulset.go b/internal/controller/typesensecluster_statefulset.go index 4e1d0a4..c1ba7f3 100644 --- a/internal/controller/typesensecluster_statefulset.go +++ b/internal/controller/typesensecluster_statefulset.go @@ -61,7 +61,7 @@ func (r *TypesenseClusterReconciler) createStatefulSet( ) (*appsv1.StatefulSet, error) { sts := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{}, - ObjectMeta: getObjectMeta(ts, &key.Name), + ObjectMeta: getObjectMeta(ts, &key.Name, nil), Spec: appsv1.StatefulSetSpec{ ServiceName: fmt.Sprintf("%s-sts-svc", ts.Name), PodManagementPolicy: appsv1.ParallelPodManagement, @@ -70,7 +70,7 @@ func (r *TypesenseClusterReconciler) createStatefulSet( MatchLabels: getLabels(ts), }, Template: corev1.PodTemplateSpec{ - ObjectMeta: getObjectMeta(ts, &key.Name), + ObjectMeta: getObjectMeta(ts, &key.Name, nil), Spec: corev1.PodSpec{ SecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To[int64](10000), @@ -215,3 +215,19 @@ func (r *TypesenseClusterReconciler) createStatefulSet( return sts, nil } + +func (r *TypesenseClusterReconciler) ScaleStatefulSet(ctx context.Context, sts *appsv1.StatefulSet, desiredReplicas int32) error { + if sts.Spec.Replicas != nil && *sts.Spec.Replicas == desiredReplicas { + r.logger.V(debugLevel).Info("statefulset already scaled to desired replicas", "name", sts.Name, "replicas", desiredReplicas) + return nil + } + + desired := sts.DeepCopy() + desired.Spec.Replicas = &desiredReplicas + if err := r.Client.Update(ctx, desired); err != nil { + r.logger.Error(err, "updating stateful replicas failed", "name", desired.Name) + return err + } + + return nil +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 9fb339e..752aed6 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -43,14 +43,21 @@ func getLabels(ts *tsv1alpha1.TypesenseCluster) map[string]string { } } -func getObjectMeta(ts *tsv1alpha1.TypesenseCluster, name *string) metav1.ObjectMeta { +func getReverseProxyLabels(ts *tsv1alpha1.TypesenseCluster) map[string]string { + return map[string]string{ + "app": fmt.Sprintf("%s-rp", ts.Name), + } +} + +func getObjectMeta(ts *tsv1alpha1.TypesenseCluster, name *string, annotations map[string]string) metav1.ObjectMeta { if name == nil { name = &ts.Name } return metav1.ObjectMeta{ - Name: *name, - Namespace: ts.Namespace, - Labels: getLabels(ts), + Name: *name, + Namespace: ts.Namespace, + Labels: getLabels(ts), + Annotations: annotations, } }