KVStore Tools

roles/splunk/defaults/main.yml

@@ -30,6 +30,7 @@ splunk_general_key: undefined # Configures a pass4SymmKey in server.conf under t
 splunk_ds_key: undefined # Configures a pass4SymmKey in server.conf for authenticating against a deployment server
 splunk_admin_username: admin
 splunk_admin_password: undefined # Use ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name'
+splunk_authenticated: false # DO NOT CHANGE. This fact is set to true in the `splunk_login.yml` task, and reset to false if `splunk_restart.yml`, `splunk_stop.yml` or `splunk_start.yml` are manually called in another task.
 splunk_configure_secret: false # If set to true, you need to update files/splunk.secret
 splunk_secret_file: splunk.secret # Used to specify your splunk.secret filename(s), files should be placed in the "files" folder of the role
 # Although there are tasks for the following Splunk configurations in this role, they are not included in any tasks by default. You can add them to your install_splunk.yml if you would like to have Ansible manage any of these files
@@ -72,6 +73,10 @@ splunk_shc_target_group: shc
 splunk_shc_deployer: "{{ groups['shdeployer'] | first }}" # If you manage multiple SHCs, configure the var value in group_vars
 splunk_shc_uri_list: "{% for h in groups[splunk_shc_target_group] %}https://{{ hostvars[h].splunk_mgmt_uri }}:{{ splunkd_port }}{% if not loop.last %},{% endif %}{% endfor %}" # If you manage multiple SHCs, configure the var value in group_vars
 start_splunk_handler_fired: false # Do not change; used to prevent unnecessary splunk restarts
+splunk_enable_kvstore: true
+splunk_kvstore_storage: undefined # Can be defined here or at the group_vars level - accepted values: "wiredTiger" or "undefined", which leaves as default
+splunk_kvstore_version: undefined # Can be defined here or at the group_vars level - accepted values: 4.2 or "undefined", which leaves as default1
+splunk_oplog_size: 1000 # Default for Splunk Enterprise - should be changed at the group_vars level only at the behest of Splunk support with special care taken
 # Linux and scripting related vars
 add_crashlog_script: false # Set to true to install a script and cron job to automatically cleanup splunk crash logs older than 7 days
 add_diag_script: false # Set to true to install a script and cron job to automatically cleanup splunk diag files older than 30 days

roles/splunk/tasks/adhoc_backup_kvstore.yml

@@ -0,0 +1,28 @@
+---
+- name: Adhoc KVStore Backup
+  block:
+  - name: Check if authenticated
+    include_tasks: splunk_login.yml
+    when: not splunk_authenticated
+
+  - name: Check if we're okay to backup
+    ansible.builtin.command: |
+      {{ splunk_home }}/bin/splunk show kvstore-status | grep backupRestoreStatus | sed -r 's/\s+backupRestoreStatus : //g'
+    register: splunk_kvstore_pre_backup_status_out
+    until: "{{ splunk_kvstore_pre_backup_status_out.stdout }} == 'Ready'"
+
+  - name: Backup KVStore on desired host
+    ansible.builtin.command: |
+      {{ splunk_home }}/bin/splunk backup kvstore {{ archive_name | default("") }}
+    register: splunk_kvstore_backup_out
+    changed_when: splunk_kvstore_backup_out.rc == 0
+    failed_when: splunk_kvstore_backup_out.rc != 0
+
+  - name: Check that backup has finished
+    ansible.builtin.shell: |
+      {{ splunk_home }}/bin/splunk show kvstore-status | grep backupRestoreStatus | sed -r 's/\s+backupRestoreStatus : //g'
+    register: splunk_kvstore_status_out
+    until: "{{ splunk_kvstore_status_out.stdout }} == 'Ready'"
+
+  become: true
+  become_user: "{{ splunk_nix_user }}"

roles/splunk/tasks/adhoc_change_oplog_shc.yml

@@ -0,0 +1,116 @@
+---
+# oplog size should not be changed unless the oplog window is too small causing members to become stale - or is gradually shrinking.
+# Do NOT use this on a standalone instance - oplog size does not matter for a standalone KV Store.
+# Deployments should monitor the oplog window and react in time. If the window is already too small - KV Store may have to be re-crated with increased oplog size.
+
+- name: Check if authenticated
+  include_tasks: splunk_login.yml
+  when: not splunk_authenticated
+
+- name: Make sure we're in an SHC
+  ansible.builin.fail:
+    msg: "SHC not found in group_names - detected group names are \"{{ group_names }}\". This play will only run on an SHC"
+  when: splunk_shc_target_group not in group_names
+
+- name: Get current oplog size
+  ansible.builtin.shell: |
+    {{ splunk_home }}/bin/splunk btool server list kvstore | grep oplogSize | sed 's/[^0-9]*//g'
+  register: current_oplog_size_out
+  failed_when: current_oplog_size_out.rc != 0
+  run_once: true
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+
+- name: Debug current OpLog Size in MB
+  ansible.builtin.debug:
+    var: current_oplog_size_out.stdout
+    verbosity: 1
+
+- name: Make sure the oplog size var differs from our current value, if they're the same, exit play
+  ansible.builtin.meta: end_play
+  when: current_oplog_size_out == splunk_oplog_size
+
+# sets fact splunk_shc_captain
+- name: Find SHC Captain
+  include_tasks: get_shcluster_captain.yml
+
+# GUID from SPLUNK_HOME/etc/instance.cfg, not just hostname
+- name: Find KVStore Captain
+  block:
+    - name: Check if authenticated
+      include_tasks: splunk_login.yml
+      when: not splunk_authenticated
+
+    # Gets KVStore captain hostname - like splunk_captain.domain.com
+    # guid for GUID, hostAndPort for host with port - like `| sed -r 's/\s+hostAndPort : //g' | sed -r 's/:[0-9]+//g'`
+    - name: Get current KVStore captain
+      ansible.builtin.command: |
+        {{ splunk_home }}/bin/splunk show kvstore-status | grep -B10 "KV store captain" | grep "guid" | sed -r 's/\s+guid : //g'
+      become: true
+      become_user: "{{ splunk_nix_user }}"
+      register: splunk_get_kvcaptain
+      changed_when: false
+      failed_when: splunk_get_kvcaptain.rc != 0
+
+    - name: Register KVStore captain fact
+      ansible.builtin.set_fact:
+        splunk_kv_captain_guid: "{{ splunk_get_kvcaptain.stdout }}"
+
+- name: Make KVCaptain SHC captain
+  ansible.builtin.command: |
+    {{ splunk_home }}/bin/splunk transfer shcluster-captain -mgmt_uri "https://{{ splunk_shc_captain }}:{{ splunkd_port }}"
+  register: transfer_captain_out
+  changed_when: transfer_captain_out.rc == 0
+  failed_when: transfer_captain_out.rc != 0
+
+- name: Get current KVCaptain
+  include_tasks: get_kvstore_captain.yml
+
+- name: Ensure SHC Captain and KV Captain are the same
+  include_tasks: get_shcluster_captain.yml
+  until: "'{{ splunk_shc_captain }}' == '{{ splunk_kv_captain }}'"
+  delay: 10
+  retries: 30
+
+- name: Make a backup of the whole kvstore located in $SPLUNK_DB/kvstore only on one member
+  include_tasks: adhoc_backup_kvstore.yml
+  run_once: true # this works here to run the entire task only once, whereas import_tasks would run this on all hosts
+  vars:
+    archive_name: "{{ inventory_hostname }}-preoplog-backup"
+#- name: Make a backup of the whole KVStore directory ($SPLUNK_DB/kvstore) on only one member
+#  ansible.builtin.shell: |
+#    echo "{{ splunk_home }}/{{ inventory_hostname }}-preoplog-backup.tar.gz"; tar -czf {{ splunk_home }}/{{ inventory_hostname }}-preoplog-backup.tar.gz {{ splunk_db_path }}/kvstore
+#  register: kvstore_backup_out
+#  changed_when: kvstore_backup_out.rc == 0
+#  failed_when: kvstore_backup_out.rc != 1
+#  become: true
+#  run_once: true
+
+# Note - make this a separate task so that we can repeat it later for the final member
+- name: For each of the other SHC cluster members - increase the oplog
+  include_tasks: adhoc_increase_oplog_helper.yml
+  when: splunk_shc_captain != inventory_hostname
+
+- name: Select new node to be SHC captain
+  ansible.builtin.set_fact:
+    splunk_new_shc_captain: "{% for h in groups[splunk_shc_target_group] %}https://{{ hostvars[h].ansible_fqdn }}:{{ splunkd_port }}{% if not loop.last and hostvars[h].ansible_fqdn != splunk_shc_captain %},{% endif %}{% endfor %}" # If you manage multiple SHCs, configure the var value in group_vars
+
+
+- name: Transfer SHC captain to a different node
+  ansible.builtin.command: |
+    {{ splunk_home }}/bin/splunk transfer shcluster-captain -mgmt_uri "{{ splunk_new_shc_captain }}"
+  register: transfer_captain_out
+  changed_when: transfer_captain_out.rc == 0
+  failed_when: transfer_captain_out.rc != 0
+
+- name: Increase oplog on final member
+  include_tasks: adhoc_increase_oplog_helper.yml
+  when: splunk_shc_captain == inventory_hostname
+
+- name: Try to check data
+  ansible.builtin.debug:
+    msg:
+      - Check if the data is available - if something went wrong during the process
+      - use backup to restore the data. Backup is on this searchhead with this name={{ kvstore_backup_out.stdout }}
+      - If members are out of sync, resync the KVStore from the SHCluster captain {{ splunk_new_shc_captain }} 
+      - with `splunk resync kvstore -source {{ splunk_kv_captain_guid }}`

roles/splunk/tasks/adhoc_clean_kvstore.yml

@@ -0,0 +1,13 @@
+---
+- name: Stop Splunkd service
+  include_tasks: splunk_stop.yml
+
+- name: Clean KVStore
+  ansible.builtin.command: "{{ splunk_home }}/bin/splunk clean kvstore --local --answer-yes"
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+  register: clean_result
+  changed_when: clean_result.rc == 0
+  failed_when: clean_result.rc != 0
+  notify:
+    - start splunk

roles/splunk/tasks/adhoc_increase_oplog_helper.yml

@@ -0,0 +1,33 @@
+---
+- name: Stop Splunk
+  include_tasks: splunk_stop.yml
+
+- name: Clean KVStore
+  ansible.builtin.command: "{{ splunk_home }}/bin/splunk clean kvstore --local --answer-yes"
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+  register: clean_result
+  changed_when: clean_result.rc == 0
+  failed_when: clean_result.rc != 0
+
+- name: Edit server.conf to increase the oplogSize setting
+  community.general.ini_file:
+    path: "{{ splunk_home }}/etc/system/local/server.conf"
+    section: kvstore
+    option: oplogSize
+    value: "{{ splunk_oplog_size }}"
+    owner: "{{ splunk_nix_user }}"
+    group: "{{ splunk_nix_group }}"
+    mode: 0644
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+
+- name: Start Splunk to trigger synchronisation
+  include_tasks: splunk_start.yml
+
+# sets fact splunk_kvstore_status_json
+- name: Verify synchronisation with show kvstore-status
+  include_tasks: get_kvstore_status.yml
+  until: "'{{ splunk_kvstore_status_json.status }}' == 'ready'"
+  delay: 10
+  retries: 30

roles/splunk/tasks/configure_kvstore.yml

@@ -0,0 +1,28 @@
+---
+- name: Disable KVStore if specified
+  include_tasks: disable_kvstore.yml
+  when: not splunk_enable_kvstore
+
+- name: Configure initial KVStore storage engine in server.conf
+  community.general.ini_file:
+    path: "{{ splunk_home }}/etc/system/local/server.conf"
+    section: kvstore
+    option: storageEngine
+    value: "{{ splunk_kvstore_storage }}"
+    owner: "{{ splunk_nix_user }}"
+    group: "{{ splunk_nix_group }}"
+    mode: 0644
+  become: true
+  when:
+    - splunk_kvstore_storage == "wiredTiger"
+    - splunk_enable_kvstore
+
+- name: Configure initial KVStore oplog size in server.conf
+  community.general.ini_file:
+    path: "{{ splunk_home }}/etc/system/local/server.conf"
+    section: kvstore
+    option: oplogSize
+    value: "{{ splunk_oplog_size }}"
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+  when: splunk_enable_kvstore

roles/splunk/tasks/disable_kvstore.yml

@@ -1,10 +1,12 @@
 ---
 - name: Disable KVStore
-  when: ansible_system == "Linux"
+  when:
+    - ansible_system == "Linux"
+    - not splunk_enable_kvstore
   ini_file:
     path: "{{ splunk_home }}/etc/system/local/server.conf"
     section: kvstore
     option: disabled
     value: "true"
   become: True
-  become_user: "{{ splunk_nix_user }}"
+  become_user: "{{ splunk_nix_user }}"

roles/splunk/tasks/get_kvstore_captain.yml

@@ -0,0 +1,18 @@
+---
+- name: Check if authenticated
+  include_tasks: splunk_login.yml
+  when: not splunk_authenticated
+
+# Gets KVStore captain hostname - like splunk_captain.domain.com
+- name: Get current KVStore captain
+  ansible.builtin.command: |
+    {{ splunk_home }}/bin/splunk show kvstore-status | grep -B10 "KV store captain" | grep "hostAndPort" | sed -r 's/\s+hostAndPort : //g' | sed -r 's/:[0-9]+//g'
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+  register: splunk_get_kvcaptain
+  changed_when: false
+  failed_when: splunk_get_kvcaptain.rc != 0
+
+- name: Register KVStore captain fact
+  ansible.builtin.set_fact:
+    splunk_kv_captain: "{{ splunk_get_kvcaptain.stdout }}"

roles/splunk/tasks/get_kvstore_status.yml

@@ -0,0 +1,20 @@
+---
+# This file gets our KVStore status from our current member as JSON
+- name: Get KVStore status
+  ansible.builtin.shell: |
+    set -o pipefail
+    {{ splunk_home }}/bin/splunk show kvstore-status -auth {{ splunk_auth }} \
+      | grep -A13 "This member:" \
+      | tail -n +2 | sed -Er 's/^\s+//g' \
+      | awk -F ' * : *' '{ printf "\"%s\":\"%s\",", $1, $2 }' \
+      | sed 's/,$/}/' | sed 's/^/{/'
+  register: get_splunk_kvstore_status_out
+  failed_when: get_splunk_kvstore_status_out.rc != 0
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+
+- name: Convert KVStore status to JSON
+  ansible.builtin.set_fact:
+    splunk_kvstore_status_json: "{{ get_splunk_kvstore_status_out.stdout_lines[0] | from_json }}"
+
+# output: {"date":"Tue Jul 21 16:42:24 2016","dateSec":"1466541744.143000","disabled":"0","guid":"6244DF36-D883-4D59-AHD3-1354FCB4BL91","oplogEndTimestamp":"Tue Jul 21 16:41:12 2016","oplogEndTimestampSec":"1466541672.000000","oplogStartTimestamp":"Tue Jul 21 16:34:55 2016","oplogStartTimestampSec":"1466541295.000000","port":"8191","replicaSet":"splunkrs","replicationStatus":"KV store captain","standalone":"0","status":"ready"}

roles/splunk/tasks/get_shcluster_captain.yml

@@ -0,0 +1,18 @@
+---
+# Gets SHC captain management domain - like splunk_captain.example
+- name: Check if authenticated
+  include_tasks: splunk_login.yml
+  when: not splunk_authenticated
+
+- name: Get current SHCluster captain
+  ansible.builtin.command: |
+    {{ splunk_home }}/bin/splunk show shcluster-status | grep -A6 Captain | grep mgmt_uri | sed -r 's/\s+mgmt_uri : //g' | sed -Er 's/http(s)?:\/\///' | sed -Er 's/:[0-9]+//'
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+  register: splunk_get_shcaptain
+  changed_when: false
+  failed_when: splunk_get_shcaptain.rc != 0
+
+- name: Register SHCluster captain fact
+  ansible.builtin.set_fact:
+    splunk_shc_captain: "{{ splunk_get_shcaptain.stdout }}"

roles/splunk/tasks/get_shcluster_status.yml

@@ -0,0 +1,20 @@
+---
+# This file gets our shcluster status from our current member as JSON
+- name: Get shcluster status
+  ansible.builtin.shell: |
+    set -o pipefail
+    {{ splunk_home }}/bin/splunk show shcluster-status -auth {{ splunk_auth }} \
+      | grep -A5 "  {{ inventory_hostname }}" \
+      | tail -n +2 | sed -Er 's/^\s+//g' \
+      | awk -F ' * : *' '{ printf "\"%s\":\"%s\",", $1, $2 }' \
+      | sed 's/,$/}/' | sed 's/^/{/'
+  register: get_splunk_shcluster_status_out
+  failed_when: get_splunk_shcluster_status_out.rc != 0
+  become: true
+  become_user: "{{ splunk_nix_user }}"
+
+- name: Convert shcluster status to JSON
+  ansible.builtin.set_fact:
+    splunk_shcluster_status_json: "{{ get_splunk_shcluster_status_out.stdout_lines[0] | from_json }}"
+
+# output: {"label":"splunk-search.example.com","last_conf_replication":"Fri Mar 14 11:12:17 2024","mgmt_uri":"https://splunk-search.example.com:8089","mgmt_uri_alias":"https://10.1.1.5:8089","status":"Up"}