Successfully test collection of event_query.yml data (#15761)

* Callback plugin method from cmeyers adapted to global collection list

Get tests passing

Mild rebranding

Put behind feature flag, flip true in dev

Add noqa flag

* Add missing wait_for_events

awx/main/tasks/jobs.py

@@ -86,6 +86,9 @@
 from rest_framework.exceptions import PermissionDenied
 from django.utils.translation import gettext_lazy as _
 
+# Django flags
+from flags.state import flag_enabled
+
 logger = logging.getLogger('awx.main.tasks.jobs')
 
 
@@ -439,20 +442,17 @@ def final_run_hook(self, instance, status, private_data_dir):
         Hook for any steps to run after job/task is marked as complete.
         """
         instance.log_lifecycle("finalize_run")
-        artifact_dir = os.path.join(private_data_dir, 'artifacts', str(self.instance.id))
-        collections_info = os.path.join(artifact_dir, 'collections.json')
-        ansible_version_file = os.path.join(artifact_dir, 'ansible_version.txt')
-
-        if os.path.exists(collections_info):
-            with open(collections_info) as ee_json_info:
-                ee_collections_info = json.loads(ee_json_info.read())
-                instance.installed_collections = ee_collections_info
-                instance.save(update_fields=['installed_collections'])
-        if os.path.exists(ansible_version_file):
-            with open(ansible_version_file) as ee_ansible_info:
-                ansible_version_info = ee_ansible_info.readline()
-                instance.ansible_version = ansible_version_info
-                instance.save(update_fields=['ansible_version'])
+        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+            artifact_dir = os.path.join(private_data_dir, 'artifacts', str(self.instance.id))
+            data_file_path = os.path.join(artifact_dir, 'ansible_data.json')
+
+            if os.path.exists(data_file_path):
+                with open(data_file_path) as f:
+                    collected_data = json.loads(f.read())
+
+                instance.installed_collections = collected_data['installed_collections']
+                instance.ansible_version = collected_data['ansible_version']
+                instance.save(update_fields=['installed_collections', 'ansible_version'])
 
         # Run task manager appropriately for speculative dependencies
         if instance.unifiedjob_blocked_jobs.exists():
@@ -927,11 +927,16 @@ def build_env(self, job, private_data_dir, private_data_files=None):
             if authorize:
                 env['ANSIBLE_NET_AUTH_PASS'] = network_cred.get_input('authorize_password', default='')
 
-        path_vars = (
+        path_vars = [
             ('ANSIBLE_COLLECTIONS_PATHS', 'collections_paths', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
             ('ANSIBLE_ROLES_PATH', 'roles_path', 'requirements_roles', '~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles'),
             ('ANSIBLE_COLLECTIONS_PATH', 'collections_path', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
-        )
+        ]
+
+        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+            path_vars.append(
+                ('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'),
+            )
 
         config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)))
 
@@ -948,6 +953,11 @@ def build_env(self, job, private_data_dir, private_data_files=None):
             paths = [os.path.join(CONTAINER_ROOT, folder)] + paths
             env[env_key] = os.pathsep.join(paths)
 
+        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+            env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count'
+            if 'callbacks_enabled' in config_values:
+                env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled']
+
         return env
 
     def build_args(self, job, private_data_dir, passwords):
@@ -1388,6 +1398,17 @@ def make_local_copy(project, job_private_data_dir):
                 shutil.copytree(cache_subpath, dest_subpath, symlinks=True)
                 logger.debug('{0} {1} prepared {2} from cache'.format(type(project).__name__, project.pk, dest_subpath))
 
+        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+            # copy the special callback (not stdout type) plugin to get list of collections
+            pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path')
+            if not os.path.exists(pdd_plugins_path):
+                os.mkdir(pdd_plugins_path)
+            from awx.playbooks import library
+
+            plugin_file_source = os.path.join(library.__path__._path[0], 'indirect_instance_count.py')
+            plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py')
+            shutil.copyfile(plugin_file_source, plugin_file_dest)
+
     def post_run_hook(self, instance, status):
         super(RunProjectUpdate, self).post_run_hook(instance, status)
         # To avoid hangs, very important to release lock even if errors happen here

awx/main/tests/live/tests/projects/test_indirect_host_counting.py

@@ -1,14 +1,20 @@
+import yaml
+
+from awx.main.tests.live.tests.conftest import wait_for_events
+
 from awx.main.tasks.host_indirect import build_indirect_host_data
 from awx.main.models import Job
 
 
 def test_indirect_host_counting(live_tmp_folder, run_job_from_playbook):
     run_job_from_playbook('test_indirect_host_counting', 'run_task.yml', scm_url=f'file://{live_tmp_folder}/test_host_query')
     job = Job.objects.filter(name__icontains='test_indirect_host_counting').order_by('-created').first()
+    wait_for_events(job)  # We must wait for events because system tasks iterate on job.job_events.filter(...)
 
     # Data matches to awx/main/tests/data/projects/host_query/meta/event_query.yml
     # this just does things in-line to be a more localized test for the immediate testing
-    event_query = {'demo.query.example': '{canonical_facts: {host_name: .direct_host_name}, facts: {device_type: .device_type}}'}
+    module_jq_str = '{canonical_facts: {host_name: .direct_host_name}, facts: {device_type: .device_type}}'
+    event_query = {'demo.query.example': module_jq_str}
 
     # Run the task logic directly with local data
     results = build_indirect_host_data(job, event_query)
@@ -18,3 +24,12 @@ def test_indirect_host_counting(live_tmp_folder, run_job_from_playbook):
     # Asserts on data that will match to the input jq string from above
     assert host_audit_entry.canonical_facts == {'host_name': 'foo_host_default'}
     assert host_audit_entry.facts == {'device_type': 'Fake Host'}
+
+    # Test collection of data
+    assert 'demo.query' in job.installed_collections
+    assert 'host_query' in job.installed_collections['demo.query']
+    hq_text = job.installed_collections['demo.query']['host_query']
+    hq_data = yaml.safe_load(hq_text)
+    assert hq_data == {'demo.query.example': module_jq_str}
+
+    assert job.ansible_version

awx/playbooks/library/indirect_instance_count.py

@@ -0,0 +1,92 @@
+# (C) 2012, Michael DeHaan, <[email protected]>
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+    callback: host_query
+    type: notification
+    short_description: for demo of indirect host data and counting, this produces collection data
+    version_added: historical
+    description:
+      - Saves collection data to artifacts folder
+    requirements:
+     - Whitelist in configuration
+     - Set AWX_ISOLATED_DATA_DIR, AWX will do this
+'''
+
+import os
+import json
+
+from ansible.plugins.callback import CallbackBase
+
+
+# NOTE: in Ansible 1.2 or later general logging is available without
+# this plugin, just set ANSIBLE_LOG_PATH as an environment variable
+# or log_path in the DEFAULTS section of your ansible configuration
+# file.  This callback is an example of per hosts logging for those
+# that want it.
+
+
+# Taken from https://github.com/ansible/ansible/blob/devel/lib/ansible/cli/galaxy.py#L1624
+
+from ansible.cli.galaxy import with_collection_artifacts_manager
+from ansible.release import __version__
+
+from ansible.galaxy.collection import find_existing_collections
+from ansible.utils.collection_loader import AnsibleCollectionConfig
+import ansible.constants as C
+
+from ansible.module_utils.common.text.converters import to_text
+
+
+@with_collection_artifacts_manager
+def list_collections(artifacts_manager=None):
+    artifacts_manager.require_build_metadata = False
+
+    default_collections_path = set(C.COLLECTIONS_PATHS)
+    collections_search_paths = default_collections_path | set(AnsibleCollectionConfig.collection_paths)
+    collections = list(find_existing_collections(list(collections_search_paths), artifacts_manager, dedupe=False))
+    return collections
+
+
+class CallbackModule(CallbackBase):
+    """
+    logs playbook results, per host, in /var/log/ansible/hosts
+    """
+
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'notification'
+    CALLBACK_NAME = 'indirect_instance_count'
+    CALLBACK_NEEDS_WHITELIST = True
+
+    TIME_FORMAT = "%b %d %Y %H:%M:%S"
+    MSG_FORMAT = "%(now)s - %(category)s - %(data)s\n\n"
+
+    def v2_playbook_on_stats(self, stats):
+        artifact_dir = os.getenv('AWX_ISOLATED_DATA_DIR')
+        if not artifact_dir:
+            raise RuntimeError('Only suitable in AWX, did not find private_data_dir')
+
+        collections_print = {}
+        for collection_obj in list_collections():
+            collection_print = {
+                'version': collection_obj.ver,
+            }
+            host_query_path = os.path.join(to_text(collection_obj.src), 'meta', 'event_query.yml')
+            if os.path.exists(host_query_path):
+                with open(host_query_path, 'r') as f:
+                    collection_print['host_query'] = f.read()
+            collections_print[collection_obj.fqcn] = collection_print
+
+        ansible_data = {'installed_collections': collections_print, 'ansible_version': __version__}
+
+        write_path = os.path.join(artifact_dir, 'ansible_data.json')
+        with open(write_path, "w") as fd:
+            fd.write(json.dumps(ansible_data, indent=2))
+
+        super().v2_playbook_on_stats(stats)

awx/settings/development.py

@@ -67,6 +67,9 @@
 
 AWX_CALLBACK_PROFILE = True
 
+# this modifies FLAGS set by defaults
+FLAGS['FEATURE_INDIRECT_NODE_COUNTING_ENABLED'] = [{'condition': 'boolean', 'value': True}]  # noqa
+
 # ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
 # Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
 # Allows user to trigger task managers directly for debugging and profiling purposes.