AAP-37282 Add parse JQ data and test it for a job object in isolati…

…on (#15774)

* Add jq dependency

* Add file in progress

* Add license for jq

* Write test and get it passing

awx/main/tasks/host_indirect.py

@@ -0,0 +1,28 @@
+from types import SimpleNamespace
+import logging
+
+import jq
+
+logger = logging.getLogger(__name__)
+
+
+def build_indirect_host_data(job, event_query: dict):
+    results = []
+    compiled_jq_expressions = {}  # Cache for compiled jq expressions
+    facts_missing_logged = False
+    for event in job.job_events.filter(task__in=event_query.keys()):
+        if 'res' not in event.event_data:
+            continue
+        jq_str_for_event = event_query[event.task]
+        if jq_str_for_event not in compiled_jq_expressions:
+            compiled_jq_expressions[event.task] = jq.compile(jq_str_for_event)
+        compiled_jq = compiled_jq_expressions[event.task]
+        for data in compiled_jq.input(event.event_data['res']).all():
+            if not data.get('canonical_facts'):
+                if not facts_missing_logged:
+                    logger.error(f'jq output missing canonical_facts for module {event.task} on event {event.id} using jq:{jq_str_for_event}')
+                continue
+            canonical_facts = data['canonical_facts']
+            facts = data.get('facts')
+            results.append(SimpleNamespace(canonical_facts=canonical_facts, facts=facts))
+    return results

awx/main/tests/data/projects/host_query/meta/event_query.yml

@@ -1,4 +1,4 @@
 ---
 {
-  "demo.query.example": ""
+  "demo.query.example": "{canonical_facts: {host_name: .direct_host_name}, facts: {device_type: .device_type}}"
 }

awx/main/tests/data/projects/host_query/plugins/modules/example.py

@@ -63,6 +63,9 @@ def run_module():
     result['direct_host_name'] = module.params['host_name']
     result['nested_host_name'] = {'host_name': module.params['host_name']}
 
+    # non-cononical facts
+    result['device_type'] = 'Fake Host'
+
     module.exit_json(**result)
 
 

awx/main/tests/live/tests/projects/test_indirect_host_counting.py

@@ -1,3 +1,20 @@
+from awx.main.tasks.host_indirect import build_indirect_host_data
+from awx.main.models import Job
+
+
 def test_indirect_host_counting(live_tmp_folder, run_job_from_playbook):
     run_job_from_playbook('test_indirect_host_counting', 'run_task.yml', scm_url=f'file://{live_tmp_folder}/test_host_query')
-    # TODO: add assertions that the host query data is populated
+    job = Job.objects.filter(name__icontains='test_indirect_host_counting').order_by('-created').first()
+
+    # Data matches to awx/main/tests/data/projects/host_query/meta/event_query.yml
+    # this just does things in-line to be a more localized test for the immediate testing
+    event_query = {'demo.query.example': '{canonical_facts: {host_name: .direct_host_name}, facts: {device_type: .device_type}}'}
+
+    # Run the task logic directly with local data
+    results = build_indirect_host_data(job, event_query)
+    assert len(results) == 1
+    host_audit_entry = results[0]
+
+    # Asserts on data that will match to the input jq string from above
+    assert host_audit_entry.canonical_facts == {'host_name': 'foo_host_default'}
+    assert host_audit_entry.facts == {'device_type': 'Fake Host'}

licenses/jq.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013, Michael Williamson
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

requirements/requirements.in

@@ -1,5 +1,6 @@
 aiohttp>=3.9.4 # CVE-2024-30251
 ansi2html  # Used to format the stdout from jobs into html for display
+jq  # used for indirect host counting feature
 asciichartpy
 asn1
 azure-identity

requirements/requirements.txt

@@ -40,7 +40,7 @@ autocommand==2.2.2
     # via jaraco-text
 automat==24.8.1
     # via twisted
-# awx-plugins-core @ git+https://git@github.com/ansible/awx-plugins.git@devel  # git requirements installed separately
+# awx-plugins-core @ git+https://github.com/ansible/awx-plugins.git@devel  # git requirements installed separately
     # via -r /awx_devel/requirements/requirements_git.txt
 awx-plugins.interfaces @ git+https://github.com/ansible/awx_plugins.interfaces.git
     # via -r /awx_devel/requirements/requirements_git.txt
@@ -236,6 +236,8 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
+jq==1.8.0
+    # via -r /awx_devel/requirements/requirements.in
 json-log-formatter==1.1
     # via -r /awx_devel/requirements/requirements.in
 jsonschema==4.23.0
@@ -528,4 +530,3 @@ setuptools==70.3.0
     #   setuptools-rust
     #   setuptools-scm
     #   zope-interface
-