From 47e0f134f0fdd797610f57555f88c12043ad0f65 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 30 Oct 2023 17:19:23 -0500 Subject: [PATCH] Add tests for stash/osdf scitokens.conf generation This also adds a test VO and ResourceGroup that's only visible to the tests, and test code to load them. This will allow us to test generation on fake data and not have the code break due to production data changes. This was originally written for PR #3457 but we changed our minds about that feature. --- src/tests/data/testrg.yaml | 246 +++++++++++++++++++++++++++++++++++ src/tests/data/testvo.yaml | 108 +++++++++++++++ src/tests/test_stashcache.py | 61 +++++++++ 3 files changed, 415 insertions(+) create mode 100644 src/tests/data/testrg.yaml create mode 100644 src/tests/data/testvo.yaml diff --git a/src/tests/data/testrg.yaml b/src/tests/data/testrg.yaml new file mode 100644 index 000000000..52a7b691d --- /dev/null +++ b/src/tests/data/testrg.yaml @@ -0,0 +1,246 @@ +Production: true +SupportCenter: Self Supported + +GroupDescription: Test Resource Group + +GroupID: 91359 + +Resources: + TEST_STASHCACHE_CACHE: + Active: true + Description: This is a StashCache cache server at TEST. + ID: 99958 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-cache.test.wisc.edu + DN: /CN=sc-cache.test.wisc.edu + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_TIGER_CACHE: + Active: true + Description: This is a StashCache cache server at TEST running on the Tiger Kubernetes cluster. + ID: 91098 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: stash-cache.osg.test.io + DN: /CN=stash-cache.osg.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 99959 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-origin.test.wisc.edu + DN: /CN=sc-origin.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN_2000: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 91069 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-origin2000.test.wisc.edu + DN: /CN=sc-origin2000.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN_AUTH_2000: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 91192 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: origin-auth2000.test.wisc.edu + FQDNAliases: + - origin-auth.test.wisc.edu + DN: /CN=origin-auth2000.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server, see OPS-198 + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_OSGDEV_SHOVELER_CACHE: + Active: true + Description: This is a testing StashCache cache server with a shoveler at TEST running on the Tiger Kubernetes cluster. + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: stash-cache-shoveler.osgdev.test.io + DN: /CN=stash-cache-shoveler.osgdev.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST-ITB-HELM-ORIGIN: + Active: false + Description: >- + This is a testing OSDF origin server on the Tiger cluster, + deployed via Helm. It serves both public and protected data. + ID: 91370 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000003 + Name: Brian Lin + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000003 + Name: Brian Lin + FQDN: helm-origin.osgdev.test.io + DN: /CN=helm-origin.osgdev.test.io + Services: + XRootD origin server: + Description: xrootd stash-origin and stash-origin-auth instances + VOOwnership: + testvo: 100 + AllowedVOs: + - testvo + + TEST-ITB-HELM-CACHE1: + Active: false + Description: >- + This is a testing StashCache cache server the Tiger cluster, + deployed via Helm. + ID: 91339 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000003 + Name: Brian Lin + Tertiary: + ID: OSG1000002 + Name: Matyas Selmeci + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: helm-cache1.osgdev.test.io + DN: /CN=helm-cache1.osgdev.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - testvo + - OSG + diff --git a/src/tests/data/testvo.yaml b/src/tests/data/testvo.yaml new file mode 100644 index 000000000..1a25cbe89 --- /dev/null +++ b/src/tests/data/testvo.yaml @@ -0,0 +1,108 @@ +AppDescription: Test VO +CertificateOnly: false +Community: Test VO +Contacts: + Administrative Contact: + - ID: OSG1000015 + Name: Aaron Moate + Miscellaneous Contact: + - ID: OSG1000018 + Name: Christina Koch + Registration Authority: + - ID: OSG1000015 + Name: Aaron Moate + Security Contact: + - ID: OSG1000015 + Name: Aaron Moate + VO Manager: + - ID: 48e1c2f26dc3479a6cf9b2de7c79d654ac27b1d1 + Name: Miron Livny +Credentials: + TokenIssuers: + - URL: https://test.wisc.edu + DefaultUnixUser: vdttest +Disable: false +FieldsOfScience: + PrimaryFields: + - Multi-Science Community + SecondaryFields: + - Community Grid +ID: 9999 +LongName: Test Virtual Organization +OASIS: + Managers: + - Name: Aaron Moate + DNs: /DC=org/DC=doegrids/OU=People/CN=Aaron Moate 717557 + ID: OSG1000015 + UseOASIS: true +PrimaryURL: https://test.wisc.edu +PurposeURL: https://test.wisc.edu/approach.shtml +SupportURL: https://test.wisc.edu + +DataFederations: + StashCache: + Namespaces: + - Path: /testvo/PUBLIC + Authorizations: + - PUBLIC + AllowedOrigins: + # sc-origin.test.wisc.edu + - TEST_STASHCACHE_ORIGIN + # sc-origin2000.test.wisc.edu + - TEST_STASHCACHE_ORIGIN_2000 + AllowedCaches: + - ANY + + - Path: /testvo/itb/helm-origin/PUBLIC + Authorizations: + - PUBLIC + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY + + - Path: /testvo/itb/helm-origin/PROTECTED + Authorizations: + - FQAN: /TESTVO + - DN: /DC=org/DC=cilogon/C=US/O=University of Wisconsin-Madison/CN=Matyas Selmeci A148276 + - SciTokens: + Issuer: https://test.wisc.edu + Base Path: /testvo + Map Subject: False + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY + + - Path: /testvo + Authorizations: + - SciTokens: + Issuer: https://test.wisc.edu + Base Path: /testvo + Map Subject: True + AllowedOrigins: + # origin-auth2000.test.wisc.edu + - TEST_STASHCACHE_ORIGIN_AUTH_2000 + AllowedCaches: + - ANY + Writeback: https://origin-auth2000.test.wisc.edu:1095 + DirList: https://origin-auth2000.test.wisc.edu:1095 + CredentialGeneration: + BasePath: /testvo + Strategy: OAuth2 + Issuer: https://test.wisc.edu + MaxScopeDepth: 3 + + - Path: /testvo/issuer2test + Authorizations: + - SciTokens: + Issuer: https://test.wisc.edu/issuer2 + BasePath: "/testvo/issuer2test" + MapSubject: False + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index a69d7d1a7..d5552cc2b 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -1,7 +1,10 @@ +from configparser import ConfigParser +import copy import flask import pytest import re from pytest_mock import MockerFixture +import time # Rewrites the path so the app can be imported like it normally is import os @@ -13,6 +16,8 @@ os.environ['TESTING'] = "True" from app import app, global_data +from webapp import models, topology, vos_data +from webapp.common import load_yaml_file import stashcache GRID_MAPPING_REGEX = re.compile(r'^"(/[^"]*CN=[^"]+")\s+([0-9a-f]{8}[.]0)$') @@ -20,6 +25,8 @@ EMPTY_LINE_REGEX = re.compile(r'^\s*(#|$)') # Empty or comment-only lines I2_TEST_CACHE = "osg-sunnyvale-stashcache.nrp.internet2.edu" # ^^ one of the Internet2 caches; these serve both public and LIGO data +TEST_ITB_HELM_ORIGIN = "helm-origin.osgdev.test.io" +# ^^ a fake origin that's in our test data # Some DNs I can use for testing and the hashes they map to. @@ -37,6 +44,37 @@ MOCK_DN_LIST = list(MOCK_DNS_AND_HASHES.keys()) +def get_test_global_data(global_data: models.GlobalData) -> models.GlobalData: + """Get a copy of the global data with some entries created for testing""" + new_global_data = copy.deepcopy(global_data) + + # Start with a fully populated set of topology data + topo = new_global_data.get_topology() + assert isinstance(topo, topology.Topology), "Unable to get Topology data" + + # Add our testing RG + testrg = load_yaml_file(topdir + "/tests/data/testrg.yaml") + topo.add_rg("University of Wisconsin", "CHTC", "testrg", testrg) + + # Put it back into global_data2 and make sure it doesn't get overwritten by future calls + new_global_data.topology.data = topo + new_global_data.topology.next_update = time.time() + 999999 + + # Start with a fully populated set of VO data + vos = new_global_data.get_vos_data() + assert isinstance(vos, vos_data.VOsData), "Unable to get VO data" + + # Load our testing VO + testvo = load_yaml_file(topdir + "/tests/data/testvo.yaml") + vos.add_vo("testvo", testvo) + + # Put it back into global_data2 and make sure it doesn't get overwritten by future calls + new_global_data.vos_data.data = vos + new_global_data.vos_data.next_update = time.time() + 999999 + + return new_global_data + + @pytest.fixture def client(): with app.test_client() as client: @@ -66,6 +104,29 @@ def test_allowedVO_excludes_LIGO_and_ANY_for_ligo_inclusion(self, client: flask. assert spy.call_count == 0 + def test_scitokens_issuer_sections(self, client: flask.Flask): + test_global_data = get_test_global_data(global_data) + origin_scitokens_conf = stashcache.generate_origin_scitokens( + test_global_data, TEST_ITB_HELM_ORIGIN) + assert origin_scitokens_conf.strip(), "Generated scitokens.conf empty" + + cp = ConfigParser() + cp.read_string(origin_scitokens_conf, "origin_scitokens.conf") + + try: + assert "Global" in cp, "Missing Global section" + assert "Issuer https://test.wisc.edu" in cp, \ + "Issuer missing" + assert "Issuer https://test.wisc.edu/issuer2" in cp, \ + "Issuer 2 missing" + assert "base_path" in cp["Issuer https://test.wisc.edu/issuer2"], \ + "Issuer 2 base_path missing" + assert cp["Issuer https://test.wisc.edu/issuer2"]["base_path"] == "/testvo/issuer2test", \ + "Issuer 2 has wrong base path" + except AssertionError: + print(f"Generated origin scitokens.conf text:\n{origin_scitokens_conf}\n", file=sys.stderr) + raise + def test_None_fdqn_isnt_error(self, client: flask.Flask): stashcache.generate_cache_authfile(global_data, None)