From 088d16b4e3258174d72257caa26c8280c502cdc2 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 27 Oct 2023 10:11:21 -0500 Subject: [PATCH 01/21] Register /ospool/ap22/.well-known public namespace The issuer and public key for ap22 live there and are served by a public XRootD stash-origin instance --- virtual-organizations/OSG.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/virtual-organizations/OSG.yaml b/virtual-organizations/OSG.yaml index 296d2ccd5..a04fb8fe4 100644 --- a/virtual-organizations/OSG.yaml +++ b/virtual-organizations/OSG.yaml @@ -227,6 +227,15 @@ DataFederations: Issuer: https://osg-htc.org/ospool MaxScopeDepth: 4 + # SciTokens issuer for ap22 + - Path: /ospool/ap22/.well-known + Authorizations: + - PUBLIC + AllowedOrigins: + - UChicago_OSGConnect_ap22 + # Do not cache this: direct access only + AllowedCaches: [] + - Path: /ospool/uc-shared/project Authorizations: - SciTokens: From 623d1977bb931db61bdedfbd5014c8a9cc7f22b3 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 27 Oct 2023 12:21:03 -0500 Subject: [PATCH 02/21] Add CSUN -> California State University, Northridge mapping see #3450 --- mappings/project_institution.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mappings/project_institution.yaml b/mappings/project_institution.yaml index 090586e77..6e6a50356 100644 --- a/mappings/project_institution.yaml +++ b/mappings/project_institution.yaml @@ -15,6 +15,7 @@ Cincinnati: "University of Cincinnati" Clarkson: "Clarkson University" Coe: "Coe College" Creighton: "Creighton University" +CSUN: "California State University, Northridge" CUBoulder: "University of Colorado Boulder" CUAnschutz: "University of Colorado Anschutz Medical Campus" CSUSB: "California State University, San Bernadino" From f5265be72ef71df4ff5859c0877d95cdf7720633 Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Mon, 30 Oct 2023 17:24:37 -0700 Subject: [PATCH 03/21] Adding NEBRASKA_NRP_OSDF_ORIGIN to NDP Adding NEBRASKA_NRP_OSDF_ORIGIN to NDP --- virtual-organizations/UCSD.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/virtual-organizations/UCSD.yaml b/virtual-organizations/UCSD.yaml index 347b42f2f..4c07d40c9 100644 --- a/virtual-organizations/UCSD.yaml +++ b/virtual-organizations/UCSD.yaml @@ -46,6 +46,7 @@ DataFederations: Map Subject: False AllowedOrigins: - SDSC_NRP_OSDF_ORIGIN + - NEBRASKA_NRP_OSDF_ORIGIN AllowedCaches: - ANY From 7a9153544f0ecaa26b8da2a15e6c60f217e66f63 Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Tue, 31 Oct 2023 11:03:11 -0700 Subject: [PATCH 04/21] Adding DN to ligo Adding a new DN to the monitoring system --- virtual-organizations/LIGO.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/virtual-organizations/LIGO.yaml b/virtual-organizations/LIGO.yaml index 508bcca40..8ae434abe 100644 --- a/virtual-organizations/LIGO.yaml +++ b/virtual-organizations/LIGO.yaml @@ -79,6 +79,7 @@ DataFederations: Namespaces: - Path: /user/ligo Authorizations: + - DN: /CN=hcc-mon2.unl.edu - FQAN: /osg/ligo - FQAN: /virgo - FQAN: /virgo/virgo From 1f14381dbd325ad8ce81323c5130dea4965a75a0 Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 31 Oct 2023 14:28:03 -0500 Subject: [PATCH 05/21] Update UChicago_OSGConnect.yaml Add UChicago_OSGConnect_Public_Origin topology entry --- .../UChicago/UChicago_OSGConnect.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml b/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml index 90fa9d097..3c61e79c9 100644 --- a/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml +++ b/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml @@ -534,3 +534,21 @@ Resources: Description: OS Pool access point Tags: - OSPool + + UChicago_OSGConnect_Public_Origin: + Active: true + Description: OSG Connect Public endpoint + ContactLists: + Administrative Contact: + Primary: + ID: 0a22bab3de2d83d723811e3fb1ebca904e924a97 + Name: Lincoln Bryant + Secondary: + ID: a418fbc5dd33637bba264c01d84d52dd317f2813 + Name: Judith Stephen + FQDN: osdf-public.uc.osg-htc.org + Services: + XRootD origin server: + Description: OSG Connect Public Origin Server + AllowedVOs: + - OSG From 45ffbefad246b6646648c28b675990a7ff99e975 Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 31 Oct 2023 14:34:29 -0500 Subject: [PATCH 06/21] Update OSG.yaml Add /ospool/uc-shared/public to list of namespaces. --- virtual-organizations/OSG.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/virtual-organizations/OSG.yaml b/virtual-organizations/OSG.yaml index 296d2ccd5..25bbbb858 100644 --- a/virtual-organizations/OSG.yaml +++ b/virtual-organizations/OSG.yaml @@ -109,6 +109,14 @@ DataFederations: - OSGCONNECT_ORIGIN AllowedCaches: - ANY + - Path: /ospool/uc-shared/public + Authorizations: + - PUBLIC + AllowedOrigins: + - UChicago_OSGConnect_Public_Origin + AllowedCaches: + - ANY + Writeback: https://osdf-public.uc.osg-htc.org:1094 # HACK: enormous hack to get us going on a demo # https://opensciencegrid.atlassian.net/browse/SOFTWARE-5398 From 2d9d46505139c6f8097437b0bfd9e475499972bb Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 31 Oct 2023 14:40:24 -0500 Subject: [PATCH 07/21] Update OSG.yaml Remove Writeback line for /ospool/uc-shared/public. --- virtual-organizations/OSG.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/virtual-organizations/OSG.yaml b/virtual-organizations/OSG.yaml index 25bbbb858..56d6d10e9 100644 --- a/virtual-organizations/OSG.yaml +++ b/virtual-organizations/OSG.yaml @@ -116,7 +116,6 @@ DataFederations: - UChicago_OSGConnect_Public_Origin AllowedCaches: - ANY - Writeback: https://osdf-public.uc.osg-htc.org:1094 # HACK: enormous hack to get us going on a demo # https://opensciencegrid.atlassian.net/browse/SOFTWARE-5398 From fd6e01ea807511d195d44b7d24103e5fdb1256a8 Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 31 Oct 2023 14:48:29 -0500 Subject: [PATCH 08/21] Update UChicago_OSGConnect.yaml Add DN for UChicago_OSGConnect_Public_Origin. --- topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml b/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml index 3c61e79c9..00596513e 100644 --- a/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml +++ b/topology/University of Chicago/UChicago/UChicago_OSGConnect.yaml @@ -547,6 +547,7 @@ Resources: ID: a418fbc5dd33637bba264c01d84d52dd317f2813 Name: Judith Stephen FQDN: osdf-public.uc.osg-htc.org + DN: /CN=osdf-public.uc.osg-htc.org Services: XRootD origin server: Description: OSG Connect Public Origin Server From 2650405080ba959ad45f56be14f9c278faafb1dc Mon Sep 17 00:00:00 2001 From: jlstephen Date: Tue, 31 Oct 2023 14:57:31 -0500 Subject: [PATCH 09/21] Update OSG.yaml Remove duplicate entry for /ospool/uc-shared/public on UChicago_OSGConnect_ap23 --- virtual-organizations/OSG.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/virtual-organizations/OSG.yaml b/virtual-organizations/OSG.yaml index 56d6d10e9..b2678bcc1 100644 --- a/virtual-organizations/OSG.yaml +++ b/virtual-organizations/OSG.yaml @@ -254,18 +254,3 @@ DataFederations: Strategy: OAuth2 Issuer: https://osg-htc.org/ospool/uc-shared MaxScopeDepth: 4 - - - Path: /ospool/uc-shared/public - Authorizations: - - PUBLIC - AllowedOrigins: - - UChicago_OSGConnect_ap23 - AllowedCaches: - - ANY - Writeback: https://ap23.uc.osg-htc.org:1095 - DirList: https://ap23.uc.osg-htc.org:1095 - CredentialGeneration: - Strategy: OAuth2 - Issuer: https://osg-htc.org/ospool/uc-shared - MaxScopeDepth: 4 - From edfaf32daedba35851f8022d228e82e9f0790931 Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Wed, 1 Nov 2023 10:35:06 -0700 Subject: [PATCH 10/21] Add downtime for Stashcache-Sunnyvale due to changing hostname Add downtime for Stashcache-Sunnyvale due to changing hostname --- .../I2SunnyvaleInfrastructure_downtime.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure_downtime.yaml b/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure_downtime.yaml index 10c35531e..ee0725c1b 100644 --- a/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure_downtime.yaml +++ b/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure_downtime.yaml @@ -9,3 +9,14 @@ Services: - XRootD cache server # --------------------------------------------------------- +- Class: UNSCHEDULED + ID: 1638600566 + Description: changing hostname + Severity: Intermittent Outage + StartTime: Nov 01, 2023 19:30 +0000 + EndTime: Nov 06, 2023 19:30 +0000 + CreatedTime: Nov 01, 2023 17:34 +0000 + ResourceName: Stashcache-Sunnyvale + Services: + - XRootD cache server +# --------------------------------------------------------- From c20a42c0c8d13d96a9d13e448297799cd60d2b40 Mon Sep 17 00:00:00 2001 From: Fabio Andrijauskas Date: Wed, 1 Nov 2023 10:51:29 -0700 Subject: [PATCH 11/21] Change Sunnyvale hostname Change Sunnyvale hostname --- .../Internet2Sunnyvale/I2SunnyvaleInfrastructure.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure.yaml b/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure.yaml index 63c665f80..1d0caac24 100644 --- a/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure.yaml +++ b/topology/Internet2/Internet2Sunnyvale/I2SunnyvaleInfrastructure.yaml @@ -17,8 +17,8 @@ Resources: Primary: Name: Fabio Andrijauskas ID: OSG1000162 - FQDN: osg-sunnyvale-stashcache.t2.ucsd.edu - DN: /DC=org/DC=incommon/C=US/ST=California/O=University of California, San Diego/CN=osg-sunnyvale-stashcache.t2.ucsd.edu + FQDN: osg-sunnyvale-stashcache.nrp.internet2.edu + DN: /DC=org/DC=incommon/C=US/ST=Michigan/O=University Corporation For Advanced Internet Development/CN=osg-sunnyvale-stashcache.nrp.internet2.edu Services: XRootD cache server: Description: Internet2Sunnyvale Cache From eb107259b71b284f1b1a8b695a036e0b44b349a6 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Wed, 1 Nov 2023 13:57:18 -0500 Subject: [PATCH 12/21] Update unit tests to reflect new osg-sunnyvale domain name --- src/tests/test_stashcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index 9c17a210f..a69d7d1a7 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -18,7 +18,7 @@ GRID_MAPPING_REGEX = re.compile(r'^"(/[^"]*CN=[^"]+")\s+([0-9a-f]{8}[.]0)$') # ^^ the DN starts with a slash and will at least have a CN in it. EMPTY_LINE_REGEX = re.compile(r'^\s*(#|$)') # Empty or comment-only lines -I2_TEST_CACHE = "osg-sunnyvale-stashcache.t2.ucsd.edu" +I2_TEST_CACHE = "osg-sunnyvale-stashcache.nrp.internet2.edu" # ^^ one of the Internet2 caches; these serve both public and LIGO data @@ -48,7 +48,7 @@ class TestStashcache: def test_allowedVO_includes_ANY_for_ligo_inclusion(self, client: flask.Flask, mocker: MockerFixture): spy = mocker.spy(global_data, "get_ligo_dn_list") - stashcache.generate_cache_authfile(global_data, "osg-sunnyvale-stashcache.t2.ucsd.edu") + stashcache.generate_cache_authfile(global_data, "osg-sunnyvale-stashcache.nrp.internet2.edu") assert spy.call_count == 5 From cdc395ff8d40600e712982411967667784a5c2b6 Mon Sep 17 00:00:00 2001 From: Irakli Chakaberia Date: Fri, 3 Nov 2023 14:39:51 -0700 Subject: [PATCH 13/21] Add downtime for LBL_HPCS due to colling work and SLURM update at ITD --- .../LBL_HPCS/LBL_HPCS_downtime.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/topology/Lawrence Berkley National Laboratory/LBL_HPCS/LBL_HPCS_downtime.yaml b/topology/Lawrence Berkley National Laboratory/LBL_HPCS/LBL_HPCS_downtime.yaml index 3a70ccef6..284531a0e 100644 --- a/topology/Lawrence Berkley National Laboratory/LBL_HPCS/LBL_HPCS_downtime.yaml +++ b/topology/Lawrence Berkley National Laboratory/LBL_HPCS/LBL_HPCS_downtime.yaml @@ -31,3 +31,14 @@ Services: - CE # --------------------------------------------------------- +- Class: SCHEDULED + ID: 1640473745 + Description: Cooling work in the machine room and SLURM upgrade + Severity: Outage + StartTime: Nov 21, 2023 15:00 +0000 + EndTime: Nov 23, 2023 00:00 +0000 + CreatedTime: Nov 03, 2023 21:36 +0000 + ResourceName: LBL_HPCS + Services: + - CE +# --------------------------------------------------------- From e958bb80a693b678f1ae407a242760b665c5905c Mon Sep 17 00:00:00 2001 From: Colby Date: Mon, 6 Nov 2023 11:01:36 -0800 Subject: [PATCH 14/21] Add downtime for The University of Montana due to cluster relocation --- .../UMontana IT/UMT-Hellgate_downtime.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml diff --git a/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml b/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml new file mode 100644 index 000000000..3a6ecc436 --- /dev/null +++ b/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml @@ -0,0 +1,12 @@ +- Class: UNSCHEDULED + ID: 1642970913 + Description: Cluster being moved location + Severity: Outage + StartTime: Nov 02, 2023 18:57 +0000 + EndTime: Nov 08, 2023 06:57 +0000 + CreatedTime: Nov 06, 2023 18:58 +0000 + ResourceName: UMT-Hellgate-CE1 + Services: + - CE +# --------------------------------------------------------- + From 21a0939d0d8de0e8a79e955bbabd7d504fec6710 Mon Sep 17 00:00:00 2001 From: Mats Rynge Date: Tue, 7 Nov 2023 07:40:53 -0700 Subject: [PATCH 15/21] New project: CMU_Isayev --- projects/CMU_Isayev.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 projects/CMU_Isayev.yaml diff --git a/projects/CMU_Isayev.yaml b/projects/CMU_Isayev.yaml new file mode 100644 index 000000000..ddce4431c --- /dev/null +++ b/projects/CMU_Isayev.yaml @@ -0,0 +1,8 @@ +Department: Chemistry +Description: Quantum chemical and machine learning insights into supra-molecular organization + of molecular crystals. +FieldOfScience: Chemistry +Organization: Carnegie-Mellon University +PIName: Olexandr Isayev + + From 62441b41fa3e7ff21967abcb2699aa42cc79b348 Mon Sep 17 00:00:00 2001 From: Ashton Graves Date: Tue, 7 Nov 2023 11:16:33 -0600 Subject: [PATCH 16/21] Extends downtime for UMT Hellgate Downtime unknown. Extend to very large num per Tim C --- .../UMontana IT/UMT-Hellgate_downtime.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml b/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml index 3a6ecc436..7c8881663 100644 --- a/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml +++ b/topology/The University of Montana/UMontana IT/UMT-Hellgate_downtime.yaml @@ -3,7 +3,7 @@ Description: Cluster being moved location Severity: Outage StartTime: Nov 02, 2023 18:57 +0000 - EndTime: Nov 08, 2023 06:57 +0000 + EndTime: Dec 31, 2023 06:57 +0000 CreatedTime: Nov 06, 2023 18:58 +0000 ResourceName: UMT-Hellgate-CE1 Services: From cd05b761efb4f108abece77ec1dd9db5b8d090b8 Mon Sep 17 00:00:00 2001 From: Farrukh Aftab Khan Date: Tue, 7 Nov 2023 15:21:11 -0600 Subject: [PATCH 17/21] Adding a downtime for FNAL_GPGRID_CE_04 Migrating gpce04.fnal.gov to Alma9 --- .../FermiGrid/GPGRID_downtime.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/topology/Fermi National Accelerator Laboratory/FermiGrid/GPGRID_downtime.yaml b/topology/Fermi National Accelerator Laboratory/FermiGrid/GPGRID_downtime.yaml index 9584a6f32..0cad1d666 100644 --- a/topology/Fermi National Accelerator Laboratory/FermiGrid/GPGRID_downtime.yaml +++ b/topology/Fermi National Accelerator Laboratory/FermiGrid/GPGRID_downtime.yaml @@ -98,3 +98,15 @@ Services: - CE # --------------------------------------------------------- +- Class: SCHEDULED + ID: 1643919815 + Description: Migration to Alma9 + Severity: Outage + StartTime: Nov 13, 2023 12:00 +0000 + EndTime: Nov 14, 2023 22:00 +0000 + CreatedTime: Nov 07, 2023 21:19 +0000 + ResourceName: FNAL_GPGRID_CE_04 + Services: + - CE +# --------------------------------------------------------- + From ac47bf824e47d348f7f8b94c0188d083e7ce760a Mon Sep 17 00:00:00 2001 From: Farrukh Aftab Khan Date: Tue, 7 Nov 2023 15:22:48 -0600 Subject: [PATCH 18/21] Update USCMS-FNAL-WC1_downtime.yaml Adding downtime for cmsosgce4.fnal.gov and cmsosgce-opp1.fnal.gov. Migrating these two CEs over to Alma9 --- .../USCMS-FNAL-WC1_downtime.yaml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml b/topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml index 52b12d511..9b9598674 100644 --- a/topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml +++ b/topology/Fermi National Accelerator Laboratory/FNAL USCMS Tier1/USCMS-FNAL-WC1_downtime.yaml @@ -983,10 +983,26 @@ Services: - CE # --------------------------------------------------------- - - - - - - +- Class: SCHEDULED + ID: 1643919211 + Description: Migration to Alma9 + Severity: Outage + StartTime: Nov 13, 2023 12:00 +0000 + EndTime: Nov 14, 2023 22:00 +0000 + CreatedTime: Nov 07, 2023 21:18 +0000 + ResourceName: USCMS-FNAL-WC1-CE4 + Services: + - CE +# --------------------------------------------------------- +- Class: SCHEDULED + ID: 1643919571 + Description: Migration to Alma9 + Severity: Outage + StartTime: Nov 13, 2023 12:00 +0000 + EndTime: Nov 14, 2023 22:00 +0000 + CreatedTime: Nov 07, 2023 21:19 +0000 + ResourceName: USCMS-FNAL-WC1-OPP1 + Services: + - CE +# --------------------------------------------------------- From 195f2848a61de4e7b2f952aeabfc1e1ae898b4d9 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 7 Nov 2023 17:12:36 -0600 Subject: [PATCH 19/21] Remove space from project name --- ...hamAndWomens_ Baratono.yaml => BrighamAndWomens_Baratono.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename projects/{BrighamAndWomens_ Baratono.yaml => BrighamAndWomens_Baratono.yaml} (100%) diff --git a/projects/BrighamAndWomens_ Baratono.yaml b/projects/BrighamAndWomens_Baratono.yaml similarity index 100% rename from projects/BrighamAndWomens_ Baratono.yaml rename to projects/BrighamAndWomens_Baratono.yaml From eea8e34ea9fc048161ca3fdc18d6ac297d683970 Mon Sep 17 00:00:00 2001 From: smithnp <60900055+smithnp@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:06:12 -0500 Subject: [PATCH 20/21] Retiring brown-osg CE We are retiring the brown cluster, so I have set the brown-osg-ce active status to false. Also I noticed hadoop-osg was still set active, it has been retired for some time now, I have also set it to false. --- topology/Purdue University/Purdue CMS/Purdue.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/topology/Purdue University/Purdue CMS/Purdue.yaml b/topology/Purdue University/Purdue CMS/Purdue.yaml index 26723a6cd..256d8028a 100644 --- a/topology/Purdue University/Purdue CMS/Purdue.yaml +++ b/topology/Purdue University/Purdue CMS/Purdue.yaml @@ -1,6 +1,6 @@ GroupDescription: CMS tier 2 facility at Purdue University, West Lafayette, IN. GroupID: 393 -Production: true +Production: false Resources: Purdue-Hadoop-CE: Active: true @@ -180,7 +180,7 @@ Resources: StorageCapacityMin: 1 TapeCapacity: 0 Purdue-Brown: - Active: true + Active: false ContactLists: Administrative Contact: Primary: From 47e0f134f0fdd797610f57555f88c12043ad0f65 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 30 Oct 2023 17:19:23 -0500 Subject: [PATCH 21/21] Add tests for stash/osdf scitokens.conf generation This also adds a test VO and ResourceGroup that's only visible to the tests, and test code to load them. This will allow us to test generation on fake data and not have the code break due to production data changes. This was originally written for PR #3457 but we changed our minds about that feature. --- src/tests/data/testrg.yaml | 246 +++++++++++++++++++++++++++++++++++ src/tests/data/testvo.yaml | 108 +++++++++++++++ src/tests/test_stashcache.py | 61 +++++++++ 3 files changed, 415 insertions(+) create mode 100644 src/tests/data/testrg.yaml create mode 100644 src/tests/data/testvo.yaml diff --git a/src/tests/data/testrg.yaml b/src/tests/data/testrg.yaml new file mode 100644 index 000000000..52a7b691d --- /dev/null +++ b/src/tests/data/testrg.yaml @@ -0,0 +1,246 @@ +Production: true +SupportCenter: Self Supported + +GroupDescription: Test Resource Group + +GroupID: 91359 + +Resources: + TEST_STASHCACHE_CACHE: + Active: true + Description: This is a StashCache cache server at TEST. + ID: 99958 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-cache.test.wisc.edu + DN: /CN=sc-cache.test.wisc.edu + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_TIGER_CACHE: + Active: true + Description: This is a StashCache cache server at TEST running on the Tiger Kubernetes cluster. + ID: 91098 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: stash-cache.osg.test.io + DN: /CN=stash-cache.osg.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 99959 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-origin.test.wisc.edu + DN: /CN=sc-origin.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN_2000: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 91069 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: sc-origin2000.test.wisc.edu + DN: /CN=sc-origin2000.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_STASHCACHE_ORIGIN_AUTH_2000: + Active: true + Description: This is a StashCache origin server at TEST. + ID: 91192 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: origin-auth2000.test.wisc.edu + FQDNAliases: + - origin-auth.test.wisc.edu + DN: /CN=origin-auth2000.test.wisc.edu + Services: + XRootD origin server: + Description: StashCache origin server, see OPS-198 + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST_OSGDEV_SHOVELER_CACHE: + Active: true + Description: This is a testing StashCache cache server with a shoveler at TEST running on the Tiger Kubernetes cluster. + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: stash-cache-shoveler.osgdev.test.io + DN: /CN=stash-cache-shoveler.osgdev.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - ANY + + TEST-ITB-HELM-ORIGIN: + Active: false + Description: >- + This is a testing OSDF origin server on the Tiger cluster, + deployed via Helm. It serves both public and protected data. + ID: 91370 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000003 + Name: Brian Lin + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000003 + Name: Brian Lin + FQDN: helm-origin.osgdev.test.io + DN: /CN=helm-origin.osgdev.test.io + Services: + XRootD origin server: + Description: xrootd stash-origin and stash-origin-auth instances + VOOwnership: + testvo: 100 + AllowedVOs: + - testvo + + TEST-ITB-HELM-CACHE1: + Active: false + Description: >- + This is a testing StashCache cache server the Tiger cluster, + deployed via Helm. + ID: 91339 + ContactLists: + Administrative Contact: + Primary: + ID: OSG1000003 + Name: Brian Lin + Tertiary: + ID: OSG1000002 + Name: Matyas Selmeci + Security Contact: + Primary: + ID: OSG1000002 + Name: Matyas Selmeci + Secondary: + ID: OSG1000015 + Name: Aaron Moate + FQDN: helm-cache1.osgdev.test.io + DN: /CN=helm-cache1.osgdev.test.io + Services: + XRootD cache server: + Description: StashCache cache server + VOOwnership: + testvo: 100 + AllowedVOs: + - testvo + - OSG + diff --git a/src/tests/data/testvo.yaml b/src/tests/data/testvo.yaml new file mode 100644 index 000000000..1a25cbe89 --- /dev/null +++ b/src/tests/data/testvo.yaml @@ -0,0 +1,108 @@ +AppDescription: Test VO +CertificateOnly: false +Community: Test VO +Contacts: + Administrative Contact: + - ID: OSG1000015 + Name: Aaron Moate + Miscellaneous Contact: + - ID: OSG1000018 + Name: Christina Koch + Registration Authority: + - ID: OSG1000015 + Name: Aaron Moate + Security Contact: + - ID: OSG1000015 + Name: Aaron Moate + VO Manager: + - ID: 48e1c2f26dc3479a6cf9b2de7c79d654ac27b1d1 + Name: Miron Livny +Credentials: + TokenIssuers: + - URL: https://test.wisc.edu + DefaultUnixUser: vdttest +Disable: false +FieldsOfScience: + PrimaryFields: + - Multi-Science Community + SecondaryFields: + - Community Grid +ID: 9999 +LongName: Test Virtual Organization +OASIS: + Managers: + - Name: Aaron Moate + DNs: /DC=org/DC=doegrids/OU=People/CN=Aaron Moate 717557 + ID: OSG1000015 + UseOASIS: true +PrimaryURL: https://test.wisc.edu +PurposeURL: https://test.wisc.edu/approach.shtml +SupportURL: https://test.wisc.edu + +DataFederations: + StashCache: + Namespaces: + - Path: /testvo/PUBLIC + Authorizations: + - PUBLIC + AllowedOrigins: + # sc-origin.test.wisc.edu + - TEST_STASHCACHE_ORIGIN + # sc-origin2000.test.wisc.edu + - TEST_STASHCACHE_ORIGIN_2000 + AllowedCaches: + - ANY + + - Path: /testvo/itb/helm-origin/PUBLIC + Authorizations: + - PUBLIC + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY + + - Path: /testvo/itb/helm-origin/PROTECTED + Authorizations: + - FQAN: /TESTVO + - DN: /DC=org/DC=cilogon/C=US/O=University of Wisconsin-Madison/CN=Matyas Selmeci A148276 + - SciTokens: + Issuer: https://test.wisc.edu + Base Path: /testvo + Map Subject: False + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY + + - Path: /testvo + Authorizations: + - SciTokens: + Issuer: https://test.wisc.edu + Base Path: /testvo + Map Subject: True + AllowedOrigins: + # origin-auth2000.test.wisc.edu + - TEST_STASHCACHE_ORIGIN_AUTH_2000 + AllowedCaches: + - ANY + Writeback: https://origin-auth2000.test.wisc.edu:1095 + DirList: https://origin-auth2000.test.wisc.edu:1095 + CredentialGeneration: + BasePath: /testvo + Strategy: OAuth2 + Issuer: https://test.wisc.edu + MaxScopeDepth: 3 + + - Path: /testvo/issuer2test + Authorizations: + - SciTokens: + Issuer: https://test.wisc.edu/issuer2 + BasePath: "/testvo/issuer2test" + MapSubject: False + AllowedOrigins: + # helm-origin.osgdev.test.io + - TEST-ITB-HELM-ORIGIN + AllowedCaches: + - ANY diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index a69d7d1a7..d5552cc2b 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -1,7 +1,10 @@ +from configparser import ConfigParser +import copy import flask import pytest import re from pytest_mock import MockerFixture +import time # Rewrites the path so the app can be imported like it normally is import os @@ -13,6 +16,8 @@ os.environ['TESTING'] = "True" from app import app, global_data +from webapp import models, topology, vos_data +from webapp.common import load_yaml_file import stashcache GRID_MAPPING_REGEX = re.compile(r'^"(/[^"]*CN=[^"]+")\s+([0-9a-f]{8}[.]0)$') @@ -20,6 +25,8 @@ EMPTY_LINE_REGEX = re.compile(r'^\s*(#|$)') # Empty or comment-only lines I2_TEST_CACHE = "osg-sunnyvale-stashcache.nrp.internet2.edu" # ^^ one of the Internet2 caches; these serve both public and LIGO data +TEST_ITB_HELM_ORIGIN = "helm-origin.osgdev.test.io" +# ^^ a fake origin that's in our test data # Some DNs I can use for testing and the hashes they map to. @@ -37,6 +44,37 @@ MOCK_DN_LIST = list(MOCK_DNS_AND_HASHES.keys()) +def get_test_global_data(global_data: models.GlobalData) -> models.GlobalData: + """Get a copy of the global data with some entries created for testing""" + new_global_data = copy.deepcopy(global_data) + + # Start with a fully populated set of topology data + topo = new_global_data.get_topology() + assert isinstance(topo, topology.Topology), "Unable to get Topology data" + + # Add our testing RG + testrg = load_yaml_file(topdir + "/tests/data/testrg.yaml") + topo.add_rg("University of Wisconsin", "CHTC", "testrg", testrg) + + # Put it back into global_data2 and make sure it doesn't get overwritten by future calls + new_global_data.topology.data = topo + new_global_data.topology.next_update = time.time() + 999999 + + # Start with a fully populated set of VO data + vos = new_global_data.get_vos_data() + assert isinstance(vos, vos_data.VOsData), "Unable to get VO data" + + # Load our testing VO + testvo = load_yaml_file(topdir + "/tests/data/testvo.yaml") + vos.add_vo("testvo", testvo) + + # Put it back into global_data2 and make sure it doesn't get overwritten by future calls + new_global_data.vos_data.data = vos + new_global_data.vos_data.next_update = time.time() + 999999 + + return new_global_data + + @pytest.fixture def client(): with app.test_client() as client: @@ -66,6 +104,29 @@ def test_allowedVO_excludes_LIGO_and_ANY_for_ligo_inclusion(self, client: flask. assert spy.call_count == 0 + def test_scitokens_issuer_sections(self, client: flask.Flask): + test_global_data = get_test_global_data(global_data) + origin_scitokens_conf = stashcache.generate_origin_scitokens( + test_global_data, TEST_ITB_HELM_ORIGIN) + assert origin_scitokens_conf.strip(), "Generated scitokens.conf empty" + + cp = ConfigParser() + cp.read_string(origin_scitokens_conf, "origin_scitokens.conf") + + try: + assert "Global" in cp, "Missing Global section" + assert "Issuer https://test.wisc.edu" in cp, \ + "Issuer missing" + assert "Issuer https://test.wisc.edu/issuer2" in cp, \ + "Issuer 2 missing" + assert "base_path" in cp["Issuer https://test.wisc.edu/issuer2"], \ + "Issuer 2 base_path missing" + assert cp["Issuer https://test.wisc.edu/issuer2"]["base_path"] == "/testvo/issuer2test", \ + "Issuer 2 has wrong base path" + except AssertionError: + print(f"Generated origin scitokens.conf text:\n{origin_scitokens_conf}\n", file=sys.stderr) + raise + def test_None_fdqn_isnt_error(self, client: flask.Flask): stashcache.generate_cache_authfile(global_data, None)