diff --git a/.github/PULL_REQUEST_TEMPLATE/bug-pull-request.md b/.github/PULL_REQUEST_TEMPLATE/bug-pull-request.md new file mode 100644 index 00000000..d4bf6d6c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/bug-pull-request.md @@ -0,0 +1,42 @@ +--- +name: Bug +about: Provide specifics about a PR for a bug fix +title: 'PR/Bug: ____(provide a descriptive name)' +labels: bug, p1 +assignees: '' + +--- + + + + +**Description** + + + +Fixes # + + +**Update Changelog** + + + +- [ ] README.md, [changelog](../../README.md#changelog) + + +**Bump version** + +v0.xx.x -> v0.xx.x + +- [ ] README.md, [installation instructions](../../README.md#installation-instructions) +- [ ] [`setup.py`](../../setup.py) +- [ ] [`requiam/__init__.py`](../../requiam/__init__.py) + + +*Screenshots or additional context* + + + +*Testing (if applicable)* + + diff --git a/.github/PULL_REQUEST_TEMPLATE/feature-pull-request.md b/.github/PULL_REQUEST_TEMPLATE/feature-pull-request.md new file mode 100644 index 00000000..14dc4bb3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/feature-pull-request.md @@ -0,0 +1,45 @@ +--- +name: Feature +about: Provide specifics about a PR for a feature +title: 'PR/Feature: ____(provide a descriptive name)' +labels: enhancement +assignees: '' + +--- + + + + +**Description** + + + +See # + + +**ToDo List** + + + - [ ] ToDo --> + - [ ] ToDo --> + - [ ] ToDo --> + + +**Test plan** + + + + +**Update Changelog** + + + +- [ ] README.md, [changelog](../../README.md#changelog) + + +*Resources* + + + +*Screenshots or additional context* + diff --git a/.github/PULL_REQUEST_TEMPLATE/release-pull-request.md b/.github/PULL_REQUEST_TEMPLATE/release-pull-request.md new file mode 100644 index 00000000..15abc6cc --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/release-pull-request.md @@ -0,0 +1,36 @@ +--- +name: Release +about: Provide specifics about a PR for a release +title: 'PR/Release: ____(provide a descriptive name)' +assignees: '' + +--- + + + + +**Description** + + + +Closes # + + +**Update Changelog** + + + +- [ ] README.md, [changelog](../../README.md#changelog) + + +**Bump version** + +v0.xx.x -> v0.xx.0 + +- [ ] README.md, [installation instructions](../../README.md#installation-instructions) +- [ ] [`setup.py`](../../setup.py) +- [ ] [`requiam/__init__.py`](../../requiam/__init__.py) + + +*Screenshots or additional context* + diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index adfc49a9..3d65de50 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -25,7 +25,7 @@ jobs: if: "!contains(github.event.head_commit.message, 'ci skip') || !contains(github.event.head_commit.message, 'skip ci')" strategy: matrix: - python-version: ['3.7', '3.7.5', '3.8'] + python-version: ['3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index ec02d574..f7a4be98 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ These instructions will have the code running on your local or virtual machine. You will need the following to have a working copy of this software. See [installation](#installation-instructions) steps: -1. Python (3.7.5) +1. Python (>=3.7.9) 2. [`pandas`](https://pandas.pydata.org/) ([0.25.3](https://pandas.pydata.org/pandas-docs/version/0.25.3/)) 3. [`ldap3`](https://ldap3.readthedocs.io/en/latest/) (2.6.1) 4. [`requests`](https://requests.readthedocs.io/en/master/) (2.22.0) @@ -74,14 +74,14 @@ You will need the following to have a working copy of this software. See #### Python and setting up a `conda` environment -First, install a working version of Python (v3.7.5). We recommend using the +First, install a working version of Python (>=3.7.9). We recommend using the [Anaconda](https://www.anaconda.com/distribution/) package installer. After you have Anaconda installed, you will want to create a separate `conda` environment and activate it: ``` -$ (sudo) conda create -n figshare_patrons python=3.7.5 +$ (sudo) conda create -n figshare_patrons python=3.7 $ conda activate figshare_patrons ``` @@ -107,7 +107,7 @@ You can confirm installation via `conda list` (figshare_patrons) $ conda list requiam ``` -You should see that the version is `0.15.1`. +You should see that the version is `0.16.0`. ### Configuration Settings @@ -296,12 +296,19 @@ Currently, there are two GitHub Action workflows: A list of released features and their issue number(s). List is sorted from moderate to minor revisions for reach release. -v0.15.0: +v0.16.0: + * Merge `grouper_admin` and `grouper_query` modules #87 + * Complete adoption of f-strings #118 + * New pull request templates #120 + * CI build testing for Python 3.9 #121 + +v0.15.0 - v0.15.1: * GitHub actions for CI #105 * Simplify scripts to use dictionary for configuration settings #104 * Improve verbosity of log messages #103 * Priority labels for project management #108 * Add/update GitHub issue templates #110 + * Bug: Fix f-string typo with batch load information #114 v0.14.0: * Travis CI integration #91 diff --git a/requiam/__init__.py b/requiam/__init__.py index 3c51bf85..d087459e 100644 --- a/requiam/__init__.py +++ b/requiam/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.1" +__version__ = "0.16.0" class TimerClass(object): @@ -44,4 +44,4 @@ def _stop(self): HH = int(sec // 3600) MM = int((sec // 60) - (HH * 60)) SS = sec - (HH * 3600) - (MM * 60) - self.format = "Total time: {0: 02d} hours {1: 02d} minutes {2: .2f} seconds".format(HH, MM, SS) + self.format = f"Total time: {HH: 02d} hours {MM: 02d} minutes {SS: .2f} seconds" diff --git a/requiam/commons.py b/requiam/commons.py index 9f72707c..7b2a26cf 100644 --- a/requiam/commons.py +++ b/requiam/commons.py @@ -34,6 +34,39 @@ def figshare_stem(stem='', production=True): return stem_query +def figshare_group(group, root_stem, production=True): + """ + Purpose: + Construct Grouper figshare groups + + :param group: str or int of group name. Cannot be empty + :param root_stem: str of associated stem folder for [group] + :param production: Bool to use production stem. Otherwise a stage/test is used. Default: True + + :return grouper_group: str containing full Grouper path + + Usage: + For active group, call as: figshare_group('active', '') + > 'arizona.edu:dept:LBRY:figshare:active' + + For a quota group, call as: figshare_group('2147483648', 'quota') + > 'arizona.edu:dept:LBRY:figshare:quota:2147483648' + Note: group can be specified as an integer for quota cases + + For a portal group, call as: figshare_group('sci_math', 'portal') + > 'arizona.edu:dept:LBRY:figshare:portal:sci_math' + """ + + if not group: + raise ValueError("WARNING: Empty [group]") + + stem_query = figshare_stem(stem=root_stem, production=production) + + grouper_group = f'{stem_query}:{group}' + + return grouper_group + + def dict_load(config_file, vargs=None): """ Purpose: diff --git a/requiam/delta.py b/requiam/delta.py index fe408601..e7c3ad6c 100644 --- a/requiam/delta.py +++ b/requiam/delta.py @@ -22,8 +22,9 @@ class Delta(object): """ - def __init__(self, ldap_members, grouper_query_instance, batch_size, - batch_timeout, batch_delay, sync_max, log=None): + def __init__(self, ldap_members, grouper_query_dict, + batch_size, batch_timeout, batch_delay, sync_max, + log=None): if isinstance(log, type(None)): self.log = log_stdout() @@ -33,7 +34,8 @@ def __init__(self, ldap_members, grouper_query_instance, batch_size, self.log.debug('entered') self.ldap_members = ldap_members - self.grouper_qry = grouper_query_instance + self.grouper_query_dict = grouper_query_dict + self.grouper_members = grouper_query_dict['members'] self.batch_size = batch_size self.batch_timeout = batch_timeout self.batch_delay = batch_delay @@ -47,19 +49,19 @@ def __init__(self, ldap_members, grouper_query_instance, batch_size, return def _common(self): - common = self.ldap_members & self.grouper_qry.members + common = self.ldap_members & self.grouper_members self.log.debug('finished common') return common def _adds(self): - adds = self.ldap_members - self.grouper_qry.members + adds = self.ldap_members - self.grouper_members self.log.debug('finished adds') return adds def _drops(self): - drops = self.grouper_qry.members - self.ldap_members + drops = self.grouper_members - self.ldap_members self.log.debug('finished drops') return drops @@ -74,7 +76,8 @@ def synchronize(self): self.log.debug('finished synchronize') return - self.log.info(f"synchronizing ldap query results to {self.grouper_qry.grouper_group}") + self.log.info("synchronizing ldap query results to " + + f"{self.grouper_query_dict['grouper_group']}") self.log.info(f"batch size = {self.batch_size}, " + f"batch timeout = {self.batch_timeout} seconds, " + f"batch delay = {self.batch_delay} seconds") @@ -87,9 +90,9 @@ def synchronize(self): n_batches += 1 start_t = datetime.datetime.now() - rsp = requests.post(self.grouper_qry.grouper_group_members_url, - auth=(self.grouper_qry.grouper_user, - self.grouper_qry.grouper_password), + rsp = requests.post(self.grouper_query_dict['grouper_members_url'], + auth=(self.grouper_query_dict['grouper_user'], + self.grouper_query_dict['grouper_password']), data=json.dumps({ 'WsRestDeleteMemberRequest': { 'replaceAllExisting': 'F', @@ -122,9 +125,9 @@ def synchronize(self): n_batches += 1 start_t = datetime.datetime.now() - rsp = requests.put(self.grouper_qry.grouper_group_members_url, - auth=(self.grouper_qry.grouper_user, - self.grouper_qry.grouper_password), + rsp = requests.put(self.grouper_query_dict['grouper_members_url'], + auth=(self.grouper_query_dict['grouper_user'], + self.grouper_query_dict['grouper_password']), data=json.dumps({ 'WsRestAddMemberRequest': { 'replaceAllExisting': 'F', diff --git a/requiam/grouper_admin.py b/requiam/grouper.py similarity index 75% rename from requiam/grouper_admin.py rename to requiam/grouper.py index 4645a262..21ba7cba 100644 --- a/requiam/grouper_admin.py +++ b/requiam/grouper.py @@ -1,21 +1,23 @@ -from os.path import dirname, join +from os.path import join import requests import pandas as pd from requests.exceptions import HTTPError -from .commons import figshare_stem -from .grouper_query import figshare_group +from .commons import figshare_stem, figshare_group +from .delta import Delta from .logger import log_stdout # Administrative groups +from .manual_override import update_entries + superadmins = figshare_group('GrouperSuperAdmins', '', production=True) admins = figshare_group('GrouperAdmins', '', production=True) managers = figshare_group('GrouperManagers', '', production=True) -class GrouperAPI: +class Grouper: """ Purpose: This class uses the Grouper API to retrieve and post a variety of @@ -35,10 +37,11 @@ class GrouperAPI: Attributes ---------- grouper_host: str - grouper_base_dn: str + grouper_base_path: str grouper_user: str grouper_password: str grouper_production: bool + grouper_auth: tuple endpoint: str Grouper endpoint @@ -50,6 +53,9 @@ class GrouperAPI: url(endpoint) Return full Grouper URL endpoint + query(group) + Query Grouper for list of members in a group. + get_group_list(group_type) Retrieve list of groups in a Grouper stem group_type must be 'portal', 'quota', 'test', 'group_active' or '' @@ -93,11 +99,11 @@ def __init__(self, grouper_host, grouper_base_path, grouper_user, self.log = log self.grouper_host = grouper_host - self.grouper_base_dn = grouper_base_path + self.grouper_base_path = grouper_base_path self.grouper_user = grouper_user self.grouper_password = grouper_password self.grouper_production = grouper_production - + self.grouper_auth = (self.grouper_user, self.grouper_password) self.endpoint = f'https://{grouper_host}/{grouper_base_path}' self.headers = {'Content-Type': 'text/x-json'} @@ -106,6 +112,30 @@ def url(self, endpoint): return join(self.endpoint, endpoint) + def query(self, group): + """ + Query Grouper for list of members in a group. + Returns a dict with Grouper metadata + """ + + endpoint = self.url(f"groups/{group}/members") + + rsp = requests.get(endpoint, auth=self.grouper_auth) + + grouper_query_dict = vars(self) + + # Append query specifics + grouper_query_dict['grouper_members_url'] = endpoint + grouper_query_dict['grouper_group'] = group + + if 'wsSubjects' in rsp.json()['WsGetMembersLiteResult']: + grouper_query_dict['members'] = \ + {s['id'] for s in rsp.json()['WsGetMembersLiteResult']['wsSubjects']} + else: + grouper_query_dict['members'] = set([]) + + return grouper_query_dict + def get_group_list(self, group_type): """Retrieve list of groups in a Grouper stem""" @@ -124,7 +154,7 @@ def get_group_list(self, group_type): } rsp = requests.post(endpoint, json=params, headers=self.headers, - auth=(self.grouper_user, self.grouper_password)) + auth=self.grouper_auth) return rsp.json() @@ -141,7 +171,7 @@ def get_group_details(self, group): } rsp = requests.post(endpoint, json=params, headers=self.headers, - auth=(self.grouper_user, self.grouper_password)) + auth=self.grouper_auth) return rsp.json()['WsFindGroupsResults']['groupResults'] @@ -186,7 +216,7 @@ def add_group(self, group, group_type, description): try: result = requests.post(endpoint, json=params, headers=self.headers, - auth=(self.grouper_user, self.grouper_password)) + auth=self.grouper_auth) metadata = result.json()['WsGroupSaveResults']['resultMetadata'] @@ -251,7 +281,7 @@ def add_privilege(self, access_group, target_group, target_group_type, privilege for privilege in privileges: params['WsRestAssignGrouperPrivilegesLiteRequest']['privilegeName'] = privilege result = requests.post(endpoint, json=params, headers=self.headers, - auth=(self.grouper_user, self.grouper_password)) + auth=self.grouper_auth) metadata = result.json()['WsAssignGrouperPrivilegesLiteResult']['resultMetadata'] if metadata['resultCode'] not in ['SUCCESS_ALLOWED', 'SUCCESS_ALLOWED_ALREADY_EXISTED']: @@ -269,7 +299,7 @@ def create_groups(groups, group_type, group_descriptions, grouper_api, log0=None :param groups: str or list of str containing group names :param group_type: str. Either 'portal', 'quota', or 'test' :param group_descriptions: str or list of str containing description - :param grouper_api: GrouperAPI object + :param grouper_api: Grouper object :param log0: logging.getLogger() object :param add: boolean. Indicate whether to perform update or dry run """ @@ -348,7 +378,7 @@ def create_active_group(group, grouper_dict, group_description=None, log=None, a log = log_stdout() # This is for figtest stem - ga_test = GrouperAPI(**grouper_dict, grouper_production=False, log=log) + ga_test = Grouper(**grouper_dict, grouper_production=False, log=log) if isinstance(group_description, type(None)): log.info("PROMPT: Provide description for group...") @@ -357,3 +387,76 @@ def create_active_group(group, grouper_dict, group_description=None, log=None, a create_groups(group, 'group_active', group_description, ga_test, log0=log, add=add) + + +def grouper_delta_user(group, stem, netid, uaid, action, grouper_dict, + delta_dict, mo=None, sync=False, log=None, + production=True): + """ + Purpose: + Construct a Delta object for addition/deletion based for a specified + user. This is designed primarily for the user_update script + + :param group: str + The Grouper group to update + :param stem: str + The Grouper stem (e.g., 'portal', 'quota') + :param netid: str + The User NetID + :param uaid: str + The User UA ID + :param action: str + The action to perform. 'add' or 'remove' + :param grouper_dict: dict + Dictionary containing grouper settings + :param delta_dict: + Dictionary containing delta settings + :param mo: ManualOverride object + For implementing change to CSV files. Default: None + :param sync: bool + Indicate whether to sync. Default: False + :param log: LogClass object + For logging + :param production: Bool to use production stem. Otherwise a stage/test is used. Default: True + + :return d: Delta object class + """ + + if isinstance(log, type(None)): + log = log_stdout() + + grouper_query = figshare_group(group, stem, production=production) + grouper = Grouper(**grouper_dict) + grouper_query_dict = grouper.query(grouper_query) + + if not isinstance(netid, list): + netid = [netid] + if not isinstance(uaid, list): + uaid = [uaid] + member_set = update_entries(grouper_query_dict['members'], + netid, uaid, action, log=log) + + d = Delta(ldap_members=member_set, + grouper_query_dict=grouper_query_dict, + **delta_dict, + log=log) + + log.info(f"ldap and grouper have {len(d.common)} members in common") + log.info(f"synchronization will drop {len(d.drops)} entries to Grouper {group} group") + log.info(f"synchronization will add {len(d.adds)} entries to Grouper {group} group") + + if sync: + log.info('synchronizing ...') + d.synchronize() + + # Update manual CSV file + if not isinstance(mo, type(None)): + if production: + mo.update_dataframe(netid, uaid, group, stem) + else: + log.info("Working with figtest stem. Not updating dataframe") + else: + log.info('dry run, not performing synchronization') + log.info('dry run, not updating dataframe') + + return d diff --git a/requiam/grouper_query.py b/requiam/grouper_query.py deleted file mode 100644 index 019d6b5f..00000000 --- a/requiam/grouper_query.py +++ /dev/null @@ -1,164 +0,0 @@ -import requests - -from os.path import join -from .delta import Delta -from .manual_override import update_entries -from .commons import figshare_stem -from .logger import log_stdout - - -class GrouperQuery(object): - """ - Purpose: - This class initializes an HTTP request to retrieve Grouper membership. - This code was adapted from the following repository: - https://github.com/ualibraries/patron-groups - - Usage: - Quick how to: - from requiam import grouper_query - grouper_host = 'grouper.iam.arizona.edu' - grouper_base_path = 'grouper-ws/servicesRest/json/v2_2_001/groups' - grouper_group = 'arizona.edu:dept:LBRY:figshare:portal:sci_math' - gq = grouper_query.GrouperQuery(grouper_host, grouper_base_path, - USERNAME, PASSWORD, grouper_group) - - You can retrieve the EDS uid via: - members = gq.members - """ - - def __init__(self, grouper_host, grouper_base_path, grouper_user, - grouper_password, grouper_group, log=None): - - if isinstance(log, type(None)): - self.log = log_stdout() - else: - self.log = log - - self.grouper_host = grouper_host - self.grouper_base_dn = grouper_base_path - self.grouper_user = grouper_user - self.grouper_password = grouper_password - self.grouper_group = grouper_group - - self.endpoint = f'https://{grouper_host}/{grouper_base_path}' - - self.grouper_group_members_url = join(self.endpoint, - f'groups/{grouper_group}/members') - - rsp = requests.get(self.grouper_group_members_url, auth=(grouper_user, grouper_password)) - - if 'wsSubjects' in rsp.json()['WsGetMembersLiteResult']: - self._members = {s['id'] for s in rsp.json()['WsGetMembersLiteResult']['wsSubjects']} - else: - self._members = [] - - return - - @property - def members(self): - return set(self._members) - - -def figshare_group(group, root_stem, production=True): - """ - Purpose: - Construct Grouper figshare groups - - :param group: str or int of group name. Cannot be empty - :param root_stem: str of associated stem folder for [group] - :param production: Bool to use production stem. Otherwise a stage/test is used. Default: True - - :return grouper_group: str containing full Grouper path - - Usage: - For active group, call as: figshare_group('active', '') - > 'arizona.edu:dept:LBRY:figshare:active' - - For a quota group, call as: figshare_group('2147483648', 'quota') - > 'arizona.edu:dept:LBRY:figshare:quota:2147483648' - Note: group can be specified as an integer for quota cases - - For a portal group, call as: figshare_group('sci_math', 'portal') - > 'arizona.edu:dept:LBRY:figshare:portal:sci_math' - """ - - if not group: - raise ValueError("WARNING: Empty [group]") - - stem_query = figshare_stem(stem=root_stem, production=production) - - grouper_group = f'{stem_query}:{group}' - - return grouper_group - - -def grouper_delta_user(group, stem, netid, uaid, action, grouper_dict, - delta_dict, mo=None, sync=False, log=None, production=True): - """ - Purpose: - Construct a Delta object for addition/deletion based for a specified - user. This is designed primarily for the user_update script - - :param group: str - The Grouper group to update - :param stem: str - The Grouper stem (e.g., 'portal', 'quota') - :param netid: str - The User NetID - :param uaid: str - The User UA ID - :param action: str - The action to perform. 'add' or 'remove' - :param grouper_dict: dict - Dictionary containing grouper settings - :param delta_dict: - Dictionary containing delta settings - :param mo: ManualOverride object - For implementing change to CSV files. Default: None - :param sync: bool - Indicate whether to sync. Default: False - :param log: LogClass object - For logging - :param production: Bool to use production stem. Otherwise a stage/test is used. Default: True - - :return d: Delta object class - """ - - if isinstance(log, type(None)): - log = log_stdout() - - grouper_query = figshare_group(group, stem, production=production) - gq = GrouperQuery(**grouper_dict, grouper_group=grouper_query) - - member_set = gq.members - if not isinstance(netid, list): - netid = [netid] - if not isinstance(uaid, list): - uaid = [uaid] - member_set = update_entries(member_set, netid, uaid, action, log=log) - - d = Delta(ldap_members=member_set, - grouper_query_instance=gq, - **delta_dict, - log=log) - - log.info(f"ldap and grouper have {len(d.common)} members in common") - log.info(f"synchronization will drop {len(d.drops)} entries to Grouper {group} group") - log.info(f"synchronization will add {len(d.adds)} entries to Grouper {group} group") - - if sync: - log.info('synchronizing ...') - d.synchronize() - - # Update manual CSV file - if not isinstance(mo, type(None)): - if production: - mo.update_dataframe(netid, uaid, group, stem) - else: - log.info("Working with figtest stem. Not updating dataframe") - else: - log.info('dry run, not performing synchronization') - log.info('dry run, not updating dataframe') - - return d diff --git a/requiam/ldap_query.py b/requiam/ldap_query.py index 743cf7ec..94bc12a9 100644 --- a/requiam/ldap_query.py +++ b/requiam/ldap_query.py @@ -41,9 +41,9 @@ def __init__(self, ldap_host, ldap_base_dn, ldap_user, ldap_password, log=None): self.ldap_user = ldap_user self.ldap_password = ldap_password - self.ldap_bind_host = 'ldaps://' + ldap_host - self.ldap_bind_dn = 'uid=' + ldap_user + ',ou=app users,' + ldap_base_dn - self.ldap_search_dn = 'ou=people,' + ldap_base_dn + self.ldap_bind_host = f"ldaps://{ldap_host}" + self.ldap_bind_dn = f"uid={ldap_user},ou=app users,{ldap_base_dn}" + self.ldap_search_dn = f"ou=people,{ldap_base_dn}" self.ldap_attribs = ['uaid'] # @@ -68,7 +68,7 @@ def uid_query(uid): :return ldap_query: list containing the str """ - ldap_query = '(uid={})'.format(uid) + ldap_query = f"(uid={uid})" return [ldap_query] @@ -96,7 +96,7 @@ def ual_grouper_base(basename): :return: str with ismemberof attribute """ - return 'ismemberof=arizona.edu:dept:LBRY:pgrps:{}'.format(basename) + return f"ismemberof=arizona.edu:dept:LBRY:pgrps:{basename}" def ual_ldap_query(org_code, classification='all'): @@ -126,9 +126,9 @@ def ual_ldap_query(org_code, classification='all'): """ if classification == 'none': - ldap_query = '(employeePrimaryDept={})'.format(org_code) + ldap_query = f"(employeePrimaryDept={org_code})" else: - ldap_query = '(& (employeePrimaryDept={}) (| '.format(org_code) + ldap_query = f"(& (employeePrimaryDept={org_code}) (| " classification_list = ['all', 'faculty', 'staff', 'students', 'dcc'] if classification not in classification_list: @@ -136,11 +136,13 @@ def ual_ldap_query(org_code, classification='all'): if classification == 'all': for member in classification_list[1:]: - ldap_query += '({}) '.format(ual_grouper_base(f'ual-{member}')) + group_str = ual_grouper_base(f"ual-{member}") + ldap_query += f"({group_str}) " else: - ldap_query += '({}) '.format(ual_grouper_base(f'ual-{classification}')) + group_str = ual_grouper_base(f"ual-{classification}") + ldap_query += f"({group_str}) " - ldap_query += ') )' + ldap_query += ") )" return [ldap_query] diff --git a/requiam/org_code_numbers.py b/requiam/org_code_numbers.py index 4cb3278c..a1ee8b48 100644 --- a/requiam/org_code_numbers.py +++ b/requiam/org_code_numbers.py @@ -44,10 +44,10 @@ def get_numbers(lc, org_url, log_func): lib_dcc = np.zeros(n_org_codes, dtype=int) # Query based on Library patron group for set logic - faculty_query = ['({})'.format(ual_grouper_base('ual-faculty'))] - staff_query = ['({})'.format(ual_grouper_base('ual-staff'))] - student_query = ['({})'.format(ual_grouper_base('ual-students'))] - dcc_query = ['({})'.format(ual_grouper_base('ual-dcc'))] + faculty_query = [f"({ual_grouper_base('ual-faculty')})"] + staff_query = [f"({ual_grouper_base('ual-staff')})"] + student_query = [f"({ual_grouper_base('ual-students')})"] + dcc_query = [f"({ual_grouper_base('ual-dcc')})"] log_func.info("Getting faculty, staff, student, and dcc members ... ") faculty_members = ldap_search(lc, faculty_query) @@ -59,7 +59,7 @@ def get_numbers(lc, org_url, log_func): for org_code, ii in zip(org_codes, range(n_org_codes)): if ii % round(n_org_codes/10) == 0 or ii == n_org_codes-1: - log_func.info("{0: >3}% completed ...".format(round((ii+1)/n_org_codes * 100))) + log_func.info(f"{round((ii + 1) / n_org_codes * 100): >3}% completed ...") total_members = ldap_search(lc, ual_ldap_query(org_code, classification='none')) diff --git a/requirements.txt b/requirements.txt index cc1f7def..f49828b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pandas==1.1.0 +pandas==1.2.1 tabulate==0.8.3 ldap3==2.6.1 requests==2.22.0 -numpy==1.18.0 \ No newline at end of file +numpy==1.19.5 \ No newline at end of file diff --git a/scripts/add_grouper_groups b/scripts/add_grouper_groups index 32c7ec3e..216289ff 100755 --- a/scripts/add_grouper_groups +++ b/scripts/add_grouper_groups @@ -8,7 +8,7 @@ from datetime import date import argparse -from requiam.grouper_admin import GrouperAPI, create_groups +from requiam.grouper import Grouper, create_groups from requiam.logger import LogClass, get_user_hostname, log_settings from requiam import TimerClass from requiam.commons import dict_load @@ -107,8 +107,8 @@ if __name__ == '__main__': else: grouper_production = False - ga = GrouperAPI(**grouper_dict, grouper_production=grouper_production, - log=log) + ga = Grouper(**grouper_dict, grouper_production=grouper_production, + log=log) # Main portals / Overall Research Themes if extras_dict['main_themes']: diff --git a/scripts/script_run b/scripts/script_run index 4cb3d240..1a214672 100755 --- a/scripts/script_run +++ b/scripts/script_run @@ -12,14 +12,13 @@ import argparse import ast from requiam import ldap_query -from requiam.grouper_query import GrouperQuery, figshare_group -from requiam.grouper_admin import GrouperAPI, create_active_group +from requiam.grouper import Grouper, create_active_group from requiam import delta from requiam import quota from requiam.logger import LogClass, get_user_hostname, pandas_write_buffer, log_settings from requiam import TimerClass from requiam import manual_override -from requiam.commons import dict_load, get_summary_dict +from requiam.commons import dict_load, get_summary_dict, figshare_group # Version and branch info from requiam import __version__ @@ -159,7 +158,7 @@ if __name__ == '__main__': # This is for checking whether the group exists grouper_production = True if not extras_dict['grouper_figtest'] else False - ga = GrouperAPI(**grouper_dict, grouper_production=grouper_production, log=log) + ga = Grouper(**grouper_dict, grouper_production=grouper_production, log=log) summary_dict = dict() # Initialize @@ -221,12 +220,12 @@ if __name__ == '__main__': grouper_portal = figshare_group(group_name, 'group_active', production=False) log.info(f"Grouper group : {grouper_portal}") - gq = GrouperQuery(**grouper_dict, grouper_group=grouper_portal, - log=log) - log.info(f" Grouper size {len(gq.members)}") + + grouper_query_dict = ga.query(grouper_portal) + log.info(f" Grouper size {len(grouper_query_dict['members'])}") d = delta.Delta(ldap_members=ldap_members, - grouper_query_instance=gq, + grouper_query_dict=grouper_query_dict, **delta_dict, log=log) @@ -285,22 +284,23 @@ if __name__ == '__main__': grouper_portal = figshare_group(portal, 'portal', production=grouper_production) log.info(f"Grouper group : {grouper_portal}") - gq = GrouperQuery(**grouper_dict, grouper_group=grouper_portal, - log=log) - log.info(f" Grouper size {len(gq.members)}") + grouper_query_dict = ga.query(grouper_portal) + log.info(f" Grouper size {len(grouper_query_dict['members'])}") # For --org_codes or --groups, only add users if not isinstance(org_codes, type(None)): log.info("Special mode with --org_codes --groups. Adding users only") # Combine grouper members with new ldap members - ldap_members = set.union(gq.members, ldap_members) + ldap_members = set.union(grouper_query_dict['members'], ldap_members) d = delta.Delta(ldap_members=ldap_members, - grouper_query_instance=gq, + grouper_query_dict=grouper_query_dict, **delta_dict, log=log) - summary_dict[portal] = get_summary_dict(ldap_members, gq.members, d) + summary_dict[portal] = \ + get_summary_dict(ldap_members, grouper_query_dict['members'], + d) log.info(f"ldap and grouper have {len(d.common)} members in common") log.info(f"synchronization will drop {len(d.drops)} entries from grouper group") @@ -359,22 +359,25 @@ if __name__ == '__main__': # Grouper query grouper_quota = figshare_group(q, 'quota', production=grouper_production) log.info(f"Grouper group : {grouper_quota}") - gq = GrouperQuery(**grouper_dict, grouper_group=grouper_quota, log=log) - log.info(f" Grouper size {len(gq.members)}") + grouper_query_dict = ga.query(grouper_quota) + log.info(f" Grouper size {len(grouper_query_dict['members'])}") # For --org_codes or --groups, only add users if not isinstance(org_codes, type(None)): log.info("Special mode with --org_codes --groups. Adding users only") # Combine grouper members with new ldap members - ldap_members = set.union(gq.members, ldap_members) + ldap_members = set.union(grouper_query_dict['members'], + ldap_members) # Delta between LDAP and Grouper d = delta.Delta(ldap_members=ldap_members, - grouper_query_instance=gq, + grouper_query_dict=grouper_query_dict, **delta_dict, log=log) - summary_dict[q] = get_summary_dict(ldap_members, gq.members, d) + summary_dict[q] = \ + get_summary_dict(ldap_members, + grouper_query_dict['members'], d) log.info(f"ldap and grouper have {len(d.common)} members in common") log.info(f"synchronization will drop {len(d.drops)} entries from grouper group") @@ -419,11 +422,12 @@ if __name__ == '__main__': # Note testing is only performed on figshare:test and not figtest:test grouper_test = figshare_group('test', '') - gq = GrouperQuery(**grouper_dict, grouper_group=grouper_test, log=log) - log.info(f" Grouper size {len(gq.members)}") + grouper_query_dict = ga.query(grouper_test) + log.info(f" Grouper size {len(grouper_query_dict['members'])}") # Delta between LDAP and Grouper - d = delta.Delta(ldap_members=ldap_members, grouper_query_instance=gq, + d = delta.Delta(ldap_members=ldap_members, + grouper_query_dict=grouper_query_dict, **delta_dict, log=log) log.info(f"ldap and grouper have {len(d.common)} members in common") diff --git a/scripts/user_update b/scripts/user_update index 5678bed4..aafb001e 100755 --- a/scripts/user_update +++ b/scripts/user_update @@ -9,11 +9,10 @@ import argparse from requiam import ldap_query from requiam.commons import dict_load, figshare_stem -from requiam.grouper_query import grouper_delta_user from requiam.logger import LogClass, get_user_hostname, log_settings from requiam import TimerClass from requiam.manual_override import ManualOverride, get_current_groups -from requiam.grouper_admin import GrouperAPI, create_active_group +from requiam.grouper import Grouper, create_active_group, grouper_delta_user # Version and branch info from requiam import __version__ @@ -164,8 +163,8 @@ if __name__ == '__main__': # Grouper API tool grouper_production = True if not extras_dict['grouper_figtest'] else False - ga = GrouperAPI(**grouper_dict, log=log, - grouper_production=grouper_production) + ga = Grouper(**grouper_dict, log=log, + grouper_production=grouper_production) # Check to see if portal exists on Grouper before proceeding portal_check = True diff --git a/setup.py b/setup.py index a07b85e2..eb2e766c 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='requiam', - version='v0.15.1', + version='v0.16.0', packages=['requiam'], url='https://github.com/ualibraries/ReQUIAM', license='MIT License', diff --git a/tests/test_commons.py b/tests/test_commons.py index e799fa99..3370f928 100644 --- a/tests/test_commons.py +++ b/tests/test_commons.py @@ -1,5 +1,5 @@ from requiam import TimerClass -from requiam.commons import figshare_stem +from requiam.commons import figshare_stem, figshare_group from datetime import datetime, timedelta @@ -41,3 +41,19 @@ def test_figshare_stem(): f"{prod_stem}:{stem}" assert figshare_stem(stem=stem, production=False) == \ f"{stage_stem}:{stem}" + + +def test_figshare_group(): + + for group in ['astro', 'sci_math', 'test']: + f_group = figshare_group(group, 'portal', production=False) + assert f_group == f"{stage_stem}:portal:{group}" + + f_group = figshare_group(group, 'portal', production=True) + assert f_group == f"{prod_stem}:portal:{group}" + + for quota in [104857600, 536870912, 2147483648]: + f_group = figshare_group(quota, 'quota', production=False) + assert f_group == f"{stage_stem}:quota:{quota}" + f_group = figshare_group(quota, 'quota', production=True) + assert f_group == f"{prod_stem}:quota:{quota}" diff --git a/tests/test_grouper_query.py b/tests/test_grouper_query.py deleted file mode 100644 index d0655081..00000000 --- a/tests/test_grouper_query.py +++ /dev/null @@ -1,20 +0,0 @@ -from requiam.grouper_query import figshare_group - -prod_stem = 'arizona.edu:dept:LBRY:figshare' -stage_stem = 'arizona.edu:dept:LBRY:figtest' - - -def test_figshare_group(): - - for group in ['astro', 'sci_math', 'test']: - f_group = figshare_group(group, 'portal', production=False) - assert f_group == f"{stage_stem}:portal:{group}" - - f_group = figshare_group(group, 'portal', production=True) - assert f_group == f"{prod_stem}:portal:{group}" - - for quota in [104857600, 536870912, 2147483648]: - f_group = figshare_group(quota, 'quota', production=False) - assert f_group == f"{stage_stem}:quota:{quota}" - f_group = figshare_group(quota, 'quota', production=True) - assert f_group == f"{prod_stem}:quota:{quota}"