From f17029f3d0f9d3dcd09062dbdac30eef0fdaad22 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Thu, 29 Sep 2022 16:28:15 -0500 Subject: [PATCH 1/6] track playbook (#31) * track playbook this assumes the url is /playbook/joplin/..... and will record the playbook in a field called playbook, that is tracked as part of influxdb and can be shown on grafana. * updated changelog Co-authored-by: Yong Wook Kim --- CHANGELOG.md | 5 +++++ incore_auth/app.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a27945..582b4d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth. +# [Unreleased] + +## Changed +- Added playbook and its sub directories to tracking resource [#32](https://github.com/IN-CORE/incore-auth/issues/32) + # [1.5.0] - 2022-09-24 ## Changed diff --git a/incore_auth/app.py b/incore_auth/app.py index d5d5ba0..812593c 100644 --- a/incore_auth/app.py +++ b/incore_auth/app.py @@ -200,6 +200,8 @@ def request_resource(request_info): request_info['resource'] = pieces[1] if request_info['resource'] == "doc" and len(pieces) > 2: request_info['fields']['manual'] = pieces[2] + if request_info['resource'] == "playbook" and len(pieces) > 2: + request_info['fields']['playbook'] = pieces[2] if request_info['resource'] == "data" and len(pieces) > 4 and uri.endswith('blob'): request_info['fields']['dataset'] = pieces[4] if request_info['resource'] == "dfr3" and len(pieces) > 4: From 3d1703fcad11c8e81c37146625469a8c7dded034 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 6 Dec 2022 14:07:38 -0600 Subject: [PATCH 2/6] Sync user (#33) * update packages * add function to sync user+groups * add user to datawolf fix requirements file cleanup dockerfile * fix pep8 * fix build no arm builds for now * update actions * remove output --- .github/workflows/docker.yml | 19 +++--- .github/workflows/release.yaml | 6 +- CHANGELOG.md | 6 +- Dockerfile | 33 ++-------- incore_auth/app.py | 111 +++++++++++++++++++++++++++++++++ incore_auth/requirements.txt | 20 +++--- 6 files changed, 145 insertions(+), 50 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e71b162..28c0c87 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,8 @@ env: MAIN_REPO: IN-CORE/incore-auth DOCKERHUB_ORG: incore NCSAHUB: hub.ncsa.illinois.edu/incore - PLATFORM: "linux/amd64,linux/arm64" + #PLATFORM: "linux/amd64,linux/arm64" missing pre-compiled packages + PLATFORM: "linux/amd64" jobs: docker: @@ -22,7 +23,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # calculate some variables that are used later - name: version information @@ -145,7 +146,7 @@ jobs: # build the docker images - name: Build and push docker - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: push: true platforms: ${{ env.PLATFORM }} @@ -161,9 +162,9 @@ jobs: # this will update the README of the dockerhub repo - name: Docker Hub Description if: env.DOCKERHUB_README == 'true' - uses: peter-evans/dockerhub-description@v2 - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }} - README_FILEPATH: README.md + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKERHUB_ORG }}/${{ github.event.repository.name }} + readme-filepath: README.md diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f3426a2..0fda39b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -27,8 +27,8 @@ jobs: changelog="${changelog//'%'/'%25'}" changelog="${changelog//$'\n'/'%0A'}" changelog="${changelog//$'\r'/'%0D'}" - echo "::set-output name=version::$version" - echo "::set-output name=changelog::$changelog" + echo "version=$version" >> $GITHUB_OUTPUT + echo "changelog=$changelog" >> $GITHUB_OUTPUT - name: create release if: github.event_name != 'pull_request' && github.repository == env.MAIN_REPO diff --git a/CHANGELOG.md b/CHANGELOG.md index 582b4d7..1a2aefa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth. -# [Unreleased] +# Unreleased + +## Added +- information about user and groups is synced every 30 minutes back to the database and datawolf ## Changed - Added playbook and its sub directories to tracking resource [#32](https://github.com/IN-CORE/incore-auth/issues/32) +- updated all packages used # [1.5.0] - 2022-09-24 diff --git a/Dockerfile b/Dockerfile index e7a7631..68e2d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,14 @@ -FROM alpine:3.7 +FROM python:3.7-alpine MAINTAINER Incore LABEL PROJECT_REPO_URL = "" \ PROJECT_REPO_BROWSER_URL = "" \ - DESCRIPTION = ")" - -RUN apk add --no-cache \ - gcc \ - g++ \ - libffi-dev \ - make \ - python3 \ - python3-dev \ - openssl-dev && \ - python3 -m ensurepip && \ - rm -r /usr/lib/python*/ensurepip && \ - pip3 install --upgrade pip setuptools && \ - if [[ ! -e /usr/bin/pip ]]; then ln -s pip3 /usr/bin/pip; fi && \ - if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ - rm -r /root/.cache + DESCRIPTION = ")" WORKDIR /srv COPY incore_auth/requirements.txt incore_auth/ -RUN pip install -Ur incore_auth/requirements.txt +RUN pip3 install -Ur incore_auth/requirements.txt COPY incore_auth incore_auth @@ -32,19 +17,11 @@ WORKDIR /srv/incore_auth ENV FLASK_APP="app.py" \ KEYCLOAK_PUBLIC_KEY="" \ KEYCLOAK_AUDIENCE="" \ + DATAWOLF_URL="http://incore-datawolf:8888/datawolf" \ + MONGODB_URI="" \ INFLUXDB_V2_URL="" \ INFLUXDB_V2_ORG="" \ INFLUXDB_V2_TOKEN="" \ INFLUXDB_V2_FILE_LOCATION="data/IP2LOCATION-LITE-DB5.BIN" -#CMD ["python", "-m", "flask", "run", "--host", "0.0.0.0"] CMD ["gunicorn", "app:app", "--config", "/srv/incore_auth/gunicorn.config.py"] - -#ENTRYPOINT ["gunicorn", \ -# "--access-logfile=-", \ -# "--log-level=info", \ -# "--workers=3", \ -# "--bind=0.0.0.0:5000", \ -# "incore_auth.app:app" \ -# ] - diff --git a/incore_auth/app.py b/incore_auth/app.py index 812593c..9ae3309 100644 --- a/incore_auth/app.py +++ b/incore_auth/app.py @@ -2,11 +2,15 @@ import json import os import time +import threading import urllib.request import IP2Location import geohash2 import influxdb_client +import pymongo + +from cachetools import cached, TTLCache from flask import Flask, request, Response, make_response, json from jose import jwt @@ -25,6 +29,10 @@ geoserver = {} geoserver_delta = 2 +cache_size = 1024 +# timeout in seconds, in this case 30 minutes +cache_timeout = 30*60 + # setup database for geolocation try: geolocation = IP2Location.IP2Location(CONTRIBUTION_DB_NAME) @@ -40,6 +48,87 @@ app.logger.setLevel(gunicorn_logger.level) +def cache_key(request_info): + """Return username from request_info to be used as cache key""" + return request_info["username"] + + +def update_services_thread(request_info): + """When a user does any action, it will check to update the groups in mongo, as well as make sure + the user has access to datawolf. The function is cached to make sure only every 30 minutes we do + the checks, since this can be expensive.""" + + # get information from request + username = request_info["username"] + if not username: + return username + groups = request_info['groups'] + + # call datawolf to add user + datawolf_url = config["datawolf_url"] + if datawolf_url: + query = urllib.parse.urlencode({ + "firstname": request_info["firstname"], + "lastname": request_info["lastname"], + "email": username + }) + datawolf_url = "%s/persons?%s" % (datawolf_url.rstrip("/"), query) + req = urllib.request.Request(datawolf_url, method='POST') + response = urllib.request.urlopen(req) + if response.code == 200: + app.logger.info(f"Added user to datawolf {username}") + elif response.code == 204: + app.logger.debug(f"User already exists in datawolf {username}") + else: + app.logger.info(f"Did not add user to datawolf {username}") + + # update database with user quota + mongo_client = config["mongo_client"] + if mongo_client: + mongo_user = mongo_client["spacedb"]["UserGroups"].find_one({"username": username}) + if not mongo_user: + # INSERT + mongo_client["spacedb"]["UserGroups"].insert_one({ + "username": username, + "className": "edu.illinois.ncsa.incore.common.models.UserGroups", + "groups": groups + }) + app.logger.info(f"Inserted groups document for {username}") + elif set(groups) != set(mongo_user["groups"]): + # UPDATE + mongo_client["spacedb"]["UserGroups"].update_one( + {"username": username}, {"$set": {"groups": groups}} + ) + app.logger.info(f"Synced groups for {username} - {groups}") + else: + # NOTHING + app.logger.debug(f"No sync needed for {username}") + + mongo_space = mongo_client["spacedb"]["Space"].find_one({"metadata.name": username}) + if not mongo_space: + mongo_client["spacedb"]["Space"].insert_one({ + "className": "edu.illinois.ncsa.incore.common.models.Space", + "metadata": { + "className": "edu.illinois.ncsa.incore.common.models.SpaceMetadata", + "name": username + }, + "privileges": { + "className": "edu.illinois.ncsa.incore.common.auth.Privileges", + "userPrivileges": { + username: "ADMIN" + } + }, + "members": [ + ] + }) + app.logger.info(f"Inserted space document for {username}") + + +@cached(cache=TTLCache(maxsize=cache_size, ttl=cache_timeout), key=cache_key) +def update_services(request_info): + threading.Thread(target=update_services_thread, args=(request_info,), daemon=True).start() + + def record_request(request_info): if 'X-Forwarded-For' not in request.headers: return @@ -173,6 +262,11 @@ def request_userinfo(request_info): request_info['error'] = 'JWT Error: invalid token' return + # get name of user + request_info["firstname"] = access_token["given_name"] + request_info["lastname"] = access_token["family_name"] + request_info["fullname"] = access_token["name"] + # retrieve the groups the user belongs to from access token request_info['username'] = access_token["preferred_username"] request_info['groups'] = access_token.get("groups", []) @@ -240,6 +334,9 @@ def verify_token(): # dict to hold all information request_info = { "username": "", + "firstname": "", + "lastname": "", + "fullname": "", "method": request.method, "url": request.path, "resource": "", @@ -255,6 +352,9 @@ def verify_token(): request_resource(request_info) request_userinfo(request_info) + # update backend services + update_services(request_info) + # record request record_request(request_info) @@ -342,6 +442,17 @@ def setup(): else: config['audience'] = None + # store datawolf url + config["datawolf_url"] = os.environ.get('DATAWOLF_URL', None) + + # setup mongodb + mongodb_uri = os.environ.get('MONGODB_URI', None) + if mongodb_uri: + mongo_client = pymongo.MongoClient(mongodb_uri) + config["mongo_client"] = mongo_client + else: + config["mongo_client"] = None + # setup influxdb try: client = influxdb_client.InfluxDBClient.from_env_properties() diff --git a/incore_auth/requirements.txt b/incore_auth/requirements.txt index f55e797..73e68bc 100644 --- a/incore_auth/requirements.txt +++ b/incore_auth/requirements.txt @@ -1,9 +1,11 @@ -bcrypt==3.1.0 -flask==0.12.2 -gunicorn==19.7.1 -gevent==1.4.0 -python-jose==3.1.0 -influxdb-client==1.14.0 -IP2Location==8.5.1 -geohash2==1.1 -python-dotenv==0.10.3 \ No newline at end of file +bcrypt==4.* +flask==2.* +gunicorn==20.* +gevent==22.* +python-jose==3.* +influxdb-client==1.* +IP2Location==8.* +geohash2==1.* +python-dotenv==0.* +pymongo==4.* +cachetools==4.* From 70bbd4a480e3765633e75ea9b9a0787610cd2f65 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 16 May 2023 15:11:53 -0500 Subject: [PATCH 3/6] 38 new users are not able to create data with incore due to an issue in allocation management (#39) * initialize the usage if user has no usage * changelog --- CHANGELOG.md | 6 ++++++ incore_auth/app.py | 19 +++++++++++++++++++ incore_auth/requirements.txt | 1 + 3 files changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a3006..0648ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth. +# [Unreleased] + +## Added +- New user default usage to zero [#38](https://github.com/IN-CORE/incore-auth/issues/38) + + # [1.6.0] - 2023-03-14 ## Added diff --git a/incore_auth/app.py b/incore_auth/app.py index 9ae3309..2df93db 100644 --- a/incore_auth/app.py +++ b/incore_auth/app.py @@ -18,6 +18,8 @@ from urllib.parse import unquote_plus from dotenv import load_dotenv +import bson + # Load .env file load_dotenv() CONTRIBUTION_DB_NAME = os.getenv('INFLUXDB_V2_FILE_LOCATION', 'data/IP2LOCATION-LITE-DB5.BIN') @@ -123,6 +125,23 @@ def update_services_thread(request_info): }) app.logger.info(f"Inserted space document for {username}") + mongo_usage = mongo_client["spacedb"]["UserAllocations"].find_one({"username": username}) + if not mongo_usage: + mongo_client["spacedb"]["UserAllocations"].insert_one({ + "className": "edu.illinois.ncsa.incore.common.models.UserAllocations", + "username": username, + "usage": { + "className": "edu.illinois.ncsa.incore.common.models.UserUsages", + "datasets": int(0), + "hazards": int(0), + "hazardDatasets": int(0), + "dfr3": int(0), + "datasetSize": bson.Int64(0), + "hazardDatasetSize": bson.Int64(0) + } + }) + app.logger.info(f"Inserted space document for {username}") + @cached(cache=TTLCache(maxsize=cache_size, ttl=cache_timeout), key=cache_key) def update_services(request_info): diff --git a/incore_auth/requirements.txt b/incore_auth/requirements.txt index 73e68bc..da11a1f 100644 --- a/incore_auth/requirements.txt +++ b/incore_auth/requirements.txt @@ -9,3 +9,4 @@ geohash2==1.* python-dotenv==0.* pymongo==4.* cachetools==4.* +bson==0.* From 2d50ee67bce05302a19376237496f524cdb29b6c Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 16 May 2023 16:01:39 -0500 Subject: [PATCH 4/6] remove bson, use pymongo bson (#40) --- incore_auth/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/incore_auth/requirements.txt b/incore_auth/requirements.txt index da11a1f..73e68bc 100644 --- a/incore_auth/requirements.txt +++ b/incore_auth/requirements.txt @@ -9,4 +9,3 @@ geohash2==1.* python-dotenv==0.* pymongo==4.* cachetools==4.* -bson==0.* From 671f901f6362e5749e46f78266ccd41262f441ea Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 13 Jun 2023 16:37:37 -0500 Subject: [PATCH 5/6] return userobject (#41) a new header X-Auth-User is returned that contains all the user information --- CHANGELOG.md | 1 + incore_auth/app.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0648ca4..f5ddf90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the do ## Added - New user default usage to zero [#38](https://github.com/IN-CORE/incore-auth/issues/38) +- Return a user object that contains username, fullname, email, groups and roles. # [1.6.0] - 2023-03-14 diff --git a/incore_auth/app.py b/incore_auth/app.py index 2df93db..1ee43d1 100644 --- a/incore_auth/app.py +++ b/incore_auth/app.py @@ -282,9 +282,10 @@ def request_userinfo(request_info): return # get name of user - request_info["firstname"] = access_token["given_name"] - request_info["lastname"] = access_token["family_name"] - request_info["fullname"] = access_token["name"] + request_info["firstname"] = access_token.get("given_name", "") + request_info["lastname"] = access_token.get("family_name", "") + request_info["fullname"] = access_token.get("name", "") + request_info["email"] = access_token.get("email", "") # retrieve the groups the user belongs to from access token request_info['username'] = access_token["preferred_username"] @@ -356,6 +357,7 @@ def verify_token(): "firstname": "", "lastname": "", "fullname": "", + "email": "", "method": request.method, "url": request.path, "resource": "", @@ -404,20 +406,24 @@ def verify_token(): # everything is ok user_info = {"preferred_username": request_info['username']} group_info = {"groups": request_info['groups']} + user_object = { + "username": request_info['username'], + "email": request_info['email'], + "fullname": request_info['fullname'], + "groups": request_info['groups'], + "roles": request_info['roles'], + } + response = Response(status=200) response.headers['X-Auth-UserInfo'] = json.dumps(user_info) response.headers['X-Auth-UserGroup'] = json.dumps(group_info) + response.headers['X-Auth-User'] = json.dumps(user_object) if request.headers.get('Authorization') is not None: response.headers['Authorization'] = unquote_plus(request.headers['Authorization']) elif request.cookies.get('Authorization') is not None: response.headers['Authorization'] = unquote_plus(request.cookies['Authorization']) - if request.headers.get('X-Auth-UserGroup') is not None: - response.headers['X-Auth-UserGroup'] = request.headers.get('X-Auth-UserGroup') - elif request.cookies.get('X-Auth-UserGroup') is not None: - response.headers['X-Auth-UserGroup'] = request.cookies['X-Auth-UserGroup'] - return response From fd3dc0832f6996f4d04bb1f517a15ec3e38dfc4b Mon Sep 17 00:00:00 2001 From: YONG WOOK KIM Date: Tue, 13 Jun 2023 16:40:19 -0500 Subject: [PATCH 6/6] release 1.7.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5ddf90..4f4745a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) From version 1.2.0 the file IP2LOCATION-LITE-DB5.BIN is no longer part of the docker image and will need to be downloaded (after registration) from [ip2location](https://lite.ip2location.com/database/ip-country?lang=en_US) and be placed in /srv/incore_auth. -# [Unreleased] +# [1.7.0] - 2023-06-14 ## Added - New user default usage to zero [#38](https://github.com/IN-CORE/incore-auth/issues/38)