diff --git a/update_items.py b/update_items.py index 3adaa0068c..aeacaed2a7 100644 --- a/update_items.py +++ b/update_items.py @@ -12,19 +12,21 @@ """ -import os -import sys import argparse -import traceback import json -import re import logging -log = logging.getLogger(__name__) +import os +import re +import sys +import traceback -import yaml import nbformat -from nbconvert import HTMLExporter +import requests +import yaml from arcgis.gis import GIS +from nbconvert import HTMLExporter + +log = logging.getLogger(__name__) ITEMS_METADATA_YAML_PATH = os.path.join(".", "items_metadata.yaml") THUMBNAILS_DIR = os.path.join(".", "static", "thumbnails") @@ -35,21 +37,27 @@ NB_PORTAL_TYPE_KEYWORDS = "Notebook, Python" NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED = \ {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced', - 'notebookRuntimeVersion': '5.0'} + 'notebookRuntimeVersion': ''} NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED_GPU = \ - {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support', - 'notebookRuntimeVersion': '5.0'} + {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support', + 'notebookRuntimeVersion': ''} NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD = \ {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Standard', - 'notebookRuntimeVersion': '5.0'} + 'notebookRuntimeVersion': ''} NB_ITEM_FOLDER = "Notebook Samples" + def _main(): """Parses arguments, connects to GIS, reads YAML, uploads NBs""" args = _parse_cmd_line_args() _setup_logging(args) gis = GIS(args.portal_url, args.username, args.password, verify_cert=False) + if args.version: + NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = args.version + NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = args.version + else: + _get_current_runtime(gis) items_metadata_yaml = _read_items_metadata_yaml() if args.replace_profiles: _replace_profiles() @@ -58,51 +66,69 @@ def _main(): if s.failed_uploads: raise Exception(f"Some uploads failed: {s.failed_uploads}") + def _parse_cmd_line_args(): """Parse CMD args, returns an object instance of all user passed in args""" - parser = argparse.ArgumentParser(description = "Takes all notebooks "\ - "this in `gallery` directory, and will upload it to the specified "\ - "portal/org in the right group with the right categories. "\ - "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)", - formatter_class=argparse.RawTextHelpFormatter) + parser = argparse.ArgumentParser(description="Takes all notebooks " + "this in `gallery` directory, and will upload it to the specified " + "portal/org in the right group with the right categories. " + "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)", + formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--username", "-u", type=str, - help="Required username for the portal/org") + help="Required username for the portal/org") parser.add_argument("--password", "-p", type=str, - help="Required password for the portal/org") + help="Required password for the portal/org") parser.add_argument("--portal-url", "-r", type=str, - help="The portal to connect to (Default:geosaurus.maps.arcgis.com)", - default="https://geosaurus.maps.arcgis.com/") + help="The portal to connect to (default: geosaurus.maps.arcgis.com)", + default="https://geosaurus.maps.arcgis.com/") + parser.add_argument("--version", "-ver", type=str, + help="The version of notebook runtime to set on sample notebooks " + "(default: the latest version)", + default="") parser.add_argument("--verbose", "-v", action="store_true", - help="Print all DEBUG log messages instead of just INFO") + help="Print all DEBUG log messages instead of just INFO") parser.add_argument("--replace-profiles", "-c", action="store_true", - help="Replace all profiles in notebooks with their appropriate username "\ - "and passwords. Does this by running misc/tools/replace_profiles.py") - args = parser.parse_args(sys.argv[1:]) #don't use filename as 1st arg + help="Replace all profiles in notebooks with their appropriate username " + "and passwords. Does this by running misc/tools/replace_profiles.py") + args = parser.parse_args(sys.argv[1:]) # don't use filename as 1st arg return args + def _setup_logging(args): """Sets up the logging based on args""" if args.verbose: log.setLevel(logging.DEBUG) else: - log.setLevel(logging.INFO) + log.setLevel(logging.INFO) stdout_handler = logging.StreamHandler(stream=sys.stdout) stdout_handler.setLevel(logging.DEBUG) stdout_handler.setFormatter(logging.Formatter( - '----- %(levelname)s | '\ - '%(asctime)s | '\ - '%(filename)s line %(lineno)d'\ - ' -----\n'\ + '----- %(levelname)s | ' + '%(asctime)s | ' + '%(filename)s line %(lineno)d' + ' -----\n' '"%(message)s"')) log.addHandler(stdout_handler) log.info("Logging at level {}.".format(logging.getLevelName(log.level))) log.debug("args passed in => {}".format(args)) + +def _get_current_runtime(gis): + ntbk_svr = gis.notebook_server[0] + rest = ntbk_svr._url.replace("/admin", "/rest") + rest_info = requests.get(f"{rest}/info?f=json", timeout=5, verify=False) + version = rest_info.json()["currentRuntimeVersion"] + NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = version + NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = version + return version + + def _read_items_metadata_yaml(): """Returns the items_metadata.yaml file as a dict""" - with open(ITEMS_METADATA_YAML_PATH) as f: + with open(ITEMS_METADATA_YAML_PATH, encoding="utf-8") as f: return yaml.safe_load(f) + def _replace_profiles(): """Runs misc/tools/replace_profiles.py to go through each notebook in the repo and replace profiles with usernames/passwords @@ -110,48 +136,50 @@ def _replace_profiles(): cmd = f"{sys.executable} {REPLACE_PROFILES_SCRIPT}" os.system(cmd) + class ItemsUploader: def __init__(self, gis, items_metadata_yaml): self._gis = gis self._items_metadata_yaml = items_metadata_yaml self.failed_uploads = [] - def upload_items(self, share_after_upload = True): + def upload_items(self, share_after_upload=True): for entry in self._items_metadata_yaml["samples"] + \ - self._items_metadata_yaml["guides"] + \ - self._items_metadata_yaml["labs"]: + self._items_metadata_yaml["guides"] + \ + self._items_metadata_yaml["labs"]: self._stage_and_upload_item(entry, share_after_upload) - def _stage_and_upload_item(self, entry, share_after_upload = True): + def _stage_and_upload_item(self, entry, share_after_upload=True): log.info(f"Uploading {entry['title']}") log.debug(f" sample: {entry}") try: nb_path = entry["path"] self._preupload_check(entry['title'], nb_path) - runtime_stamp = self._infer_runtime_stamp(entry.get("runtime", "standard")) + runtime_stamp = self._infer_runtime_stamp( + entry.get("runtime", "standard")) categories = entry.get("categories", None) self._stamp_file_with_runtime(nb_path, runtime_stamp) item_id = self._infer_item_id(entry["url"]) item = self.update_item( - item_id = item_id, - item_type = NB_PORTAL_TYPE, - item_type_keywords = NB_PORTAL_TYPE_KEYWORDS, - title = entry['title'], - categories = categories, - snippet = entry['snippet'], - description = entry['description'], - license_info = entry['licenseInfo'], - tags = entry['tags'], - nb_path = nb_path, - runtime_stamp = runtime_stamp, - thumbnail = entry['thumbnail']) + item_id=item_id, + item_type=NB_PORTAL_TYPE, + item_type_keywords=NB_PORTAL_TYPE_KEYWORDS, + title=entry['title'], + categories=categories, + snippet=entry['snippet'], + description=entry['description'], + license_info=entry['licenseInfo'], + tags=entry['tags'], + nb_path=nb_path, + runtime_stamp=runtime_stamp, + thumbnail=entry['thumbnail']) if share_after_upload: - item.share(everyone = True) + item.sharing.sharing_level = "EVERYONE" item.protect() if categories: self._assign_categories_to_item(item, categories) self._apply_html_preview_to_item(item, nb_path) - log.info(f" Uploaded succeded -> {item.homepage}") + log.info(f" Uploaded succeeded -> {item.homepage}") except Exception as e: self.failed_uploads.append(entry['title']) log.warn(f" Couldn't upload {entry['title']}: {e}") @@ -170,11 +198,11 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories, snippet, description, license_info, tags, nb_path, runtime_stamp, thumbnail): """Actually uploads the notebook item to the portal""" - item_properties = {"title" : title, - "snippet" : snippet, - "description" : description, - "licenseInfo" : license_info, - "tags" : tags, + item_properties = {"title": title, + "snippet": snippet, + "description": description, + "licenseInfo": license_info, + "tags": tags, "properties": runtime_stamp} if categories: item_properties["categories"] = categories @@ -188,11 +216,12 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories, log.debug(f'item {existing_item.homepage} exists, updating...') item_properties["url"] = existing_item.homepage existing_item.update(item_properties, - data = nb_path, - thumbnail = thumbnail) + data=nb_path, + thumbnail=thumbnail) resp = existing_item else: - raise Exception(f"Could not find item {item_id} to update. Failing!") + raise Exception( + f"Could not find item {item_id} to update. Failing!") return resp def _assign_categories_to_item(self, item, categories): @@ -207,8 +236,8 @@ def _apply_html_preview_to_item(self, item, nb_path): json_file_name = "notebook_preview.json" json_file_path = os.path.join(".", json_file_name) - with open(json_file_path, 'w') as f: - json.dump({"html" : html_str}, f) + with open(json_file_path, 'w', encoding="utf-8") as f: + json.dump({"html": html_str}, f) if item.resources.list(): item.resources.remove() @@ -239,11 +268,12 @@ def _stamp_file_with_runtime(self, notebook_file_path, runtime_stamp): nb['metadata']['esriNotebookRuntime'] = runtime_stamp nbformat.write(nb, notebook_file_path, nbformat.NO_CONVERT) + if __name__ == "__main__": try: _main() sys.exit(0) except Exception as e: log.exception(e) - log.info("Program did not succesfully complete (unhandled exception)") + log.info("Program did not successfully complete (unhandled exception)") sys.exit(1)