Skip to content

Commit

Permalink
Merge pull request #15 from USEPA/owld_stubs
Browse files Browse the repository at this point in the history
merge for v1.1
  • Loading branch information
pauldzy authored Jan 6, 2025
2 parents eba2fbb + 52c58a9 commit b74ca7c
Show file tree
Hide file tree
Showing 128 changed files with 10,824 additions and 327 deletions.
12 changes: 7 additions & 5 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ Dockerfile.pg
Dockerfile.pr
Dockerfile.gs
*.gdb
*.ags
*.sde
ImportLog*
demo/arcgis/CIP-service/.backups/
demo/arcgis/CIP-service/GpMessages/
demo/arcgis/CIP-service/scratch/
demo/arcgis/CIP-service/Index/
/demo/jupyter/.ipynb_checkpoints
**/.backups/
**/GpMessages/
**/scratch/
**/Index/
**/jupyter/.ipynb_checkpoints
10 changes: 0 additions & 10 deletions admin/jupyter/notebooks/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,6 @@ def pg_restore(host_name,host_port,database_name,user_name,database_password,dum

return p.communicate(bytes('{}\n'.format(database_password),'utf-8'));

def ogr2ogr(cmdstring):

command = 'ogr2ogr {0}'.format(cmdstring);

command = shlex.split(command);

p = Popen(command,shell=False,stdin=PIPE,stdout=PIPE,stderr=PIPE);

return p.communicate();

def ogr2ogr(cmdstring):

command = 'ogr2ogr {0}'.format(cmdstring);
Expand Down
36 changes: 22 additions & 14 deletions admin/jupyter/notebooks/setup/git_checkout_cipsrv_engine.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"from IPython.display import display;\n",
"parser = argparse.ArgumentParser();\n",
"parser.add_argument(\"--override_branch\" ,required=False);\n",
"parser.add_argument(\"--use_existing_sql\",required=False);\n",
"args, _ = parser.parse_known_args();\n",
"\n",
"sys.path.append(os.path.join(os.path.expanduser('~'),'notebooks'));\n",
Expand Down Expand Up @@ -71,16 +72,23 @@
"metadata": {},
"outputs": [],
"source": [
"td = tempfile.TemporaryDirectory();\n",
"\n",
"repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name \n",
" ,depth = 1\n",
");\n",
"\n",
"print(\"checkout complete.\");"
"if args.use_existing_sql is not None:\n",
" trg = args.use_existing_sql;\n",
" print(\"using existing \" + trg);\n",
" \n",
"else:\n",
" td = tempfile.TemporaryDirectory();\n",
" \n",
" repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name\n",
" ,depth = 1\n",
" );\n",
" \n",
" print(\"checkout of \" + td.name + \" complete.\");\n",
" trg = os.path.join(td.name,depf);\n",
" "
]
},
{
Expand All @@ -89,10 +97,8 @@
"metadata": {},
"outputs": [],
"source": [
"trg = os.path.join(td.name,depf);\n",
"\n",
"if not os.path.exists(trg):\n",
" raise Exception(trg + ' file not found.');\n",
" raise Exception(trg + ' not found.');\n",
" \n",
"z = common.load_sqlfile(conn,trg,echo=True);\n"
]
Expand All @@ -104,7 +110,9 @@
"outputs": [],
"source": [
"conn.close();\n",
"td.cleanup();\n",
"\n",
"if args.use_existing_sql is None:\n",
" td.cleanup();\n",
"\n",
"print(\"logic load complete.\");"
]
Expand Down
36 changes: 22 additions & 14 deletions admin/jupyter/notebooks/setup/git_checkout_cipsrv_nhdplus_h.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"from IPython.display import display;\n",
"parser = argparse.ArgumentParser();\n",
"parser.add_argument(\"--override_branch\" ,required=False);\n",
"parser.add_argument(\"--use_existing_sql\",required=False);\n",
"args, _ = parser.parse_known_args();\n",
"\n",
"sys.path.append(os.path.join(os.path.expanduser('~'),'notebooks'));\n",
Expand Down Expand Up @@ -71,16 +72,23 @@
"metadata": {},
"outputs": [],
"source": [
"td = tempfile.TemporaryDirectory();\n",
"\n",
"repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name \n",
" ,depth = 1\n",
");\n",
"\n",
"print(\"checkout complete.\");"
"if args.use_existing_sql is not None:\n",
" trg = args.use_existing_sql;\n",
" print(\"using existing \" + trg);\n",
" \n",
"else:\n",
" td = tempfile.TemporaryDirectory();\n",
" \n",
" repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name\n",
" ,depth = 1\n",
" );\n",
" \n",
" print(\"checkout of \" + td.name + \" complete.\");\n",
" trg = os.path.join(td.name,depf);\n",
" "
]
},
{
Expand All @@ -89,10 +97,8 @@
"metadata": {},
"outputs": [],
"source": [
"trg = os.path.join(td.name,depf);\n",
"\n",
"if not os.path.exists(trg):\n",
" raise Exception(trg + ' file not found.');\n",
" raise Exception(trg + ' not found.');\n",
" \n",
"z = common.load_sqlfile(conn,trg,echo=True);\n"
]
Expand All @@ -104,7 +110,9 @@
"outputs": [],
"source": [
"conn.close();\n",
"td.cleanup();\n",
"\n",
"if args.use_existing_sql is None:\n",
" td.cleanup();\n",
"\n",
"print(\"logic load complete.\");"
]
Expand Down
36 changes: 22 additions & 14 deletions admin/jupyter/notebooks/setup/git_checkout_cipsrv_nhdplus_m.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"from IPython.display import display;\n",
"parser = argparse.ArgumentParser();\n",
"parser.add_argument(\"--override_branch\" ,required=False);\n",
"parser.add_argument(\"--use_existing_sql\",required=False);\n",
"args, _ = parser.parse_known_args();\n",
"\n",
"sys.path.append(os.path.join(os.path.expanduser('~'),'notebooks'));\n",
Expand Down Expand Up @@ -71,16 +72,23 @@
"metadata": {},
"outputs": [],
"source": [
"td = tempfile.TemporaryDirectory();\n",
"\n",
"repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name \n",
" ,depth = 1\n",
");\n",
"\n",
"print(\"checkout complete.\");"
"if args.use_existing_sql is not None:\n",
" trg = args.use_existing_sql;\n",
" print(\"using existing \" + trg);\n",
" \n",
"else:\n",
" td = tempfile.TemporaryDirectory();\n",
" \n",
" repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name\n",
" ,depth = 1\n",
" );\n",
" \n",
" print(\"checkout of \" + td.name + \" complete.\");\n",
" trg = os.path.join(td.name,depf);\n",
" "
]
},
{
Expand All @@ -89,10 +97,8 @@
"metadata": {},
"outputs": [],
"source": [
"trg = os.path.join(td.name,depf);\n",
"\n",
"if not os.path.exists(trg):\n",
" raise Exception(trg + ' file not found.');\n",
" raise Exception(trg + ' not found.');\n",
" \n",
"z = common.load_sqlfile(conn,trg,echo=True);\n"
]
Expand All @@ -104,7 +110,9 @@
"outputs": [],
"source": [
"conn.close();\n",
"td.cleanup();\n",
"\n",
"if args.use_existing_sql is None:\n",
" td.cleanup();\n",
"\n",
"print(\"logic load complete.\");"
]
Expand Down
149 changes: 149 additions & 0 deletions admin/jupyter/notebooks/setup/git_checkout_cipsrv_owld.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## git_checkout cipsrv_owld"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os,sys,git,tempfile,psycopg2,argparse;\n",
"from ipywidgets import IntProgress,HTML,VBox;\n",
"from IPython.display import display;\n",
"parser = argparse.ArgumentParser();\n",
"parser.add_argument(\"--override_branch\" ,required=False);\n",
"parser.add_argument(\"--use_existing_sql\",required=False);\n",
"args, _ = parser.parse_known_args();\n",
"\n",
"sys.path.append(os.path.join(os.path.expanduser('~'),'notebooks'));\n",
"import common;\n",
"\n",
"gith = os.environ['GITHUB_REPO_URL'];\n",
"# override: \"[email protected]:USEPA/CIPv2.git\"\n",
"if args.override_branch is not None:\n",
" brnh = args.override_branch;\n",
"else:\n",
" brnh = os.environ['GITHUB_DEFAULT_BRANCH'];\n",
"# override: \"mybranch\"\n",
"\n",
"dbse = os.environ['POSTGRESQL_DB'];\n",
"host = os.environ['POSTGRESQL_HOST'];\n",
"port = os.environ['POSTGRESQL_PORT'];\n",
"user = 'cipsrv';\n",
"pasw = os.environ['POSTGRESQL_CIP_PASS'];\n",
"\n",
"depf = os.path.join('src','database','cipsrv_owld','cipsrv_owld_deploy.sql');\n",
"\n",
"print(\"repo: \" + gith);\n",
"print(\"branch: \" + brnh);"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cs = \"dbname=%s user=%s password=%s host=%s port=%s\" % (\n",
" dbse\n",
" ,user\n",
" ,pasw\n",
" ,host\n",
" ,port\n",
");\n",
"\n",
"try:\n",
" conn = psycopg2.connect(cs);\n",
"except:\n",
" raise Exception(\"database connection error\");\n",
" \n",
"print(\"database is ready\");"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if args.use_existing_sql is not None:\n",
" trg = args.use_existing_sql;\n",
" print(\"using existing \" + trg);\n",
" \n",
"else:\n",
" td = tempfile.TemporaryDirectory();\n",
" \n",
" repo = git.Repo.clone_from(\n",
" url = gith\n",
" ,branch = brnh\n",
" ,to_path = td.name\n",
" ,depth = 1\n",
" );\n",
" \n",
" print(\"checkout of \" + td.name + \" complete.\");\n",
" trg = os.path.join(td.name,depf);\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if not os.path.exists(trg):\n",
" raise Exception(trg + ' not found.');\n",
" \n",
"z = common.load_sqlfile(conn,trg,echo=True);\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"conn.close();\n",
"\n",
"if args.use_existing_sql is None:\n",
" td.cleanup();\n",
"\n",
"print(\"logic load complete.\");"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit b74ca7c

Please sign in to comment.