From d8c878c4f8bb01baa7512a5afb9eecc3ba8dd466 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 8 Sep 2022 10:52:11 +0200 Subject: [PATCH] feat: test images before creating sessions --- .gitignore | 5 + Dockerfile | 14 +- Pipfile | 4 +- Pipfile.lock | 159 +++++++++++------- controller/config_types.py | 8 +- controller/metrics/events.py | 2 + controller/metrics/prometheus.py | 57 +++++-- controller/metrics/queue.py | 1 + controller/metrics/s3.py | 25 +-- controller/metrics/utils.py | 16 +- controller/server_controller.py | 40 ++++- controller/templates/session_validation.yml | 15 ++ controller/utils.py | 6 +- tests/integration/conftest.py | 3 +- tests/integration/test_culling.py | 22 ++- .../__pycache__/__init__.cpython-38.pyc | Bin 3915 -> 4633 bytes 16 files changed, 262 insertions(+), 115 deletions(-) create mode 100644 controller/templates/session_validation.yml diff --git a/.gitignore b/.gitignore index 70f2f3c4..c20d18b1 100644 --- a/.gitignore +++ b/.gitignore @@ -217,3 +217,8 @@ acceptance-tests/cypress/videos # Helm requirements lock files helm-chart/amalthea/requirements.lock helm-chart/amalthea/Chart.lock + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class diff --git a/Dockerfile b/Dockerfile index 9a90078a..2fb56cdd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,13 +7,25 @@ RUN pip install --no-cache-dir --disable-pip-version-check -U pip && \ groupadd -g 1000 amalthea && \ useradd -u 1000 -g 1000 amalthea && \ apt-get update && \ - apt-get install tini -y && \ + apt-get install tini curl -y && \ rm -rf /var/lib/apt/lists/* # Install all packages WORKDIR /app COPY Pipfile Pipfile.lock ./ RUN pipenv install --system --deploy +RUN curl -L https://github.com/NVIDIA/container-canary/releases/download/v0.2.1/canary_linux_amd64 > canary_linux_amd64 && \ + curl -L https://github.com/NVIDIA/container-canary/releases/download/v0.2.1/canary_linux_amd64.sha256sum > canary_linux_amd64.sha256sum && \ + sha256sum --check --status canary_linux_amd64.sha256sum && \ + chmod +x canary_linux_amd64 && \ + mv canary_linux_amd64 /usr/local/bin/canary +# fix pyngrok permissions +RUN mkdir /usr/local/lib/python3.9/site-packages/pyngrok/bin && \ + chown :1000 /usr/local/lib/python3.9/site-packages/pyngrok/bin && \ + chmod 770 /usr/local/lib/python3.9/site-packages/pyngrok/bin && \ + mkdir /home/amalthea/ && \ + chown :1000 /home/amalthea && \ + chmod 770 /home/amalthea COPY controller /app/controller COPY kopf_entrypoint.py ./ diff --git a/Pipfile b/Pipfile index 96ead2f9..5c924cf8 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -kopf = "*" +kopf = { version = "*", extras = ["dev"]} pyyaml = "*" kubernetes = "*" jsonpatch = "*" @@ -18,7 +18,7 @@ boto3 = "*" [dev-packages] black = "*" -flake8 = "*" +flake8 = ">=4.0.0,<5.0.0" pytest = "*" pylint = "*" pytest-black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 9d386c5f..05fd9a88 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1d4bb7685b385d2c8705ce0833871f2eb776b8a2aa39258be3890e9139b072d2" + "sha256": "7d5faa9b547ea63126b775fd0115735df0dfa176ddf18e423c956f056055c3fb" }, "pipfile-spec": 6, "requires": { @@ -91,7 +91,7 @@ "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642", "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==3.8.1" }, "aiosignal": { @@ -99,15 +99,22 @@ "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a", "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.2.0" }, + "asn1crypto": { + "hashes": [ + "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", + "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67" + ], + "version": "==1.5.1" + }, "async-timeout": { "hashes": [ "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.0.2" }, "attrs": { @@ -120,19 +127,19 @@ }, "boto3": { "hashes": [ - "sha256:818a40b82e4f66b4bdd4fa38fcc3ed0fb26542f7d8c4d15279d4ba1d4762cd95", - "sha256:84b962f18506ad495dc973aeb5168697ee882497c01430dddeba5e8339db7932" + "sha256:428d94d7c79b6ea65ade428afbd89fb1b869da631bb5acf22230b5b0b7486290", + "sha256:d8785ada565bd3118b9c4c26d8fa64c32d86e62c3404aab612571ae32c270be2" ], "index": "pypi", - "version": "==1.24.62" + "version": "==1.24.68" }, "botocore": { "hashes": [ - "sha256:69682c874dc8ed1856bffd203786c9591fb76a1946d3dcc516bda1ee6a6989f3", - "sha256:8563c7d8b80e8041667cf35b397f1c399537f6c887e1c501f0064bfd7ae541ba" + "sha256:aa8ce593e8d6e1aeb5852f8847e3d1750a1e840b221c01ff63ac0cb1dc583aba", + "sha256:d7decff8d5d94c265ad533c11ba0d653acf0819fcdf1049452076cf8fc0f8946" ], "markers": "python_version >= '3.7'", - "version": "==1.27.62" + "version": "==1.27.68" }, "cachetools": { "hashes": [ @@ -142,20 +149,34 @@ "markers": "python_version ~= '3.7'", "version": "==5.2.0" }, + "certbuilder": { + "hashes": [ + "sha256:56a8aee8ed31a211678647797dfdcdc85ec25d5d1bb1515e44ebae45cce363f9", + "sha256:feed83d15b20c149debc1b73a7eb74b8e8041c78d3a8deb989e21905b4394a30" + ], + "version": "==0.14.2" + }, "certifi": { "hashes": [ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, + "certvalidator": { + "hashes": [ + "sha256:77520b269f516d4fb0902998d5bd0eb3727fe153b659aa1cb828dcf12ea6b8de", + "sha256:922d141c94393ab285ca34338e18dd4093e3ae330b1f278e96c837cb62cffaad" + ], + "version": "==0.11.1" + }, "charset-normalizer": { "hashes": [ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "click": { @@ -311,6 +332,9 @@ "version": "==2.3" }, "kopf": { + "extras": [ + "dev" + ], "hashes": [ "sha256:152dfa5b964cb5f77be00f9b18360f12fe0b2c032d7b5d7e95f4690558ab9ba7", "sha256:b3f812072da7c16aa20e3807d9a35bb9d65df4423a68b039c6424f4e68014b82" @@ -442,9 +466,16 @@ "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2", "sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==3.2.0" }, + "oscrypto": { + "hashes": [ + "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085", + "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4" + ], + "version": "==1.3.0" + }, "prometheus-client": { "hashes": [ "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01", @@ -475,6 +506,12 @@ ], "version": "==0.3.59" }, + "pyngrok": { + "hashes": [ + "sha256:4d03f44a69c3cbc168b17377956a9edcf723e77dbc864eba34c272db15da443c" + ], + "version": "==5.1.0" + }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", @@ -567,7 +604,7 @@ "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.9" }, "s3transfer": { @@ -612,11 +649,11 @@ }, "websocket-client": { "hashes": [ - "sha256:33ad3cf0aef4270b95d10a5a66b670a66be1f5ccf10ce390b3644f9eddfdca9d", - "sha256:79d730c9776f4f112f33b10b78c8d209f23b5806d9a783e296b3813fc5add2f1" + "sha256:398909eb7e261f44b8f4bd474785b6ec5f5b499d4953342fe9755e01ef624090", + "sha256:f9611eb65c8241a67fb373bef040b3cf8ad377a9f6546a12b620b6511e8ea9ef" ], "markers": "python_version >= '3.7'", - "version": "==1.4.0" + "version": "==1.4.1" }, "yarl": { "hashes": [ @@ -703,39 +740,39 @@ }, "black": { "hashes": [ - "sha256:074458dc2f6e0d3dab7928d4417bb6957bb834434516f21514138437accdbe90", - "sha256:187d96c5e713f441a5829e77120c269b6514418f4513a390b0499b0987f2ff1c", - "sha256:2ea29072e954a4d55a2ff58971b83365eba5d3d357352a07a7a4df0d95f51c78", - "sha256:4af5bc0e1f96be5ae9bd7aaec219c901a94d6caa2484c21983d043371c733fc4", - "sha256:560558527e52ce8afba936fcce93a7411ab40c7d5fe8c2463e279e843c0328ee", - "sha256:568ac3c465b1c8b34b61cd7a4e349e93f91abf0f9371eda1cf87194663ab684e", - "sha256:6797f58943fceb1c461fb572edbe828d811e719c24e03375fd25170ada53825e", - "sha256:6c1734ab264b8f7929cef8ae5f900b85d579e6cbfde09d7387da8f04771b51c6", - "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9", - "sha256:7ba9be198ecca5031cd78745780d65a3f75a34b2ff9be5837045dce55db83d1c", - "sha256:94783f636bca89f11eb5d50437e8e17fbc6a929a628d82304c80fa9cd945f256", - "sha256:a218d7e5856f91d20f04e931b6f16d15356db1c846ee55f01bac297a705ca24f", - "sha256:a3db5b6409b96d9bd543323b23ef32a1a2b06416d525d27e0f67e74f1446c8f2", - "sha256:ac609cf8ef5e7115ddd07d85d988d074ed00e10fbc3445aee393e70164a2219c", - "sha256:b154e6bbde1e79ea3260c4b40c0b7b3109ffcdf7bc4ebf8859169a6af72cd70b", - "sha256:b270a168d69edb8b7ed32c193ef10fd27844e5c60852039599f9184460ce0807", - "sha256:b9fd45787ba8aa3f5e0a0a98920c1012c884622c6c920dbe98dbd05bc7c70fbf", - "sha256:c85928b9d5f83b23cee7d0efcb310172412fbf7cb9d9ce963bd67fd141781def", - "sha256:c9a3ac16efe9ec7d7381ddebcc022119794872abce99475345c5a61aa18c45ad", - "sha256:cfaf3895a9634e882bf9d2363fed5af8888802d670f58b279b0bece00e9a872d", - "sha256:e439798f819d49ba1c0bd9664427a05aab79bfba777a6db94fd4e56fae0cb849", - "sha256:f586c26118bc6e714ec58c09df0157fe2d9ee195c764f630eb0d8e7ccce72e69", - "sha256:f6fe02afde060bbeef044af7996f335fbe90b039ccf3f5eb8f16df8b20f77666" + "sha256:0a12e4e1353819af41df998b02c6742643cfef58282915f781d0e4dd7a200411", + "sha256:0ad827325a3a634bae88ae7747db1a395d5ee02cf05d9aa7a9bd77dfb10e940c", + "sha256:32a4b17f644fc288c6ee2bafdf5e3b045f4eff84693ac069d87b1a347d861497", + "sha256:3b2c25f8dea5e8444bdc6788a2f543e1fb01494e144480bc17f806178378005e", + "sha256:4a098a69a02596e1f2a58a2a1c8d5a05d5a74461af552b371e82f9fa4ada8342", + "sha256:5107ea36b2b61917956d018bd25129baf9ad1125e39324a9b18248d362156a27", + "sha256:53198e28a1fb865e9fe97f88220da2e44df6da82b18833b588b1883b16bb5d41", + "sha256:5594efbdc35426e35a7defa1ea1a1cb97c7dbd34c0e49af7fb593a36bd45edab", + "sha256:5b879eb439094751185d1cfdca43023bc6786bd3c60372462b6f051efa6281a5", + "sha256:78dd85caaab7c3153054756b9fe8c611efa63d9e7aecfa33e533060cb14b6d16", + "sha256:792f7eb540ba9a17e8656538701d3eb1afcb134e3b45b71f20b25c77a8db7e6e", + "sha256:8ce13ffed7e66dda0da3e0b2eb1bdfc83f5812f66e09aca2b0978593ed636b6c", + "sha256:a05da0430bd5ced89176db098567973be52ce175a55677436a271102d7eaa3fe", + "sha256:a983526af1bea1e4cf6768e649990f28ee4f4137266921c2c3cee8116ae42ec3", + "sha256:bc4d4123830a2d190e9cc42a2e43570f82ace35c3aeb26a512a2102bce5af7ec", + "sha256:c3a73f66b6d5ba7288cd5d6dad9b4c9b43f4e8a4b789a94bf5abfb878c663eb3", + "sha256:ce957f1d6b78a8a231b18e0dd2d94a33d2ba738cd88a7fe64f53f659eea49fdd", + "sha256:cea1b2542d4e2c02c332e83150e41e3ca80dc0fb8de20df3c5e98e242156222c", + "sha256:d2c21d439b2baf7aa80d6dd4e3659259be64c6f49dfd0f32091063db0e006db4", + "sha256:d839150f61d09e7217f52917259831fe2b689f5c8e5e32611736351b89bb2a90", + "sha256:dd82842bb272297503cbec1a2600b6bfb338dae017186f8f215c8958f8acf869", + "sha256:e8166b7bfe5dcb56d325385bd1d1e0f635f24aae14b3ae437102dedc0c186747", + "sha256:e981e20ec152dfb3e77418fb616077937378b322d7b26aa1ff87717fb18b4875" ], "index": "pypi", - "version": "==22.6.0" + "version": "==22.8.0" }, "certifi": { "hashes": [ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, "charset-normalizer": { @@ -743,16 +780,16 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "chartpress": { "hashes": [ - "sha256:3adaed1a80a095e6fd84c7f035229a01bc375c38723b203184d90df2fa0221f2", - "sha256:7453649179d745885774dac93a6fea8fde833cc03bc4ef3febae677d750e476e" + "sha256:988ae1aed0e84ebc561e6b6b2c10464411780445e881784e790e0c90997e95cd", + "sha256:aaa905f91b14058107471bc4059b1f6e0390b2cdbfdec418ee3d64a3e2e392dc" ], "index": "pypi", - "version": "==1.3.0" + "version": "==2.1.0" }, "click": { "hashes": [ @@ -857,6 +894,7 @@ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" ], + "markers": "python_version >= '3.6'", "version": "==0.6.1" }, "mypy-extensions": { @@ -876,10 +914,11 @@ }, "pathspec": { "hashes": [ - "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a", - "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1" + "sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93", + "sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d" ], - "version": "==0.9.0" + "markers": "python_version >= '3.7'", + "version": "==0.10.1" }, "platformdirs": { "hashes": [ @@ -939,11 +978,11 @@ }, "pytest": { "hashes": [ - "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", - "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" + "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7", + "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39" ], "index": "pypi", - "version": "==7.1.2" + "version": "==7.1.3" }, "pytest-black": { "hashes": [ @@ -1028,14 +1067,6 @@ "markers": "python_version >= '3.7'", "version": "==65.3.0" }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", @@ -1049,7 +1080,7 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_full_version < '3.11.0a7'", + "markers": "python_version < '3.11'", "version": "==2.0.1" }, "tomlkit": { @@ -1078,11 +1109,11 @@ }, "websocket-client": { "hashes": [ - "sha256:33ad3cf0aef4270b95d10a5a66b670a66be1f5ccf10ce390b3644f9eddfdca9d", - "sha256:79d730c9776f4f112f33b10b78c8d209f23b5806d9a783e296b3813fc5add2f1" + "sha256:398909eb7e261f44b8f4bd474785b6ec5f5b499d4953342fe9755e01ef624090", + "sha256:f9611eb65c8241a67fb373bef040b3cf8ad377a9f6546a12b620b6511e8ea9ef" ], "markers": "python_version >= '3.7'", - "version": "==1.4.0" + "version": "==1.4.1" }, "wrapt": { "hashes": [ diff --git a/controller/config_types.py b/controller/config_types.py index 4d29368d..9906bb08 100644 --- a/controller/config_types.py +++ b/controller/config_types.py @@ -7,6 +7,7 @@ @dataclass class S3Config: """The configuration needed to upload metrics to S3.""" + endpoint: str bucket: str path_prefix: str @@ -22,6 +23,7 @@ def __post_init__(self): @dataclass class MetricsBaseConfig: """Base metrics/auditlog configuration.""" + enabled: Union[str, bool] = False extra_labels: Union[str, List[str]] = field(default_factory=list) @@ -35,12 +37,15 @@ def __post_init__(self): @dataclass class AuditlogConfig(MetricsBaseConfig): """The configuration used for the auditlogs.""" + s3: Optional[S3Config] = None def __post_init__(self): super().__post_init__() if self.enabled and not self.s3: - raise ValueError("If auditlog is enabled then the S3 configuration has to be provided.") + raise ValueError( + "If auditlog is enabled then the S3 configuration has to be provided." + ) @classmethod def dataconf_from_env(cls, prefix="AUDITLOG_"): @@ -50,6 +55,7 @@ def dataconf_from_env(cls, prefix="AUDITLOG_"): @dataclass class PrometheusMetricsConfig(MetricsBaseConfig): """The configuration for prometheus metrics""" + port: Union[str, int] = 8765 def __post_init__(self): diff --git a/controller/metrics/events.py b/controller/metrics/events.py index 6d0d8099..7677aa44 100644 --- a/controller/metrics/events.py +++ b/controller/metrics/events.py @@ -12,6 +12,7 @@ class MetricEvent: """Every element in the metrics queue that is created by amalthea and consumed by the metrics handlers conforms to this structure.""" + event_timestamp: datetime session: Dict[str, Any] sessionCreationTimestamp: Optional[datetime] = None @@ -40,6 +41,7 @@ class MetricEventHandler(ABC): """Abstract class for the queue workers that will be doing the final publishing or persisting of any metrics that are generated by amalthea in the metrics queue.""" + @abstractmethod def publish(self, metric_event: MetricEvent): pass diff --git a/controller/metrics/prometheus.py b/controller/metrics/prometheus.py index 942f6049..1b4b177f 100644 --- a/controller/metrics/prometheus.py +++ b/controller/metrics/prometheus.py @@ -6,11 +6,15 @@ from controller.server_status_enum import ServerStatusEnum from controller.metrics.events import MetricEventHandler, MetricEvent -from controller.metrics.utils import resource_request_from_manifest, additional_labels_from_manifest +from controller.metrics.utils import ( + resource_request_from_manifest, + additional_labels_from_manifest, +) class PrometheusMetricAction(Enum): """The different methods that can be used to manipulate prometheus metrics.""" + inc = "inc" dec = "dec" set = "set" @@ -20,23 +24,30 @@ class PrometheusMetricAction(Enum): class PrometheusMetricType(NamedTuple): """A generic prometheus metric "struct" with its allowed methods and the specific metric type.""" + type: MetricWrapperBase actions: List[PrometheusMetricAction] class PrometheusMetricTypesEnum(Enum): """All the different prometheus metrics supported.""" + counter = PrometheusMetricType(Counter, [PrometheusMetricAction.inc]) gauge = PrometheusMetricType( Gauge, - [PrometheusMetricAction.inc, PrometheusMetricAction.dec, PrometheusMetricAction.set], + [ + PrometheusMetricAction.inc, + PrometheusMetricAction.dec, + PrometheusMetricAction.set, + ], ) histogram = PrometheusMetricType(Histogram, [PrometheusMetricAction.observe]) summary = PrometheusMetricType(Summary, [PrometheusMetricAction.observe]) -class PrometheusMetric(): +class PrometheusMetric: """A generic wrapper class for all prometheus metrics.""" + _label_name_invalid_first_letter = re.compile(r"^[^a-zA-Z_]") _label_name_invalid_all_letters = re.compile(r"[^a-zA-Z0-9_]") @@ -47,7 +58,7 @@ def __init__( documentation: str, labelnames: Optional[List[str]], *args, - **kwargs + **kwargs, ): if type(metric_type) is str: self._metric_type = PrometheusMetricTypesEnum[metric_type].value @@ -60,7 +71,9 @@ def __init__( ) if labelnames is None: labelnames = [] - sanitized_label_names = [self.sanitize_label_name(label) for label in labelnames] + sanitized_label_names = [ + self.sanitize_label_name(label) for label in labelnames + ] self._metric = self._metric_type.type( name, documentation, sanitized_label_names, *args, **kwargs ) @@ -73,9 +86,7 @@ def sanitize_label_name(self, val: str) -> str: return val def _sanitize_labels(self, labels: Dict[str, str]) -> Dict[str, str]: - return { - self.sanitize_label_name(name): val for name, val in labels.items() - } + return {self.sanitize_label_name(name): val for name, val in labels.items()} def __call__( self, @@ -106,7 +117,9 @@ def __call__( else: sanitized_labels = {} operation_method = getattr( - self._metric if len(sanitized_labels) == 0 else self._metric.labels(**sanitized_labels), + self._metric + if len(sanitized_labels) == 0 + else self._metric.labels(**sanitized_labels), action.value, None, ) @@ -116,6 +129,7 @@ def __call__( class PrometheusMetricNames(Enum): """Used to avoid errors in metric names and to ensure that always the same set of metric names are used.""" + sessions_total_created = "sessions_total_created" sessions_total_deleted = "sessions_total_deleted" sessions_status_changes = "sessions_status_changes" @@ -128,6 +142,7 @@ class PrometheusMetricNames(Enum): class PrometheusMetricHandler(MetricEventHandler): """Handles metric events from the queue that are created by amalthea.""" + def __init__(self, manifest_labelnames: List[str] = []): self.manifest_labelnames = manifest_labelnames self._sessions_total_created = PrometheusMetric( @@ -150,7 +165,7 @@ def __init__(self, manifest_labelnames: List[str] = []): *self.manifest_labelnames, "status_from", "status_to", - ] + ], ) self._sessions_launch_duration = PrometheusMetric( PrometheusMetricTypesEnum["histogram"].value, @@ -158,7 +173,7 @@ def __init__(self, manifest_labelnames: List[str] = []): "How long did it take for a session to transition into running state", self.manifest_labelnames, unit="seconds", - buckets=[30, 60, 90, 120, 180, 240, 300, 480] + buckets=[30, 60, 90, 120, 180, 240, 300, 480], ) self._sessions_cpu_request = PrometheusMetric( PrometheusMetricTypesEnum["histogram"].value, @@ -166,7 +181,7 @@ def __init__(self, manifest_labelnames: List[str] = []): "CPU millicores requested by a user for a session.", self.manifest_labelnames, unit="m", - buckets=[100, 500, 1000, 2000, 3000, 4000] + buckets=[100, 500, 1000, 2000, 3000, 4000], ) self._sessions_memory_request = PrometheusMetric( PrometheusMetricTypesEnum["histogram"].value, @@ -174,14 +189,14 @@ def __init__(self, manifest_labelnames: List[str] = []): "Memory requested by a user for a session.", self.manifest_labelnames, unit="byte", - buckets=[500e6, 1e9, 2e9, 4e9, 8e9, 16e9, 32e9] + buckets=[500e6, 1e9, 2e9, 4e9, 8e9, 16e9, 32e9], ) self._sessions_gpu_request = PrometheusMetric( PrometheusMetricTypesEnum["histogram"].value, PrometheusMetricNames["sessions_gpu_request"].value, "GPUs requested by a user for a session.", self.manifest_labelnames, - buckets=[0, 1, 2, 3, 4] + buckets=[0, 1, 2, 3, 4], ) self._sessions_disk_request = PrometheusMetric( PrometheusMetricTypesEnum["histogram"].value, @@ -189,7 +204,7 @@ def __init__(self, manifest_labelnames: List[str] = []): "Disk space requested by a user for a session.", self.manifest_labelnames, unit="byte", - buckets=[1e9, 4e9, 16e9, 32e9, 64e9, 128e9] + buckets=[1e9, 4e9, 16e9, 32e9, 64e9, 128e9], ) def _collect_labels_from_manifest(self, manifest: Dict[str, Any]) -> Dict[str, str]: @@ -244,10 +259,16 @@ def _on_any_status_change(self, metric_event: MetricEvent): if metric_event.status != metric_event.old_status: status_change_labels = { **manifest_labels, - "status_from": metric_event.old_status.value if metric_event.old_status else "None", - "status_to": metric_event.status.value if metric_event.status else "None", + "status_from": metric_event.old_status.value + if metric_event.old_status + else "None", + "status_to": metric_event.status.value + if metric_event.status + else "None", } - self._sessions_status_changes(PrometheusMetricAction.inc, 1, status_change_labels) + self._sessions_status_changes( + PrometheusMetricAction.inc, 1, status_change_labels + ) def publish(self, metric_event: MetricEvent): """Publishes (i.e. persists) the proper prometheus metrics diff --git a/controller/metrics/queue.py b/controller/metrics/queue.py index 016995a7..8ec389cf 100644 --- a/controller/metrics/queue.py +++ b/controller/metrics/queue.py @@ -12,6 +12,7 @@ class MetricsQueue: metrics handlers subscribe to this queue and persist or further publish the metrics to the proper place. """ + def __init__(self, metric_handlers=List[MetricEventHandler]): self.q = Queue() self.metric_handlers = metric_handlers diff --git a/controller/metrics/s3.py b/controller/metrics/s3.py index d5d5c4b8..7e9a1895 100644 --- a/controller/metrics/s3.py +++ b/controller/metrics/s3.py @@ -24,6 +24,7 @@ @dataclass class SesionMetricData: """The data that is included for each metric event uploaded to S3.""" + name: str namespace: str uid: str @@ -46,7 +47,9 @@ def _default_json_serializer(obj): return obj.value def __str__(self): - return json.dumps(asdict(self), default=self._default_json_serializer, indent=None) + return json.dumps( + asdict(self), default=self._default_json_serializer, indent=None + ) @classmethod def from_metric_event( @@ -65,7 +68,8 @@ def from_metric_event( metric_event.status, metric_event.old_status, additional_labels_from_manifest( - metric_event.session, additional_label_names, + metric_event.session, + additional_label_names, ), ) @@ -76,11 +80,10 @@ class S3RotatingLogHandler(BaseRotatingHandler): not kept locally. The maximum rotation period (in seconds) can be specified. """ + _datetime_format = "_%Y%m%d_%H%M%S%z" - def __init__( - self, filename, mode, config: S3Config, encoding=None - ): + def __init__(self, filename, mode, config: S3Config, encoding=None): super().__init__(filename, mode, encoding, delay=False) self._period_timedelta = timedelta(seconds=config.rotation_period_seconds) self._start_timestamp = pytz.UTC.localize(datetime.utcnow()) @@ -110,9 +113,7 @@ def _upload(self, fname: str, remove_after_upload: bool = False): resp = None if file_stats.st_size > 0: resp = self._client.upload_file( - fname, - self._bucket, - self._s3_path_prefix + "/" + Path(fname).name + fname, self._bucket, self._s3_path_prefix + "/" + Path(fname).name ) if remove_after_upload: os.remove(fname) @@ -121,7 +122,9 @@ def _upload(self, fname: str, remove_after_upload: bool = False): def _namer(self, default_name: str) -> str: path = Path(default_name) new_file = path.parent / ( - path.stem + self._start_timestamp.strftime(self._datetime_format) + path.suffix + path.stem + + self._start_timestamp.strftime(self._datetime_format) + + path.suffix ) return os.fspath(new_file) @@ -147,10 +150,11 @@ def shouldRollover(self, _: str) -> bool: class S3Formatter(Formatter): """Logging formatter that has ISO8601 timestamps and produces valid json logs.""" + def __init__(self, validate: bool = True) -> None: datefmt = "%Y-%m-%dT%H:%M:%S%z" style = "%" - fmt = "{\"time\":\"%(asctime)s\", \"message\":%(message)s}" + fmt = '{"time":"%(asctime)s", "message":%(message)s}' super().__init__(fmt, datefmt, style, validate) def formatTime(self, record: LogRecord, datefmt: Optional[str] = None) -> str: @@ -167,6 +171,7 @@ class S3MetricHandler(MetricEventHandler): """A simple metric handler that persists the metrics that are published by Amalthea to a S3 bucket. """ + def __init__( self, logger: Logger, diff --git a/controller/metrics/utils.py b/controller/metrics/utils.py index 0e528cd1..5605efd5 100644 --- a/controller/metrics/utils.py +++ b/controller/metrics/utils.py @@ -9,16 +9,22 @@ class ResourceRequest: """The structure of the parsed resource requests when they are extracted from a jupyterserver manifest.""" + cpu_millicores: float memory_bytes: float disk_bytes: float gpus: int = 0 -def resource_request_from_manifest(manifest: Dict[str, Any]) -> Optional[ResourceRequest]: +def resource_request_from_manifest( + manifest: Dict[str, Any] +) -> Optional[ResourceRequest]: """Parses the resource requests from an amalthea manifest.""" - resources = manifest.get("spec", {}).get("jupyterServer", {}).get("resources", {}).get( - "requests", {} + resources = ( + manifest.get("spec", {}) + .get("jupyterServer", {}) + .get("resources", {}) + .get("requests", {}) ) resources = dict(**resources) disk_request = manifest.get("spec", {}).get("storage", {}).get("size") @@ -70,7 +76,9 @@ def additional_labels_from_manifest( both the k8s annotations and labels. In case of duplicates the value from the label will be used.""" - def _filter_labels(labels: Dict[str, str], label_names: List[str]) -> Dict[str, str]: + def _filter_labels( + labels: Dict[str, str], label_names: List[str] + ) -> Dict[str, str]: output = {} for label_name in label_names: label_value = labels.get(label_name) diff --git a/controller/server_controller.py b/controller/server_controller.py index 56294c50..ce5af46f 100644 --- a/controller/server_controller.py +++ b/controller/server_controller.py @@ -1,4 +1,5 @@ import logging +import subprocess import kopf from datetime import datetime from kubernetes.client.models import V1DeleteOptions @@ -7,7 +8,12 @@ import pytz from controller import config -from controller.k8s_resources import CONTENT_TYPES, get_children_specs, get_urls +from controller.k8s_resources import ( + CONTENT_TYPES, + TEMPLATE_DIR, + get_children_specs, + get_urls, +) from controller.culling import get_cpu_usage_for_culling, get_js_server_status from controller.utils import ( get_pod_metrics, @@ -80,6 +86,9 @@ def configure(logger, settings, **_): except AttributeError as e: logger.error(f"Problem when configuring the Operator: {e}") + settings.admission.server = kopf.WebhookAutoTunnel(addr="0.0.0.0", port=8181) + settings.admission.managed = "auto.kopf.dev" + @kopf.on.create( config.api_group, @@ -206,7 +215,8 @@ def is_pod_unschedulable(pod) -> bool: and sorted_conditions[0].get("reason") == "Unschedulable" # NOTE: every pod is initially unschedulable until a PV is provisioned # therefore to avoid "flashing" this state when a sessions starts this case is ignored - and "persistentvolumeclaim" not in sorted_conditions[0].get("message", "").lower() + and "persistentvolumeclaim" + not in sorted_conditions[0].get("message", "").lower() ): return True return False @@ -266,7 +276,9 @@ def get_status(pod) -> ServerStatusEnum: "status": { "state": new_status.value, "failedSince": ( - now.isoformat() if new_status is ServerStatusEnum.Failed else None + now.isoformat() + if new_status is ServerStatusEnum.Failed + else None ), }, }, @@ -417,7 +429,9 @@ def cull_pending_jupyter_servers(body, name, namespace, logger, **kwargs): failed_seconds = 0 if starting_since is not None: - starting_seconds = (now - datetime.fromisoformat(starting_since)).total_seconds() + starting_seconds = ( + now - datetime.fromisoformat(starting_since) + ).total_seconds() if failed_since is not None: failed_seconds = (now - datetime.fromisoformat(failed_since)).total_seconds() @@ -457,6 +471,20 @@ def cull_pending_jupyter_servers(body, name, namespace, logger, **kwargs): return +@kopf.on.validate(config.api_group, config.api_version, config.custom_resource_name) +def validate_session_image(spec, **_): + """Validate session image.""" + try: + image = spec["jupyterServer"]["image"] + subprocess.check_output( + ["canary", "validate", "--file", f"{TEMPLATE_DIR}/session_validation.yml", f"{image}"], + stderr=subprocess.STDOUT, + shell=True + ) + except subprocess.CalledProcessError as e: + raise kopf.AdmissionError(f"Invalid session image:\n{e.output}") + + # create @kopf.on.event(...) type of decorators # Go to the bottom of the update_status function definition to see how # those decorators are applied. @@ -623,14 +651,14 @@ def publish_metrics(old, new, body, name, **_): config.api_group, config.api_version, config.custom_resource_name, - field='status.state', + field="status.state", )(publish_metrics) # NOTE: The 'on.field' handler cannot catch the server deletion so this is needed. kopf.on.delete( config.api_group, config.api_version, config.custom_resource_name, - field='status.state', + field="status.state", )(publish_metrics) # INFO: Start the prometheus metrics server if enabled if config.METRICS.enabled: diff --git a/controller/templates/session_validation.yml b/controller/templates/session_validation.yml new file mode 100644 index 00000000..d9f05bac --- /dev/null +++ b/controller/templates/session_validation.yml @@ -0,0 +1,15 @@ +apiVersion: container-canary.nvidia.com/v1 +kind: Validator + +# Metadata +name: renku-sessions # The name of the platform that this manifest validates for +description: Checks that a session image is compatible with renku # A description of that platform +ports: + - port: 8888 + protocol: TCP +checks: + - name: tcp + description: Is listening via TCP on port 8888 + probe: + tcpSocket: + port: 8888 diff --git a/controller/utils.py b/controller/utils.py index 1b39fb29..559030cb 100644 --- a/controller/utils.py +++ b/controller/utils.py @@ -225,7 +225,9 @@ def convert_to_bytes(value): "Pi": 1024**5, "Ei": 1024**6, } - res = re.match(r"^(?ZLzByngDGNKJ?rgkaP3D&_j>?BY5edKmq>=bsekU8*+CgSuMf^XXp34H}8G# zd;EB2re@&z)<6Ef`Ij?>@lR?TeJnISz^lGO!41wLqm%!$4%2P3Q_}xd$HKoE*IgA%VjVFwnUdcyW?KFw#oG=}y~ z;|sR$@Vag}MQiqG>-PpUZJ?(z)Z@@~E{$p~p4+0wg>)ge8{G9};`%-AZiP`&bjE|E zKS}tmL z)B0Tzb=^dC`;ngrH$CT4M_V7hJi0=+-E-3x$#?8&w-tnVLj`ePEWr*!_5DDo>(AL6 z8&6qgD{K{|FxFQ4y|mPBw*!WI9z7;1Hr{o->NOPG#?bhzzRwclfRP#gWZtvZji(0e zZ0-5oD61-eOL$T2^QUHJF1?j8PtIVWE}s8z`EJ}5%chhR)viUMN`koCfB!N`Lw@2IrjcI3Eqlcxq2hsrdP~=|GpYAc z8LfInG1@^Zb`BM-SYsbbabveCtzn^Bi>>AIH!x|UiCmy&6U7z0%0WTGRE%$S3WpEx zI+Igi32PVD|Nlq5`3;UDxzt4zTA^mH(4z4Nv5qa; zbfn|7(XumZCk}g=sruP;a90GoUhedn!$lBt0l(@;VbUsRV~BfW=gZ~@2|`YKlq3^m z(^5pjS3(bT=Ee(@rxFQZbS6@guc;$#9tKGv^;y!5M9&sZ@r*fA#YJIE&6^ghGxN|f z&95EX{Ey|B&&@-#T1Ri4$!}v$R$tFSdn3neX59u*vneV+%+Z>iTN$V%?%pmg@TS-z zr$u~wC6rBJ#}&LPML`I9VlX2?h}>T?2~;vZ6@{(!r2gkjk8B3 zW_)VgyJQ*YE$`N(WX8V3jPzns;U+I3tT{VX3T0m~`J0~kJIww8!vDVWxpT+(6;|HC z%YoLGoxZ&y1%;B`_mn%bymGf>j4*?+HjXJe@Mvr;wOCdiTjYOx`OzRTWIN!aT{907 z*r2ikZirwi)E!8m;$F*?WRk1|S~v|}*Xh2-SbsCna(j^58_Rtf*w?P znx=dQvyS;2(I%xD+TZ?#$|?S4fduUY-gU(9{<`lZ4rfD$n~OxJCDg1%kSnzwV+8|! z?v!^xi`-d+&(SK)9(z+JJrd2c&$z|D8b+}kILk~z4ze1wVf&W@81}_AN~&- z&UZO^8lQb^(|PdS%XesAK4TLvp z>cv^@RxHJ~1nKiz&#qy+XAeZuHmd*EN-1oPAiWy!%20r$(s;~BX0et*k*4&n0 zY_Bh57;{e{z9K+p%yAlr~?)KsYnF!L6bn2?BcEf%j+AO7K*9QSu zacdAod%0qQyU=Z|HLX!1zY94uFy!~BMwFk&Kf8~>o5;6F&Jw~Y1^7rLBAX?z=6PmE zNXTZ^;^Y>osA1BoOc0@0Yb?m`(>^8-hxtt`Q6Hi(ssKU` z1qL`U^F_rm+4)1OmC{L#?4A?4B)^pqD33c9=k7*z4r7IiQ`u=VOGos&)v8Jv6O zvAsRn%^sgnFXq^M7VBxBhKAnUvH5`IK^Xv;$N0jcaZq_;Ag`^0I?DQ_=Fr3$XNF~h zX)ZI73p995kDS%zIm(V71GJ=5F}_G6kIhjZKMxpPNI%vvS_e#YqC1ch*}WzVIA9_k zCT`f%Lpp;b#0=DgIj$%m>{z;6AZ&p`agaoA9*y1mp?I(%i5b(`AU~vC9B?Yh5V4=0 z-S5Hn3wxik>QMXCmJB}^Fi89r*WUtzLOLuZRPR}Bqg)>F7dRQr%_E-DRxv~ zWVm;^XrdA!GY~SJAL&fr1InjN{W){firbFe9dKa!+OY`&VJe|XSs7?YU1@Fd`8vtA zbnvm0cvtk&*GG$Ui7B$Rv5Lc7p&uzXt-9^Dp6aF-PMcepGi{#uaDYAKbwoaV6Xc3q ze|~a3ri`qGU{aHHy3ScD&Qo!L?!zJ#Gn;r=WzHt@yLaEos+;e=Q>2QH=uyfjDQO*r z4VM-+@KHFjmQD$Etm;e*nM;K%)X!#>?`a(=K;p{eFR(*y;*Y3CcFxY6YSRQm>L)0S zN)vIViHs18`jTGDwh&3?8ES`?gSvU~&@7oR(5jp-A$tT3JM@n=md+wA>|nl>9qblR zqA_}?PC16A;}1|tj2#A=B~l$DPxs-=g9$pRX8Oa1lw^?Semz2{G6W?BTE z>wo?GXSWsz`3pA29}5~kg`z%#iW5##64q}TQqyLkW!`pZo3|4>oP{ow2DiBVl7t?2 zxC`$lZ*ULZEk4UzeCj0)r}!M7<}>g$&FL0t&;Jc}A)Ccw)QwWTFQR~pXF;0BQ5sVm z8a@o(sM%vE>Lyg2a>lJa>zI~AUMuNvp#M{%6@GacGb^HhXW zSpO`^c#`df*BA%KI`FPsDDLB|6EoHm%&K*Ap3UvuY~I;1k~>k{ zk;4_8Ly)9i6KR7&r$1T$L)N!5D1A4%OoQR|%! z?%C`K2bxs&q!YV%6PO>9^ zU!*-DeVzMC%cSRL;(3~6LixKg@A?NS&&)u!r~D-IA3uGvR6#-K;r0AtlEZQ_| z;^J%%bT&t>D+l(hW{ud+NbD8pQ|L3oWE1KbeK&gW2w1RB{T?dRb^x9`p;rmFu99O* z(*wrsl6*zDQ(92FuN=hEmuz71<_R6xyrrEJHgNgW2|czArbuZ&Aus5$18AFhNlItf znj3HZf$UN~4?SKz4mMgS(v}a;Q*b!iXkYloLw=NTe>X|>uru%LUSFdP!BIZIL9Lo~ z;B`crAJtG(tG)!3DYG2E{GA$bYRybPrN29+tEcq1_zrlb>ub>k`)Ywc>IP=#bHMmP z<`->LZsG|71sdE|ao!VaS=1G(2M$sXPU-FK*R*mJh)N5muXlqV7Sm5RKHhk=wGnJS z`o%_J1wk-@AM_Hsa0|VGe%k{CeL8zgH5pL!$)DKg_EwE?xDNJF)J?0VigraO%_IJr zRc!6Uigqp|qIEw_l#cUm@8K$nMvS_Ik>U=*e0T+hU{`DR&6QUV;bXMwEiF-k(em0z z%Xzr{z)6QqX_rGi+jO(mWDzD^(n-DteE}3zLy!?~@-Cq*-nk10bRN1FyMU2xy!`)t ztF`}Qt0kO+gl5dXTpwtKd_l&yljxU=ynL<|#%a=SRQ5rhWEE4r zYC7H*@nNS1bLDao=YV?Tm}-QS4k86m)JqfHcB^x4?wmudT0qbXdC{Zf3Wx|7&ZnqO zO%~yWst6r5*5*m9RjoR@R2n8(dIvmi;Ca3a6>%4tO+CuaT*m(GI_#ge%LqHQ8|)wK zjJ0Md{^V_#xjn%-NJ3Z)=lmQv6Va{NfmO$mFB!0iRYLSyA%7%>BQSz_(9>8U$Rpfa zA-oBtwL*?<4ct^x?Ud{VePKNzpWZumw0qDPct+-wZtB)xs&r213kvKtZJ2BZI1FMI zaM<(-!C3YACd`^KD`w4%IXw%r=U+K{Cb}7|+;dGA>N%y$7eNoyXK4i)a(QBYL(dK7 z%LacB#;>f9!2<4uJ091%UDb=Q=341}MPAvpK9m?8R?NtCQ{O0U7>AgS`VPl$;>asD z>f`SNcit-gY)EYj_^99WBOk)IFS2Ke%(Je@w7+u%0Y&L3g>bWHxD|rS>UbYgWKE>w zWNFy*60@4VIL~&Iy*|bv{|ExNANM6hTJ0b9cSNjfDu+p0{MK-FUkTVYQT}CNsFsL$ zP0Gd6aJgU0_n#%=`L>(|$P{zK9fG0McM{Xr-(z+`8xT!xyddLOIUI@a48>o%%RdGc z#_n=)r&h2pk$)jg>VU=HMW3NJl&2)72vuX)3>%Qnz{%yQE8_j!zf*j0$1g|Ln+P+b z2INBNk&m(H)bWE|KPVPQ*1|}rA~Qt&Vj&o1I)0FwtfIIwF;m167J#6zq0G0E-$E06 zINwATn@2jT9GxGEtoX)gaeZcRp$4gvs$ zd{9im>Y*_noL*iJQ$`N=H`_~c9`uxuFcXPvB$2WW&zOv?vec2PT%|eiYUMna2^bjp z28b7OtIFa1ZcoYgvB$x%spKvEc6TBr?%fTWJNNF^OkBAlLq@LVpKSc9^Z1vW+Z(^$ zHiX=6N*~t9Z(@Z!7P_@r#t>9{w(>^NA!I5F=Ty`&M(Wfymmi=6jC1naSk<&!%bI}I z_hD3Vs7PZONOu`%mofGRi|sH9#7o>W&vGF%T^v%fcjh+jy8XsJvt9URH_w>G{sk@X zx@FT@0_2TvSYaCyH-!F1-h+K&)N1sgUTDL|P}CPtX>ve07*U4B6$@CERk63Vi{+w{SfiM+Ey4W8(k5 zzx5KlcJdA^*lbUaVfZfeTmV^Ot|l=@l9;_13`WBOev>D}+<9R7cMh^(nFFYy5T=I7YVw!DVtcvo9L HB4_^r