hystax · maxb-hystax · Dec 21, 2023 · Dec 18, 2023 · Dec 18, 2023 · Dec 18, 2023
diff --git a/auth/requirements.txt b/auth/requirements.txt
@@ -8,7 +8,7 @@ apispec==0.39.0
 google-auth==1.23.0
 requests==2.31.0
 PyJWT>=2.4.0
-cryptography==41.0.4
+cryptography==41.0.6
 pyyaml==5.4.1
 zcrmsdk==3.1.0
 # OptScale packages

diff --git a/diworker/diworker/importers/aws.py b/diworker/diworker/importers/aws.py
@@ -235,7 +235,7 @@ def _is_flavor_usage(expense):
             (service_code == 'AmazonECS' and 'Fargate' in usage_type) or
             (service_code == 'AmazonSageMaker' and (
                 'ml.' in description or item_type == 'SavingsPlanNegation')) or
-            (service_code == 'AWSLambda' and'Lambda-GB-Second' in usage_type) or
+            (service_code == 'AWSLambda' and 'Lambda-GB-Second' in usage_type) or
             ('BoxUsage' in usage_type)
         )
 

diff --git a/diworker/diworker/migrations/202103111020000_update_system_tags.py b/diworker/diworker/migrations/202103111020000_update_system_tags.py
@@ -11,7 +11,8 @@
 
 
 class AWSUpdateSystemTagsImporter(AWSReportImporter):
-    def save_clean_expenses(self, cloud_account_id, chunk, groupings_map):
+    def save_clean_expenses(self, cloud_account_id, chunk,
+                            unique_id_field='resource_id'):
         info_map = {
             r_id: self.get_resource_info_from_expenses(expenses)
             for r_id, expenses in chunk.items()
@@ -72,8 +73,7 @@ def upgrade(self):
                 )
                 importer = AWSUpdateSystemTagsImporter(**parameters)
                 LOG.info('Started updating tags for %s', cloud_account_id)
-                groupings_map = {'month_resource': {}, 'month_ca': {}}
-                importer.generate_clean_records(groupings_map, regeneration=True)
+                importer.generate_clean_records(regeneration=True)
                 LOG.info('Update tags completed')
 
     def downgrade(self):

diff --git a/diworker/diworker/migrations/202105101020000_regenerate_resource_tags.py b/diworker/diworker/migrations/202105101020000_regenerate_resource_tags.py
@@ -82,8 +82,7 @@ def upgrade(self):
                 )
                 importer = AWSUpdateSystemTagsImporter(**parameters)
                 LOG.info('Started updating tags for %s', cloud_account_id)
-                groupings_map = {'month_resource': {}, 'month_ca': {}}
-                importer.generate_clean_records(groupings_map, regeneration=True)
+                importer.generate_clean_records(regeneration=True)
                 LOG.info('Update tags completed')
 
     def downgrade(self):

diff --git a/diworker/diworker/migrations/2023040614450000_set_box_usage_flag.py b/diworker/diworker/migrations/2023040614450000_set_box_usage_flag.py
@@ -48,7 +48,7 @@ def _is_flavor_usage(expense):
         description = expense.get('lineItem/LineItemDescription', '')
         return ((service_code == 'AmazonECS' and 'Fargate' in usage_type) or
                 (service_code == 'AmazonSageMaker' and 'ml.' in description) or
-                (service_code == 'AWSLambda' and'Lambda-GB-Second' in usage_type) or
+                (service_code == 'AWSLambda' and 'Lambda-GB-Second' in usage_type) or
                 ('BoxUsage' in usage_type))
 
     def upgrade(self):

diff --git a/diworker/run_test.sh b/diworker/run_test.sh
@@ -6,13 +6,14 @@ TEST_IMAGE=diworker_tests:${BUILD_TAG}
 
 docker build -t ${TEST_IMAGE} --build-arg BUILDTAG=${BUILD_TAG} -f diworker/Dockerfile_tests .
 
-echo "PEP8 tests>>>"
-docker run -i --rm ${TEST_IMAGE} bash -c "pep8 --max-line-length=120 ./diworker"
-echo "<<<PEP8 tests"
+echo "Pycodestyle tests>>>"
+docker run -i --rm ${TEST_IMAGE} \
+    bash -c "pycodestyle --max-line-length=120 diworker"
+echo "<<<Pycodestyle tests"
 
 echo "Pylint tests>>>"
 docker run -i --rm ${TEST_IMAGE} bash -c \
-  "pylint --ignore=diworker/diworker/migrations --rcfile=diworker/.pylintrc ./diworker/; exit \$(( \$? & 3 ))"
+  "pylint --rcfile=diworker/.pylintrc --fail-under=9 --fail-on=E,F ./diworker"
 echo "<<Pylint tests"
 
 docker rmi ${TEST_IMAGE}
diff --git a/diworker/test-requirements.txt b/diworker/test-requirements.txt
@@ -1,2 +1,2 @@
-pep8==1.7.1
-pylint==2.6.0
+pycodestyle==2.11.1
+pylint==3.0.2
diff --git a/gemini/gemini_scheduler/Dockerfile_tests b/gemini/gemini_scheduler/Dockerfile_tests
@@ -3,7 +3,7 @@ FROM gemini_scheduler:${BUILDTAG}
 
 WORKDIR /usr/src/app/
 
-COPY gemini/gemini_scheduler/test-requirements.txt gemini/gemini_scheduler/test-requirements.txt
-COPY gemini/.pylintrc gemini/.coveragerc ./gemini/gemini_scheduler/
+COPY gemini/gemini_scheduler/test-requirements.txt ./test-requirements.txt
+COPY gemini/.pylintrc gemini/.coveragerc ./
 
-RUN pip install --no-cache-dir -r gemini/gemini_scheduler/test-requirements.txt
+RUN pip install --no-cache-dir -r test-requirements.txt
diff --git a/gemini/gemini_scheduler/main.py b/gemini/gemini_scheduler/main.py
@@ -42,7 +42,8 @@ def _set_status(self, gemini_id: str, status: str) -> None:
             LOG.info(f"Status updated to {status} for gemini {gemini_id}")
         except Exception as exc:
             LOG.exception(
-                f"Not able to update status to {status} for gemini {gemini_id}: {ex}"
+                f"Not able to update status to {status} for gemini "
+                f"{gemini_id}: {exc}"
             )
             raise exc
 

diff --git a/gemini/gemini_scheduler/run_test.sh b/gemini/gemini_scheduler/run_test.sh
@@ -6,12 +6,14 @@ TEST_IMAGE=gemini_scheduler_tests:${BUILD_TAG}
 
 docker build -t ${TEST_IMAGE} --build-arg BUILDTAG=${BUILD_TAG} -f gemini/gemini_scheduler/Dockerfile_tests .
 
-echo "PEP8 tests>>>"
+echo "Pycodestyle tests>>>"
 docker run -i --rm ${TEST_IMAGE} \
-    bash -c "pep8 --max-line-length=120 --ignore=E701 ."
-echo "<<<PEP8 tests"
+    bash -c "pycodestyle --max-line-length=120 gemini"
+echo "<<<Pycodestyle tests"
 
 echo "Pylint tests>>>"
-docker run -i --rm ${TEST_IMAGE} bash -c "cd gemini/gemini_scheduler && ls && pylint --rcfile=.pylintrc ./"
+docker run -i --rm ${TEST_IMAGE} \
+    bash -c "pylint --rcfile=.pylintrc --fail-under=8 --fail-on=E,F ./gemini"
+echo "<<<Pylint tests"
 
 docker rmi ${TEST_IMAGE}
diff --git a/gemini/gemini_scheduler/test-requirements.txt b/gemini/gemini_scheduler/test-requirements.txt
@@ -1,3 +1,3 @@
 coverage==4.2
-pep8==1.7.1
-pylint==2.6.0
+pycodestyle==2.11.1
+pylint==3.0.2
diff --git a/gemini/gemini_worker/Dockerfile_tests b/gemini/gemini_worker/Dockerfile_tests
@@ -3,7 +3,7 @@ FROM gemini_worker:${BUILDTAG}
 
 WORKDIR /usr/src/app/
 
-COPY gemini/gemini_worker/test-requirements.txt gemini/gemini_worker/test-requirements.txt
-COPY gemini/.pylintrc gemini/.coveragerc ./gemini/gemini_worker/
+COPY gemini/gemini_worker/test-requirements.txt ./test-requirements.txt
+COPY gemini/.pylintrc gemini/.coveragerc ./
 
-RUN pip install --no-cache-dir -r gemini/gemini_worker/test-requirements.txt
+RUN pip install --no-cache-dir -r test-requirements.txt
diff --git a/gemini/gemini_worker/duplicate_object_finder/aws/object_info.py b/gemini/gemini_worker/duplicate_object_finder/aws/object_info.py
@@ -30,10 +30,10 @@ def from_aws_object_info(bucket: str, obj: dict):
                 size=obj["Size"],
             )
         except KeyError as exc:
-            LOG.error(f"Wrong object {obj}: {ex}")
+            LOG.error(f"Wrong object {obj}: {exc}")
             raise
         except Exception as exc:
-            LOG.error(f"Unexpected exception {ex}")
+            LOG.error(f"Unexpected exception {exc}")
             raise
 
     @property

diff --git a/gemini/gemini_worker/duplicate_object_finder/factory.py b/gemini/gemini_worker/duplicate_object_finder/factory.py
@@ -1,4 +1,5 @@
-from duplicate_object_finder.aws.object_info import ObjectInfo
+from gemini.gemini_worker.duplicate_object_finder.aws.object_info import (
+    ObjectInfo)
 from gemini.gemini_worker.duplicate_object_finder.aws.main import find_duplicates
 
 

diff --git a/gemini/gemini_worker/main.py b/gemini/gemini_worker/main.py
@@ -6,7 +6,7 @@
 from kombu import Connection, Exchange, Queue, Message
 from kombu.mixins import ConsumerMixin
 from kombu.utils.debug import setup_logging
-from duplicate_object_finder.factory import Factory
+from gemini.gemini_worker.duplicate_object_finder.factory import Factory
 from etcd import Lock as EtcdLock
 from clickhouse_driver import Client as ClickHouseClient
 from pymongo import MongoClient

diff --git a/gemini/gemini_worker/run_test.sh b/gemini/gemini_worker/run_test.sh
@@ -6,12 +6,14 @@ TEST_IMAGE=gemini_worker_tests:${BUILD_TAG}
 
 docker build -t ${TEST_IMAGE} --build-arg BUILDTAG=${BUILD_TAG} -f gemini/gemini_worker/Dockerfile_tests .
 
-echo "PEP8 tests>>>"
+echo "Pycodestyle tests>>>"
 docker run -i --rm ${TEST_IMAGE} \
-    bash -c "pep8 --max-line-length=120 --ignore=E701 ."
-echo "<<<PEP8 tests"
+    bash -c "pycodestyle --max-line-length=120 gemini"
+echo "<<<Pycodestyle tests"
 
 echo "Pylint tests>>>"
-docker run -i --rm ${TEST_IMAGE} bash -c "cd gemini/gemini_worker && ls && pylint --rcfile=.pylintrc ./"
+docker run -i --rm ${TEST_IMAGE} \
+    bash -c "pylint --rcfile=.pylintrc --fail-under=8 --fail-on=E,F ./gemini"
+echo "<<<Pylint tests"
 
 docker rmi ${TEST_IMAGE}
diff --git a/gemini/gemini_worker/test-requirements.txt b/gemini/gemini_worker/test-requirements.txt
@@ -1,3 +1,3 @@
 coverage==4.2
-pep8==1.7.1
-pylint==2.6.0
+pycodestyle==2.11.1
+pylint==3.0.2
diff --git a/jira_ui/ui/.env.sample b/jira_ui/ui/.env.sample
@@ -1,3 +1,3 @@
 # Local development only
 # Cluster URL to proxy API requests to
-REACT_APP_PROXY=
+VITE_PROXY=
diff --git a/jira_ui/ui/.eslintrc.json b/jira_ui/ui/.eslintrc.json
@@ -2,8 +2,7 @@
   "env": {
     "browser": true,
     "es6": true,
-    "node": true,
-    "jest": true
+    "node": true
   },
   "extends": ["react-app", "eslint:recommended", "plugin:react/recommended", "airbnb-base", "prettier"],
   "globals": {
@@ -92,6 +91,7 @@
   "settings": {
     "import/resolver": {
       "node": {
+        "extensions": [".js", ".jsx", ".ts", ".tsx"],
         "moduleDirectory": ["node_modules", "src/"]
       }
     },

diff --git a/jira_ui/ui/public/index.html → jira_ui/ui/index.html b/jira_ui/ui/public/index.html → jira_ui/ui/index.html
@@ -1,34 +1,23 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
-    <meta
-      name="description"
-      content="Integrate OptScale resources into Jira issues"
-    />
+    <meta name="description" content="Integrate OptScale resources into Jira issues" />
     <!--
       manifest.json provides metadata used when your web app is installed on a
       user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
     -->
-    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <link rel="manifest" href="/manifest.json" />
     <!-- css-reset should be the first stylesheet on the page -->
-    <link rel="stylesheet" href="https://unpkg.com/@atlaskit/[email protected]/dist/bundle.css" media="all">
+    <link rel="stylesheet" href="https://unpkg.com/@atlaskit/[email protected]/dist/bundle.css" media="all" />
     <script src="https://connect-cdn.atl-paas.net/all.js"></script>
-    <!--
-      Notice the use of %PUBLIC_URL% in the tags above.
-      It will be replaced with the URL of the `public` folder during the build.
-      Only files inside the `public` folder can be referenced from the HTML.
-
-      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-      work correctly both with client-side routing and a non-root public URL.
-      Learn how to configure a non-root public URL by running `npm run build`.
-    -->
     <title>OptScale for Jira</title>
   </head>
   <body>
     <noscript>You need to enable JavaScript to run this app.</noscript>
     <div id="root"></div>
+    <script type="module" src="/src/index.jsx"></script>
   </body>
 </html>
diff --git a/jira_ui/ui/jsconfig.json b/jira_ui/ui/jsconfig.json