Pull request update/231118

arcee/arcee_receiver/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.5
+aiohttp==3.8.6
 sanic==22.6.2
 motor==3.0.0
 websockets==10.4

bulldozer/bulldozer_api/requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.5
+aiohttp==3.8.6
 sanic==22.6.2
 motor==3.0.0
 websockets==10.4

diworker/requirements.txt

@@ -4,12 +4,12 @@ boto3==1.18.9
 mongodb-migrations==0.7.0
 retrying==1.3.3
 pandas == 1.3.3
-pyarrow==5.0.0
+pyarrow==14.0.1
 clickhouse-driver==0.2.2
 pyrabbit==1.1.0
 # OptScale packages
 -e tools/cloud_adapter
 -e optscale_client/config_client
 -e optscale_client/herald_client
 -e optscale_client/insider_client
--e optscale_client/rest_api_client
+-e optscale_client/rest_api_client

herald/send_templates.py

@@ -922,17 +922,14 @@
         'template_params': {
             'texts': {
                 'organization': {
-                    'id': '6946211f-47ff-43a3-a9a3-3e5f57d52415'},
-                'reason': '<class \'bumi_worker.tasks.BumiTimeoutError\'> - '
-                          'Timeout error while process task 223322 '
-                          '(organization_id 6946211f-47ff-43a3-a9a3-3e5f57d52415 '
-                          'module None), step SetStarted',
+                    'id': '6946211f-47ff-43a3-a9a3-3e5f57d52415'
+                },
                 'failed_modules': [{
-                    'instance_migration': 'Timeout error while process task '
-                                          '1639638582 (organization_id '
-                                          'b74f3ca6-c392-4bcb-96b3-2ffaa0281810,'
-                                          ' module instance_migration, try 0),'
-                                          ' step Process'}]
+                    'module': 'instance_migration',
+                    'error': 'Timeout error while process task 1639638582 '
+                             '(organization_id b74f3ca6-c392-4bcb-96b3-2ffaa0281810, '
+                             'module instance_migration, try 0) step Process'
+                }]
             }}},
     'bumi_module_execution_failed': {
         'email': '[email protected]',

jira_bus/jira_bus_server/atlassian_client/client.py

@@ -51,7 +51,7 @@ def _create_token(self, method, relative_url):
             },
             key=self.shared_secret,
             algorithm="HS256",
-        ).decode()
+        )
 
     @_wrap_request_errors(AtlassianClientException)
     def request(self, method, url, params=None, data=None, json=None, headers=None):

ngui/ui/prepare-and-run.sh

@@ -11,7 +11,7 @@ function initializeEnvironmentVariables(){
 }
 
 function allowAppIndexing(){
-    ROBOTS_TXT_PATH=/usr/src/app/ui/public/robots.txt
+    ROBOTS_TXT_PATH=/usr/src/app/ui/build/robots.txt
     > $ROBOTS_TXT_PATH
     {
         echo "User-agent: *"

ngui/ui/src/components/CurrencyCodeAutocomplete/CurrencyCodeAutocomplete.tsx

@@ -31,17 +31,6 @@ const CurrencyCodeAutocomplete = forwardRef(({ name, onBlur, value, onChange, er
         <Input
           {...params}
           ref={ref}
-          inputProps={{
-            ...params.inputProps,
-            style: {
-              ...params.inputProps?.style,
-              /*
-                  MuiAutocomplete-input class from params.inputProps.className overrides width property defined by MuiOutlinedInput-input
-                  Set width to 100% to fix input width in flex layout
-                */
-              width: "100%"
-            }
-          }}
           InputProps={{ ...params.InputProps, ...InputProps }}
           margin="none"
           error={error}

ngui/ui/src/components/EditOrganizationCurrencyForm/EditOrganizationCurrencyForm.tsx

@@ -15,22 +15,28 @@ const EditOrganizationCurrencyForm = ({ defaultCurrency, onSubmit, onCancel, isL
 
   return (
     <FormProvider {...methods}>
-      <form
-        onSubmit={handleSubmit((formData) => {
-          onSubmit(formData[CURRENCY_FIELD_NAME]);
-        })}
-        noValidate
+      <Box
+        sx={{
+          width: { md: "50%" }
+        }}
       >
-        <Box display="flex">
-          <Box>
-            <OrganizationCurrencyField />
+        <form
+          onSubmit={handleSubmit((formData) => {
+            onSubmit(formData[CURRENCY_FIELD_NAME]);
+          })}
+          noValidate
+        >
+          <Box display="flex">
+            <Box flexGrow={1}>
+              <OrganizationCurrencyField />
+            </Box>
+            <Box display="flex" height="max-content">
+              <IconButton isLoading={isLoading} icon={<CheckOutlinedIcon />} type="submit" />
+              <IconButton icon={<CloseIcon />} onClick={onCancel} />
+            </Box>
           </Box>
-          <Box display="flex" height="max-content">
-            <IconButton isLoading={isLoading} icon={<CheckOutlinedIcon />} type="submit" />
-            <IconButton icon={<CloseIcon />} onClick={onCancel} />
-          </Box>
-        </Box>
-      </form>
+        </form>
+      </Box>
     </FormProvider>
   );
 };

optscale-deploy/runkube.py

@@ -17,6 +17,7 @@
 from kubernetes.stream import stream as k8s_stream
 from docker import DockerClient
 from kubernetes.stream.ws_client import ERROR_CHANNEL
+from docker.errors import ImageNotFound
 
 DESCRIPTION = "Script to deploy OptScale on k8s. " \
               "See deployment instructions at https://github.com/hystax/optscale"
@@ -32,6 +33,7 @@
 JOB_NAME = 'configurator'
 OPTSCALE_K8S_NAMESPACE = 'default'
 COMPONENTS_FILE = 'components.yaml'
+LOCAL_TAG = 'local'
 
 LOG = logging.getLogger(__name__)
 
@@ -81,6 +83,10 @@ def master_ip(self):
                     context['clusters'][0]['cluster']['server']).hostname
         return self._master_ip
 
+    @staticmethod
+    def _get_image(docker_cl, image_name, tag):
+        return docker_cl.images.get('{}:{}'.format(image_name, tag))
+
     def get_node_ips(self):
         LOG.debug("Getting node ips...")
         ips = []
@@ -118,47 +124,63 @@ def versions_info(self):
             }
         return self._versions_info
 
-    def _pull_image(self, docker_cl, image_name, tag, auth_config,
-                    same_tag=False):
+    def _pull_image(self, docker_cl, image_name, tag, auth_config):
         full_image_name = os.path.join(self.dregistry, image_name)
         LOG.info("Pulling image %s with tag %s", full_image_name, tag)
         params = {'repository': full_image_name, 'tag': tag}
         if auth_config:
             params['auth_config'] = auth_config
         image = docker_cl.images.pull(**params)
         LOG.debug("Pulled image with id %s", image.id)
+        return image
 
-        if same_tag:
-            result_tag = tag
-        else:
-            # this dirty hack required to make etcd operator work with our etcd image
-            if image_name == 'etcd':
-                image.tag(repository=image_name, tag='vlocal')
-            result_tag = 'local'
-        LOG.info("Tagging %s:%s as %s:%s", full_image_name, tag, image_name,
-                 result_tag)
-        image.tag(repository=image_name, tag=result_tag)
-
-    def pull_images(self, node):
-        LOG.info("Pulling images for %s", node)
-        docker_cl = self.get_docker_cl(node)
+    def pull_images(self, docker_cl):
         LOG.debug("Logging into docker registry %s", self.dregistry)
         auth_config = {}
         if self.dregistry_user or self.dregistry_password:
             auth_config = {
                 'username': self.dregistry_user,
                 'password': self.dregistry_password
             }
-        for image in self.versions_info['images']:
-            self._pull_image(
-                docker_cl, image, self.version, auth_config, same_tag=False)
+        images = {}
+        for image_name in self.versions_info['images']:
+            image = self._pull_image(docker_cl, image_name, self.version,
+                                     auth_config)
+            images[image_name] = image
+        return images
+
+    def get_local_images(self, docker_cl):
+        images_map = self.get_image_id_map()
+        images = {}
+        for name, image_id in images_map.items():
+            try:
+                image = self._get_image(docker_cl, name, self.version)
+            except ImageNotFound:
+                image = None
+            if not image:
+                try:
+                    image = self._get_image(docker_cl, os.path.join(
+                        self.dregistry, name), self.version)
+                except ImageNotFound:
+                    continue
+            if image.id != image_id:
+                images[name] = image
+        return images
+
+    def tag_images_local(self, images):
+        for image_name, image in images.items():
+            # this dirty hack required to make etcd operator work with our etcd image
+            if image_name == 'etcd':
+                image.tag(repository=image_name, tag='vlocal')
+            LOG.info("Tagging %s as %s:%s" % (image, image_name, LOCAL_TAG))
+            image.tag(repository=image_name, tag=LOCAL_TAG)
 
     def get_image_id_map(self):
         LOG.debug("Getting map of image ids...")
         docker_cl = self.get_docker_cl(self.master_ip)
         images = {}
         for service in self.versions_info['images']:
-            image = docker_cl.images.get('{}:local'.format(service))
+            image = self._get_image(docker_cl, service, LOCAL_TAG)
             images[service] = image.id
         LOG.debug("Image ids map: %s", images)
         return images
@@ -301,11 +323,16 @@ def get_old_overlay_list_for_update(self):
 
     def start(self, check, update):
         self.check_releases(update)
-
-        if not self.no_pull:
-            for node in self.get_node_ips():
-                self.pull_images(node)
-
+        for node in self.get_node_ips():
+            docker_cl = self.get_docker_cl(node)
+            if not self.no_pull:
+                LOG.info("Pulling images for %s", node)
+                images = self.pull_images(docker_cl)
+            else:
+                LOG.info('Сomparing local images for %s' % node)
+                images = self.get_local_images(docker_cl)
+            LOG.info('images for tag: %s' % images)
+            self.tag_images_local(images)
         overlays = []
         LOG.debug("Creating temp dir %s", TEMP_DIR)
         os.makedirs(TEMP_DIR, mode=0o755, exist_ok=True)

optscale_client/aconfig_cl/setup.py

@@ -9,6 +9,6 @@
       url='http://hystax.com',
       author_email='[email protected]',
       package_dir={'aconfig_cl': ''},
-      install_requires=['aiohttp==3.8.5'],
+      install_requires=['aiohttp==3.8.6'],
       packages=['aconfig_cl']
       )

tools/cloud_adapter/clouds/nebius.py

@@ -797,8 +797,7 @@ def configure_report(self):
             LOG.warning('Report files not found')
 
     @staticmethod
-    def find_csv_reports(s3_objects, prefix):
-        reports = {}
+    def find_csv_reports(s3_objects, prefix, reports):
         report_regex_fmt = '^{0}/[0-9]{{8}}.csv$'
         try:
             report_regex = re.compile(
@@ -810,8 +809,7 @@ def find_csv_reports(s3_objects, prefix):
                     reports[group] = []
                 reports[group].append(report)
         except KeyError:
-            reports = {}
-        return reports
+            pass
 
     def download_report_file(self, report_name, file_obj):
         self.s3.download_fileobj(
@@ -822,9 +820,17 @@ def get_report_files(self):
         prefix = self.config.get('bucket_prefix', DEFAULT_BUCKET_PREFIX)
         if prefix.endswith('/'):
             prefix = prefix[:-1]
-        resp = self.s3.list_objects_v2(
-            Bucket=bucket_name, Prefix=prefix)
-        reports = self.find_csv_reports(resp, prefix)
+        reports = {}
+        params = {
+            'Bucket': bucket_name,
+            'Prefix': prefix
+        }
+        while True:
+            resp = self.s3.list_objects_v2(**params)
+            self.find_csv_reports(resp, prefix, reports)
+            if not resp['IsTruncated']:
+                break
+            params['ContinuationToken'] = resp['NextContinuationToken']
         if not reports:
             raise ReportFilesNotFoundException(
                 'Report files for report {} not found in bucket {}'.format(