Pull request update/241125

ea7b1b2 OS-8011. Update profiling integration examples ccc2e55 OS-7958. Ability to update cluster with the latest release c4b8875 OS-8011. Update Profiling integration side modal 850cd7b OS-8007. Add GCP filter support for InstancesForShutdown and ShortLivingInstances recommendations 046c2f3 OS-8003. Updated aiohttp version
hystax · Nov 25, 2024 · 8b77567 · 8b77567
2 parents 6aeb7c1 + ea7b1b2
commit 8b77567
Show file tree

Hide file tree

Showing 9 changed files with 153 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -158,8 +158,8 @@ or if you want to use socket:
 
 **version**:
 
-- Use hystax/optscale git tag (eg: 2024101501-public) if you use optscale public version.
-- Use your own tag version if you build your optscale images (eg: latest).
+- Use hystax/optscale git tag (eg: latest) if you use optscale public version.
+- Use your own tag version if you build your optscale images (eg: local).
 
 **please note**: if you use key authentication, you should have the required key (id_rsa) on the machine
 

diff --git a/arcee/arcee_receiver/requirements.txt b/arcee/arcee_receiver/requirements.txt
@@ -1,4 +1,4 @@
-aiohttp==3.10.2
+aiohttp==3.10.11
 sanic==23.12.1
 sanic-ext==23.12.0
 motor==3.6.0

diff --git a/bulldozer/bulldozer_api/requirements.txt b/bulldozer/bulldozer_api/requirements.txt
@@ -1,4 +1,4 @@
-aiohttp==3.10.2
+aiohttp==3.10.11
 sanic==23.12.1
 motor==3.6.0
 pymongo==4.9.1

diff --git a/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx b/ngui/ui/src/components/ProfilingIntegration/ProfilingIntegration.tsx
@@ -116,7 +116,7 @@ const Installation = () => (
       <FormattedMessage id="mlProfilingIntegration.installation.description" values={{ ...codeFormatMessageValues }} />
       <HtmlSymbol symbol="colon" />
     </Typography>
-    <CodeBlock text="pip install optscale_arcee" />
+    <CodeBlock text="pip install optscale-arcee" />
   </>
 );
 
@@ -131,6 +131,10 @@ const Import = () => (
   </>
 );
 
+const ENDPOINT_URL = `https://${window.location.hostname}:443/arcee/v2` as const;
+
+const ENDPOINT_URL_PARAMETER = `endpoint_url="${ENDPOINT_URL}"` as const;
+
 const Initialization = ({
   profilingToken,
   taskKey,
@@ -144,17 +148,20 @@ const Initialization = ({
 
   const { onClose } = useContext(ProfilingIntegrationModalContext);
 
-  const endpointUrlParameter = isProduction() ? "" : `, endpoint_url="https://${window.location.hostname}:433/arcee/v2"`;
-
   const arceeInitUsingContextManager = (
     <CodeBlock
-      text={`with arcee.init("${profilingToken}", "${taskKey ?? "task_key"}"${endpointUrlParameter}):
+      text={`with arcee.init(token="${profilingToken}",
+                task_key="${taskKey ?? "YOUR-TASK-KEY"}",
+                run_name="YOUR-RUN-NAME",
+                endpoint_url="https://YOUR-OPTSCALE-PUBLIC-IP:443/arcee/v2",
+                ssl=SSL,
+                period=PERIOD):
     # ${intl.formatMessage({ id: "mlProfilingIntegration.common.someCode" })}`}
     />
   );
   const arceeInitUsingFunctionCall = (
     <CodeBlock
-      text={`arcee.init("${profilingToken}", "${taskKey ?? "task_key"}"${endpointUrlParameter})
+      text={`arcee.init(token="${profilingToken}", task_key="${taskKey ?? "YOUR-TASK-KEY"}"${isProduction() ? "" : `, ${ENDPOINT_URL_PARAMETER}`})
 # ${intl.formatMessage({ id: "mlProfilingIntegration.common.someCode" })}
 arcee.finish()
 # ${intl.formatMessage({ id: "mlProfilingIntegration.common.orInCaseOfError" })}
@@ -168,21 +175,40 @@ arcee.error()
       <SectionTitle messageId="mlProfilingIntegration.initialization.title" />
       <Box mb={1}>
         <Typography gutterBottom>
-          <FormattedMessage id="mlProfilingIntegration.initialization.description" />
+          <FormattedMessage
+            id="mlProfilingIntegration.initialization.description"
+            values={{
+              ...codeFormatMessageValues
+            }}
+          />
         </Typography>
         <ul>
           <li>
             {isLoading ? (
               <Skeleton />
             ) : (
-              <KeyValueLabel
-                keyMessageId="mlProfilingIntegration.initialization.profilingToken"
-                value={<CopyText text={profilingToken}>{profilingToken}</CopyText>}
-                isBoldValue={false}
-              />
+              <div>
+                <FormattedMessage
+                  id="mlProfilingIntegration.initialization.parameters.1.token"
+                  values={{
+                    strong: (chunks) => <strong>{chunks}</strong>
+                  }}
+                />
+                <KeyValueLabel
+                  keyMessageId="mlProfilingIntegration.initialization.profilingToken"
+                  value={<CopyText text={profilingToken}>{profilingToken}</CopyText>}
+                  isBoldValue={false}
+                />
+              </div>
             )}
           </li>
           <li>
+            <FormattedMessage
+              id="mlProfilingIntegration.initialization.parameters.2.task_key"
+              values={{
+                strong: (chunks) => <strong>{chunks}</strong>
+              }}
+            />
             <Typography>
               {taskKey ? (
                 <KeyValueLabel
@@ -209,6 +235,38 @@ arcee.error()
               )}
             </Typography>
           </li>
+          <li>
+            <FormattedMessage
+              id="mlProfilingIntegration.initialization.parameters.3.run_name"
+              values={{
+                strong: (chunks) => <strong>{chunks}</strong>
+              }}
+            />
+          </li>
+          <li>
+            <FormattedMessage
+              id="mlProfilingIntegration.initialization.parameters.4.endpoint_url"
+              values={{
+                strong: (chunks) => <strong>{chunks}</strong>
+              }}
+            />
+          </li>
+          <li>
+            <FormattedMessage
+              id="mlProfilingIntegration.initialization.parameters.5.ssl"
+              values={{
+                strong: (chunks) => <strong>{chunks}</strong>
+              }}
+            />
+          </li>
+          <li>
+            <FormattedMessage
+              id="mlProfilingIntegration.initialization.parameters.6.period"
+              values={{
+                strong: (chunks) => <strong>{chunks}</strong>
+              }}
+            />
+          </li>
         </ul>
         <Typography gutterBottom>
           <FormattedMessage
@@ -220,16 +278,41 @@ arcee.error()
           <HtmlSymbol symbol="colon" />
         </Typography>
         {isLoading ? <Skeleton width="100%">{arceeInitUsingContextManager}</Skeleton> : arceeInitUsingContextManager}
+      </Box>
+      <Box mb={1}>
         <Typography gutterBottom>
-          <FormattedMessage
-            id="mlProfilingIntegration.initialization.initCollectorUsingContextManagerDescription"
-            values={{
-              ...codeFormatMessageValues
-            }}
-          />
+          <FormattedMessage id="mlProfilingIntegration.common.example" />
+          <HtmlSymbol symbol="colon" />
         </Typography>
+        <Box>
+          {isProduction() ? (
+            <CodeBlock
+              text={`with arcee.init("00000000-0000-0000-0000-000000000000", "linear_regression",
+                run_name="My run name",
+                ssl=True,
+                period=1):
+    # some code`}
+            />
+          ) : (
+            <CodeBlock
+              text={`with arcee.init("00000000-0000-0000-0000-000000000000", "linear_regression",
+                run_name="My run name",${isProduction() ? "" : `\n                ${ENDPOINT_URL_PARAMETER},`}
+                ssl=False,
+                period=5):
+    # some code`}
+            />
+          )}
+        </Box>
       </Box>
-      <Box mb={1}>
+      <Typography gutterBottom>
+        <FormattedMessage
+          id="mlProfilingIntegration.initialization.initCollectorUsingContextManagerDescription"
+          values={{
+            ...codeFormatMessageValues
+          }}
+        />
+      </Typography>
+      <Box>
         <Typography gutterBottom>
           <FormattedMessage
             id="mlProfilingIntegration.initialization.alternativeInit"
@@ -240,30 +323,6 @@ arcee.error()
         </Typography>
         {isLoading ? <Skeleton width="100%">{arceeInitUsingFunctionCall}</Skeleton> : arceeInitUsingFunctionCall}
       </Box>
-      <Box mb={1}>
-        <Typography gutterBottom>
-          <FormattedMessage id="mlProfilingIntegration.initialization.customEndpointAdnSslChecks" />
-          <HtmlSymbol symbol="colon" />
-        </Typography>
-        <CodeBlock
-          text={`with arcee.init("${profilingToken}", "${
-            taskKey ?? "task_key"
-          }", endpoint_url="https://my.custom.endpoint:443/arcee/v2", ssl=False):
-    # ${intl.formatMessage({ id: "mlProfilingIntegration.common.someCode" })}
-`}
-        />
-      </Box>
-      <Box>
-        <Typography gutterBottom>
-          <FormattedMessage id="mlProfilingIntegration.initialization.arceeDaemonProcess" />
-          <HtmlSymbol symbol="colon" />
-        </Typography>
-        <CodeBlock
-          text={`with arcee.init("${profilingToken}", "${taskKey ?? "task_key"}", period=5):
-    # ${intl.formatMessage({ id: "mlProfilingIntegration.common.someCode" })}
-`}
-        />
-      </Box>
     </>
   );
 };
@@ -283,7 +342,7 @@ const SendingMetrics = () => (
         }
       }}
       parameterMessageIds={["mlProfilingIntegration.sendingMetrics.parameters.1.data"]}
-      method={`arcee.send({"metric_key_1": value_1, "metric_key_2": value_2})`}
+      method={`arcee.send({ "YOUR-METRIC-1-KEY": YOUR_METRIC_1_VALUE, "YOUR-METRIC-2-KEY": YOUR_METRIC_2_VALUE })`}
       example={`arcee.send({ "accuracy": 71.44, "loss": 0.37 })`}
     />
   </>
@@ -307,7 +366,7 @@ const AddingHyperparameters = () => (
         "mlProfilingIntegration.addingHyperparameters.parameters.1.key",
         "mlProfilingIntegration.addingHyperparameters.parameters.2.value"
       ]}
-      method={`arcee.hyperparam(key, value)`}
+      method={`arcee.hyperparam(key="YOUR-PARAM-KEY", value=YOUR_PARAM_VALUE)`}
       example={`arcee.hyperparam("EPOCHS", 100)`}
     />
   </>
@@ -324,7 +383,7 @@ const TaggingTaskRun = () => (
         "mlProfilingIntegration.taggingTaskRun.parameters.1.key",
         "mlProfilingIntegration.taggingTaskRun.parameters.2.value"
       ]}
-      method={`arcee.tag(key, value)`}
+      method={`arcee.tag(key="YOUR-TAG-KEY", value=YOUR_TAG_VALUE)`}
       example={`arcee.tag("Algorithm", "Linear Learn Algorithm")`}
     />
   </>
@@ -338,7 +397,7 @@ const AddingMilestone = () => (
         id: "mlProfilingIntegration.addingMilestone.description"
       }}
       parameterMessageIds={["mlProfilingIntegration.addingMilestone.parameters.1.name"]}
-      method={`arcee.milestone(name)`}
+      method={`arcee.milestone(name="YOUR-MILESTONE-NAME")`}
       example={`arcee.milestone("Download training data")`}
     />
   </>
@@ -352,7 +411,7 @@ const AddingStage = () => (
         id: "mlProfilingIntegration.addingStage.description"
       }}
       parameterMessageIds={["mlProfilingIntegration.addingStage.parameters.1.name"]}
-      method={`arcee.stage(name)`}
+      method={`arcee.stage(name="YOUR-STAGE-NAME")`}
       example={`arcee.stage("preparing")`}
     />
   </>
@@ -371,7 +430,10 @@ const LoggingDatasets = () => (
         "mlProfilingIntegration.loggingDataset.parameters.3.description",
         "mlProfilingIntegration.loggingDataset.parameters.4.labels"
       ]}
-      method={`arcee.dataset(path, name, description, labels)`}
+      method={`arcee.dataset(path="YOUR-DATASET-PATH",
+              name="YOUR-DATASET-NAME",
+              description="YOUR-DATASET-DESCRIPTION",
+              labels=["YOUR-DATASET-LABEL-1", "YOUR-DATASET-LABEL-2"])`}
       example={`arcee.dataset("https://s3/ml-bucket/datasets/training_dataset.csv",
               name="Training dataset",
               description="Training dataset (100k rows)",
@@ -391,7 +453,7 @@ const CreatingModels = () => (
         "mlProfilingIntegration.creatingModels.parameters.1.key",
         "mlProfilingIntegration.creatingModels.parameters.2.path"
       ]}
-      method={`arcee.model(key, path)`}
+      method={`arcee.model(key="YOUR-MODEL-KEY", path="YOUR-MODEL-PATH")`}
       example={`arcee.model("my_model", "/home/user/my_model")`}
     />
   </>
@@ -405,7 +467,7 @@ const SettingModelVersion = () => (
         id: "mlProfilingIntegration.settingModelVersion.description"
       }}
       parameterMessageIds={["mlProfilingIntegration.settingModelVersion.parameters.1.version"]}
-      method={`arcee.model_version(version)`}
+      method={`arcee.model_version(version="YOUR-MODEL-VERSION")`}
       example={`arcee.model_version("1.2.3-release")`}
     />
   </>
@@ -419,7 +481,7 @@ const SettingModelVersionAlias = () => (
         id: "mlProfilingIntegration.settingModelVersionAlias.description"
       }}
       parameterMessageIds={["mlProfilingIntegration.settingModelVersionAlias.parameters.1.alias"]}
-      method={`arcee.model_version_alias(alias)`}
+      method={`arcee.model_version_alias(alias="YOUR-MODEL-VERSION-ALIAS")`}
       example={`arcee.model_version_alias("winner")`}
     />
   </>
@@ -436,7 +498,7 @@ const SettingModelVersionTag = () => (
         "mlProfilingIntegration.settingModelVersionTag.parameters.1.key",
         "mlProfilingIntegration.settingModelVersionTag.parameters.2.value"
       ]}
-      method={`arcee.model_version_tag(key, value)`}
+      method={`arcee.model_version_tag(key="YOUR-MODEL-VERSION-TAG-KEY", value=YOUR_MODEL_VERSION_TAG_VALUE)`}
       example={`arcee.model_version_tag("env", "staging demo")`}
     />
   </>
@@ -455,7 +517,10 @@ const CreatingArtifacts = () => (
         "mlProfilingIntegration.creatingArtifacts.parameters.3.description",
         "mlProfilingIntegration.creatingArtifacts.parameters.4.tags"
       ]}
-      method={`arcee.artifact(path, name, description, tags)`}
+      method={`arcee.artifact(path="YOUR-ARTIFACT-PATH",
+               name="YOUR-ARTIFACT-NAME",
+               description="YOUR-ARTIFACT-DESCRIPTION",
+               tags={"YOUR-ARTIFACT-TAG-KEY": YOUR_ARTIFACT_TAG_VALUE})`}
       example={`arcee.artifact("https://s3/ml-bucket/artifacts/AccuracyChart.png",
                name="Accuracy line chart",
                description="The dependence of accuracy on the time",
@@ -476,8 +541,10 @@ const SettingArtifactTag = () => (
         "mlProfilingIntegration.settingArtifactTag.parameters.2.key",
         "mlProfilingIntegration.settingArtifactTag.parameters.3.value"
       ]}
-      method={`arcee.artifact_tag(path, key, value)`}
-      example={`arcee.artifact_tag("https://s3/ml-bucket/artifacts/AccuracyChart.png", 
+      method={`arcee.artifact_tag(path="YOUR-ARTIFACT-PATH",
+                   key="YOUR-ARTIFACT-TAG-KEY",
+                   value=YOUR_ARTIFACT_TAG_VALUE)`}
+      example={`arcee.artifact_tag("https://s3/ml-bucket/artifacts/AccuracyChart.png",
                    "env", "staging demo")`}
     />
   </>

diff --git a/.../src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx b/.../src/containers/RecommendationsOverviewContainer/recommendations/InstancesForShutdown.tsx
@@ -10,7 +10,7 @@ import {
   NEBIUS_SERVICE
 } from "hooks/useRecommendationServices";
 import { resource, resourceLocation, poolAndOwner, possibleShutdownPeriods, savings } from "utils/columns";
-import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, NEBIUS } from "utils/constants";
+import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, GCP_CNR, NEBIUS } from "utils/constants";
 import BaseRecommendation, { CATEGORY_COST } from "./BaseRecommendation";
 
 const columns = [
@@ -44,7 +44,7 @@ class InstancesForShutdown extends BaseRecommendation {
 
   services = [AWS_EC2, AWS_RDS, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, ALIBABA_ECS, NEBIUS_SERVICE];
 
-  appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS];
+  appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS, GCP_CNR];
 
   categories = [CATEGORY_COST];
 

diff --git a/.../src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx b/.../src/containers/RecommendationsOverviewContainer/recommendations/ShortLivingInstances.tsx
@@ -5,7 +5,7 @@ import ShortLivingInstancesModal from "components/SideModalManager/SideModals/re
 import TextWithDataTestId from "components/TextWithDataTestId";
 import { ALIBABA_ECS, AWS_EC2, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, NEBIUS_SERVICE } from "hooks/useRecommendationServices";
 import { detectedAt, possibleMonthlySavings, resource, resourceLocation } from "utils/columns";
-import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, NEBIUS } from "utils/constants";
+import { ALIBABA_CNR, AWS_CNR, AZURE_CNR, FORMATTED_MONEY_TYPES, GCP_CNR, NEBIUS } from "utils/constants";
 import BaseRecommendation, { CATEGORY_COST } from "./BaseRecommendation";
 
 const columns = [
@@ -51,7 +51,7 @@ class ShortLivingInstances extends BaseRecommendation {
 
   services = [AWS_EC2, AZURE_COMPUTE, GCP_COMPUTE_ENGINE, ALIBABA_ECS, NEBIUS_SERVICE];
 
-  appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS];
+  appliedDataSources = [ALIBABA_CNR, AWS_CNR, AZURE_CNR, NEBIUS, GCP_CNR];
 
   categories = [CATEGORY_COST];