Merge branch 'master' into fix_active_deadline_bug

Signed-off-by: JamesMurkin <[email protected]> # Conflicts: # internal/executor/service/job_manager_test.go
armadaproject · Feb 9, 2024 · c22dc30 · c22dc30
2 parents e27e1eb + 5955432
commit c22dc30
Show file tree

Hide file tree

Showing 150 changed files with 6,689 additions and 28,192 deletions.
diff --git a/.github/workflows/airflow-operator.yml b/.github/workflows/airflow-operator.yml
@@ -98,7 +98,7 @@ jobs:
           # Manually create folders to ensure perms are correct.
           mkdir -p .kube/internal
           mkdir -p .kube/external
-          go run github.com/magefile/[email protected] -v localdev minimal-legacy
+          go run github.com/magefile/[email protected] -v localdev minimal
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:

diff --git a/.github/workflows/python-client.yml b/.github/workflows/python-client.yml
@@ -81,7 +81,7 @@ jobs:
           # Manually create folders to ensure perms are correct.
           mkdir -p .kube/internal
           mkdir -p .kube/external
-          go run github.com/magefile/[email protected] -v localdev minimal-pulsar
+          go run github.com/magefile/[email protected] -v localdev minimal
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -116,7 +116,7 @@ jobs:
           cache-tools: true
 
       - name: Setup Integration Tests
-        run: go run github.com/magefile/[email protected] -v localdev minimal-pulsar
+        run: go run github.com/magefile/[email protected] -v localdev minimal
 
       - name: Run Integration Tests
         run: go run github.com/magefile/[email protected] -v testsuite

diff --git a/.run/Armada (Legacy Scheduler).run.xml b/.run/Armada (Legacy Scheduler).run.xml
diff --git a/.run/Armada Infrastructure Services.run.xml b/.run/Armada Infrastructure Services.run.xml
@@ -5,7 +5,7 @@
         <option name="envFilePath" value="" />
         <option name="services">
           <list>
-            <option value="lookoutv2-postgres-migration" />
+            <option value="lookoutv2-migration" />
             <option value="scheduler-migration" />
           </list>
         </option>
@@ -16,4 +16,4 @@
       <option name="RunConfigurationTask" enabled="true" run_configuration_name="Armada Infrastructure Services" run_configuration_type="docker-deploy" />
     </method>
   </configuration>
-</component>
+</component>
diff --git a/.run/Legacy Executor.run.xml b/.run/Legacy Executor.run.xml
diff --git a/.run/Pulsar Executor.run.xml b/.run/Pulsar Executor.run.xml
@@ -4,8 +4,6 @@
     <working_directory value="$PROJECT_DIR$" />
     <envs>
       <env name="ARMADA_APICONNECTION_FORCENOTLS" value="true" />
-      <env name="ARMADA_APPLICATION_USEEXECUTORAPI" value="true" />
-      <env name="ARMADA_APPLICATION_USELEGACYAPI" value="false" />
       <env name="ARMADA_EXECUTORAPICONNECTION_ARMADAURL" value="localhost:50052" />
       <env name="ARMADA_EXECUTORAPICONNECTION_FORCENOTLS" value="true" />
       <env name="ARMADA_HTTPPORT" value="8085" />
@@ -19,4 +17,4 @@
     <filePath value="$PROJECT_DIR$/cmd/executor/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/.run/Pulsar Server.run.xml b/.run/Pulsar Server.run.xml
@@ -8,9 +8,7 @@
       <env name="ARMADA_EVENTSAPIREDIS_ADDRS" value="localhost:6379" />
       <env name="ARMADA_METRICSPORT" value="9005" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
-      <env name="ARMADA_PROBABILITYOFUSINGPULSARSCHEDULER" value="1" />
       <env name="ARMADA_PULSAR_URL" value="pulsar://localhost:6650" />
-      <env name="ARMADA_PULSARSCHEDULERENABLED" value="true" />
       <env name="ARMADA_REDIS_ADDRS" value="localhost:6379" />
       <env name="EXECUTOR_UPDATE_INTERVAL" value="&quot;1s&quot;" />
     </envs>
@@ -20,4 +18,4 @@
     <filePath value="$PROJECT_DIR$/cmd/armada/main.go" />
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/client/DotNet/Armada.Client/ClientGenerated.cs b/client/DotNet/Armada.Client/ClientGenerated.cs
@@ -868,74 +868,6 @@ public async System.Threading.Tasks.Task<object> DeleteQueueAsync(string name, S
             }
         }
 
-        /// <returns>A successful response.</returns>
-        /// <exception cref="ApiException">A server side error occurred.</exception>
-        public System.Threading.Tasks.Task<ApiQueueInfo> GetQueueInfoAsync(string name)
-        {
-            return GetQueueInfoAsync(name, System.Threading.CancellationToken.None);
-        }
-
-        /// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
-        /// <returns>A successful response.</returns>
-        /// <exception cref="ApiException">A server side error occurred.</exception>
-        public async System.Threading.Tasks.Task<ApiQueueInfo> GetQueueInfoAsync(string name, System.Threading.CancellationToken cancellationToken)
-        {
-            if (name == null)
-                throw new System.ArgumentNullException("name");
-
-            var urlBuilder_ = new System.Text.StringBuilder();
-            urlBuilder_.Append(BaseUrl != null ? BaseUrl.TrimEnd('/') : "").Append("/v1/queue/{name}/info");
-            urlBuilder_.Replace("{name}", System.Uri.EscapeDataString(ConvertToString(name, System.Globalization.CultureInfo.InvariantCulture)));
-
-            var client_ = _httpClient;
-            try
-            {
-                using (var request_ = new System.Net.Http.HttpRequestMessage())
-                {
-                    request_.Method = new System.Net.Http.HttpMethod("GET");
-                    request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json"));
-
-                    PrepareRequest(client_, request_, urlBuilder_);
-                    var url_ = urlBuilder_.ToString();
-                    request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute);
-                    PrepareRequest(client_, request_, url_);
-
-                    var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
-                    try
-                    {
-                        var headers_ = System.Linq.Enumerable.ToDictionary(response_.Headers, h_ => h_.Key, h_ => h_.Value);
-                        if (response_.Content != null && response_.Content.Headers != null)
-                        {
-                            foreach (var item_ in response_.Content.Headers)
-                                headers_[item_.Key] = item_.Value;
-                        }
-
-                        ProcessResponse(client_, response_);
-
-                        var status_ = ((int)response_.StatusCode).ToString();
-                        if (status_ == "200") 
-                        {
-                            var objectResponse_ = await ReadObjectResponseAsync<ApiQueueInfo>(response_, headers_).ConfigureAwait(false);
-                            return objectResponse_.Object;
-                        }
-                        else
-                        {
-                            var objectResponse_ = await ReadObjectResponseAsync<RuntimeError>(response_, headers_).ConfigureAwait(false);
-                            throw new ApiException<RuntimeError>("An unexpected error response.", (int)response_.StatusCode, objectResponse_.Text, headers_, objectResponse_.Object, null);
-                        }
-                    }
-                    finally
-                    {
-                        if (response_ != null)
-                            response_.Dispose();
-                    }
-                }
-            }
-            finally
-            {
-            }
-        }
-
         protected struct ObjectResponseResult<T>
         {
             public ObjectResponseResult(T responseObject, string responseText)
@@ -1804,21 +1736,6 @@ public partial class ApiJobSetFilter
         public System.Collections.Generic.ICollection<ApiJobState> States { get; set; }
 
 
-    }
-
-    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
-    public partial class ApiJobSetInfo 
-    {
-        [Newtonsoft.Json.JsonProperty("leasedJobs", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public int? LeasedJobs { get; set; }
-
-        [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public string Name { get; set; }
-
-        [Newtonsoft.Json.JsonProperty("queuedJobs", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public int? QueuedJobs { get; set; }
-
-
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
@@ -2184,18 +2101,6 @@ public partial class ApiQueueCreateResponse
         public ApiQueue Queue { get; set; }
 
 
-    }
-
-    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
-    public partial class ApiQueueInfo 
-    {
-        [Newtonsoft.Json.JsonProperty("activeJobSets", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public System.Collections.Generic.ICollection<ApiJobSetInfo> ActiveJobSets { get; set; }
-
-        [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
-        public string Name { get; set; }
-
-
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]

diff --git a/client/python/armada_client/asyncio_client.py b/client/python/armada_client/asyncio_client.py
@@ -15,7 +15,6 @@
     event_pb2_grpc,
     submit_pb2,
     submit_pb2_grpc,
-    usage_pb2_grpc,
     health_pb2,
 )
 from armada_client.event import Event
@@ -37,7 +36,6 @@ class ArmadaAsyncIOClient:
     def __init__(self, channel: grpc.aio.Channel) -> None:
         self.submit_stub = submit_pb2_grpc.SubmitStub(channel)
         self.event_stub = event_pb2_grpc.EventStub(channel)
-        self.usage_stub = usage_pb2_grpc.UsageStub(channel)
 
     async def get_job_events_stream(
         self,
@@ -295,18 +293,6 @@ async def get_queue(self, name: str) -> submit_pb2.Queue:
         response = await self.submit_stub.GetQueue(request)
         return response
 
-    async def get_queue_info(self, name: str) -> submit_pb2.QueueInfo:
-        """Get the queue info by name.
-
-        Uses the GetQueueInfo RPC to get queue info.
-
-        :param name: The name of the queue
-        :return: A queue info object.  See the api definition.
-        """
-        request = submit_pb2.QueueInfoRequest(name=name)
-        response = await self.submit_stub.GetQueueInfo(request)
-        return response
-
     @staticmethod
     def unwatch_events(event_stream) -> None:
         """Closes gRPC event streams

diff --git a/client/python/armada_client/client.py b/client/python/armada_client/client.py
@@ -14,7 +14,6 @@
     event_pb2_grpc,
     submit_pb2,
     submit_pb2_grpc,
-    usage_pb2_grpc,
     health_pb2,
 )
 from armada_client.event import Event
@@ -36,7 +35,6 @@ class ArmadaClient:
     def __init__(self, channel):
         self.submit_stub = submit_pb2_grpc.SubmitStub(channel)
         self.event_stub = event_pb2_grpc.EventStub(channel)
-        self.usage_stub = usage_pb2_grpc.UsageStub(channel)
 
     def get_job_events_stream(
         self,
@@ -291,18 +289,6 @@ def get_queue(self, name: str) -> submit_pb2.Queue:
         response = self.submit_stub.GetQueue(request)
         return response
 
-    def get_queue_info(self, name: str) -> submit_pb2.QueueInfo:
-        """Get the queue info by name.
-
-        Uses the GetQueueInfo RPC to get queue info.
-
-        :param name: The name of the queue
-        :return: A queue info object.  See the api definition.
-        """
-        request = submit_pb2.QueueInfoRequest(name=name)
-        response = self.submit_stub.GetQueueInfo(request)
-        return response
-
     @staticmethod
     def unwatch_events(event_stream) -> None:
         """Closes gRPC event streams

diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "armada_client"
-version = "0.2.7"
+version = "0.2.8"
 description = "Armada gRPC API python client"
 readme = "README.md"
 requires-python = ">=3.7"

diff --git a/client/python/tests/integration/test_no_auth.py b/client/python/tests/integration/test_no_auth.py
@@ -138,12 +138,6 @@ def test_get_queue(client: ArmadaClient, queue_name):
     assert queue.name == queue_name
 
 
-def test_get_queue_info(client: ArmadaClient, queue_name):
-    queue = client.get_queue_info(name=queue_name)
-    assert queue.name == queue_name
-    assert not queue.active_job_sets
-
-
 def test_submit_job_and_cancel_by_id(client: ArmadaClient, queue_name):
     job_set_name = f"set-{uuid.uuid1()}"
     jobs = client.submit_jobs(

diff --git a/client/python/tests/unit/test_asyncio_client.py b/client/python/tests/unit/test_asyncio_client.py
@@ -172,12 +172,6 @@ async def test_delete_queue(aio_client):
     await aio_client.delete_queue("test")
 
 
-@pytest.mark.asyncio
-async def test_get_queue_info(aio_client):
-    queue = await aio_client.get_queue_info(name="test")
-    assert queue.name == "test"
-
-
 @pytest.mark.asyncio
 async def test_cancel_jobs(aio_client):
     await test_create_queue(aio_client)

diff --git a/client/python/tests/unit/test_client.py b/client/python/tests/unit/test_client.py
@@ -161,10 +161,6 @@ def test_delete_queue():
     tester.delete_queue("test")
 
 
-def test_get_queue_info():
-    assert tester.get_queue_info(name="test").name == "test"
-
-
 def test_cancel_jobs():
     test_create_queue()
     test_submit_job()

diff --git a/cmd/armadactl/cmd/commands.go b/cmd/armadactl/cmd/commands.go
@@ -47,15 +47,6 @@ func updateCmd() *cobra.Command {
 	return cmd
 }
 
-func describeCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "describe",
-		Short: "Retrieve information about armada resource. Supported: queue",
-	}
-	cmd.AddCommand(queueDescribeCmd())
-	return cmd
-}
-
 func getCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "get",

diff --git a/cmd/armadactl/cmd/params.go b/cmd/armadactl/cmd/params.go
@@ -20,7 +20,6 @@ func initParams(cmd *cobra.Command, params *armadactl.Params) error {
 	// Setup the armadactl to use pkg/client as its backend for queue-related commands
 	params.QueueAPI.Create = cq.Create(client.ExtractCommandlineArmadaApiConnectionDetails)
 	params.QueueAPI.Delete = cq.Delete(client.ExtractCommandlineArmadaApiConnectionDetails)
-	params.QueueAPI.GetInfo = cq.GetInfo(client.ExtractCommandlineArmadaApiConnectionDetails)
 	params.QueueAPI.Get = cq.Get(client.ExtractCommandlineArmadaApiConnectionDetails)
 	params.QueueAPI.Update = cq.Update(client.ExtractCommandlineArmadaApiConnectionDetails)
 

diff --git a/cmd/armadactl/cmd/queue.go b/cmd/armadactl/cmd/queue.go
@@ -97,28 +97,6 @@ func queueDeleteCmdWithApp(a *armadactl.App) *cobra.Command {
 	return cmd
 }
 
-func queueDescribeCmd() *cobra.Command {
-	return queueDescribeCmdWithApp(armadactl.New())
-}
-
-// Takes a caller-supplied app struct; useful for testing.
-func queueDescribeCmdWithApp(a *armadactl.App) *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "queue <queueName>",
-		Short: "Prints out queue info.",
-		Long:  "Prints out queue info including all jobs sets where jobs are running or queued.",
-		Args:  cobra.ExactArgs(1),
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			return initParams(cmd, a.Params)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			name := args[0]
-			return a.DescribeQueue(name)
-		},
-	}
-	return cmd
-}
-
 func queueGetCmd() *cobra.Command {
 	return queueGetCmdWithApp(armadactl.New())
 }