Merge branch 'f/chrisma/zerlog-minimal' of github.com:armadaproject/a…

…rmada into f/chrisma/add-logfile-config

.gitignore

@@ -14,6 +14,10 @@ junit.xml
 docker-compose.dev.yaml
 delve
 
+# VS Code debugging
+__debug_bin
+**/__debug_bin*
+
 # Build artifacts
 dist
 .goreleaser-minimal.yml

.goreleaser.yml

@@ -224,6 +224,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
     dockerfile: ./build/bundles/lookout/Dockerfile
 
   - id: full-bundle
@@ -258,6 +259,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
     dockerfile: ./build/bundles/full/Dockerfile
 
   - id: server
@@ -359,6 +361,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
       - config/lookoutv2/config.yaml
       - config/lookoutingesterv2/config.yaml
       - config/logging.yaml

.run/LookoutV2.run.xml

@@ -7,6 +7,8 @@
       <env name="ARMADA_LOG_CONFIG" value="config/logging-dev.yaml" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
+      <env name="ARMADA_UICONFIG_ARMADAAPIBASEURL" value="http://localhost:8080" />
+      <env name="ARMADA_UICONFIG_BINOCULARSBASEURLPATTERN" value="http://localhost:8082" />
     </envs>
     <kind value="FILE" />
     <package value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />

build/bundles/full/Dockerfile

@@ -9,6 +9,7 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada-ful
 COPY internal/lookout/ui /project/internal/lookout/ui
 COPY pkg/api/*.swagger.json /project/pkg/api/
 COPY pkg/api/binoculars/*.swagger.json /project/pkg/api/binoculars/
+COPY pkg/api/schedulerobjects/*.swagger.json /project/pkg/api/schedulerobjects/
 RUN ./project/internal/lookout/ui/openapi.sh
 
 FROM ${NODE_BUILD_IMAGE} AS NODE

build/lookoutv2/Dockerfile

@@ -7,6 +7,7 @@ FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI
 COPY internal/lookout/ui /project/internal/lookout/ui
 COPY pkg/api/*.swagger.json /project/pkg/api/
 COPY pkg/api/binoculars/*.swagger.json /project/pkg/api/binoculars/
+COPY pkg/api/schedulerobjects/*.swagger.json /project/pkg/api/schedulerobjects/
 RUN ./project/internal/lookout/ui/openapi.sh
 
 FROM ${NODE_BUILD_IMAGE} AS NODE

cmd/server/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/armadaproject/armada/internal/server"
 	"github.com/armadaproject/armada/internal/server/configuration"
 	"github.com/armadaproject/armada/pkg/api"
+	"github.com/armadaproject/armada/pkg/api/schedulerobjects"
 )
 
 const CustomConfigLocation string = "config"
@@ -92,6 +93,7 @@ func main() {
 		api.RegisterSubmitHandler,
 		api.RegisterEventHandler,
 		api.RegisterJobsHandler,
+		schedulerobjects.RegisterSchedulerReportingHandler,
 	)
 	defer shutdownGateway()
 

docs/python_airflow_operator.md

@@ -63,7 +63,7 @@ and handles job cancellation if the Airflow task is killed.
     * **reattach_policy** (*Optional**[**str**] **| **Callable**[**[**JobState**, **str**]**, **bool**]*) – 
 
 
-    * **extra_links** (*Optional**[**Dict**[**str**, **str**]**]*) – 
+    * **extra_links** (*Optional**[**Dict**[**str**, **Union**[**str**, **re.Pattern**]**]**]*) – 
 
 
 
@@ -199,74 +199,10 @@ acknowledged by Armada.
 :param reattach_policy: Operator reattach policy to use (defaults to: never)
 :type reattach_policy: Optional[str] | Callable[[JobState, str], bool]
 :param kwargs: Additional keyword arguments to pass to the BaseOperator.
-:param extra_links: Extra links to be shown in addition to Lookout URL.
-:type extra_links: Optional[Dict[str, str]]
+:param extra_links: Extra links to be shown in addition to Lookout URL. Regex patterns will be extracted from container logs (taking first match).
+:type extra_links: Optional[Dict[str, Union[str, re.Pattern]]]
 :param kwargs: Additional keyword arguments to pass to the BaseOperator.
 
-
-### _class_ armada.operators.armada.DynamicLink(name)
-Bases: `BaseOperatorLink`, `LoggingMixin`
-
-
-* **Parameters**
-
-    **name** (*str*) – 
-
-
-
-#### get_link(operator, \*, ti_key)
-Link to external system.
-
-Note: The old signature of this function was `(self, operator, dttm: datetime)`. That is still
-supported at runtime but is deprecated.
-
-
-* **Parameters**
-
-
-    * **operator** (*BaseOperator*) – The Airflow operator object this link is associated to.
-
-
-    * **ti_key** (*TaskInstanceKey*) – TaskInstance ID to return link for.
-
-
-
-* **Returns**
-
-    link to external system
-
-
-
-#### name(_: st_ )
-
-### _class_ armada.operators.armada.LookoutLink()
-Bases: `BaseOperatorLink`
-
-
-#### get_link(operator, \*, ti_key)
-Link to external system.
-
-Note: The old signature of this function was `(self, operator, dttm: datetime)`. That is still
-supported at runtime but is deprecated.
-
-
-* **Parameters**
-
-
-    * **operator** (*BaseOperator*) – The Airflow operator object this link is associated to.
-
-
-    * **ti_key** (*TaskInstanceKey*) – TaskInstance ID to return link for.
-
-
-
-* **Returns**
-
-    link to external system
-
-
-
-#### name(_ = 'Lookout_ )
 ## armada.triggers.armada module
 
 ## armada.auth module

internal/armadactl/scheduling.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/armadaproject/armada/internal/common"
-	"github.com/armadaproject/armada/internal/scheduler/schedulerobjects"
+	"github.com/armadaproject/armada/pkg/api/schedulerobjects"
 	"github.com/armadaproject/armada/pkg/client"
 )
 

internal/common/logging/global.go

@@ -6,11 +6,11 @@ import (
 	"github.com/rs/zerolog"
 )
 
-// The global Logger.  Comes configured with some sensible defaults for e.g. unit tests, but applications should
-// generally configure their own logging config via ReplaceStdLogger
 var (
 	StdSkipFrames = 4
-	stdLogger     = createDefaultLogger()
+	// The global Logger.  Comes configured with some sensible defaults for e.g. unit tests, but applications should
+	// generally configure their own logging config via ReplaceStdLogger
+	stdLogger = createDefaultLogger()
 )
 
 // ReplaceStdLogger Replaces the global logger.  This should be called once at app startup!
@@ -23,57 +23,57 @@ func StdLogger() *Logger {
 	return stdLogger
 }
 
-// Debug logs a message at level Debug.
+// Debug logs a message at level Debug on the default logger.
 func Debug(args ...any) {
 	stdLogger.Debug(args...)
 }
 
-// Info logs a message at level Info.
+// Info logs a message at level Info on the default logger.
 func Info(args ...any) {
 	stdLogger.Info(args...)
 }
 
-// Warn logs a message at level Warn on the standard logger.
+// Warn logs a message at level Warn on the default logger.
 func Warn(args ...any) {
 	stdLogger.Warn(args...)
 }
 
-// Error logs a message at level Error on the standard logger.
+// Error logs a message at level Error on the default logger.
 func Error(args ...any) {
 	stdLogger.Error(args...)
 }
 
-// Panic logs a message at level Panic on the standard logger.
+// Panic logs a message at level Panic on the default logger.
 func Panic(args ...any) {
 	stdLogger.Panic(args...)
 }
 
-// Fatal logs a message at level Fatal on the standard logger then the process will exit with status set to 1.
+// Fatal logs a message at level Fatal on the default logger then the process will exit with status set to 1.
 func Fatal(args ...any) {
 	stdLogger.Fatal(args...)
 }
 
-// Debugf logs a message at level Debug on the standard logger.
+// Debugf logs a message at level Debug on the default logger.
 func Debugf(format string, args ...any) {
 	stdLogger.Debugf(format, args...)
 }
 
-// Infof logs a message at level Info on the standard logger.
+// Infof logs a message at level Info on the default logger.
 func Infof(format string, args ...any) {
 	stdLogger.Infof(format, args...)
 }
 
-// Warnf logs a message at level Warn on the standard logger.
+// Warnf logs a message at level Warn on the default logger.
 func Warnf(format string, args ...any) {
 	stdLogger.Warnf(format, args...)
 }
 
-// Errorf logs a message at level Error on the standard logger.
+// Errorf logs a message at level Error on the default logger.
 func Errorf(format string, args ...any) {
 	stdLogger.Errorf(format, args...)
 }
 
-// Fatalf logs a message at level Fatal on the standard logger then the process will exit with status set to 1.
+// Fatalf logs a message at level Fatal on the default logger then the process will exit with status set to 1.
 func Fatalf(format string, args ...any) {
 	stdLogger.Fatalf(format, args...)
 }

internal/common/resource/resource_test.go

@@ -142,7 +142,7 @@ func TestTotalResourceRequest_ShouldSumAllContainers(t *testing.T) {
 func TestTotalResourceRequestShouldReportMaxInitContainerValues(t *testing.T) {
 	highCpuResource := makeContainerResource(1000, 5)
 	highRamResource := makeContainerResource(100, 500)
-	// WithField init containers, it should take the max of each individual resource from all init containers
+	// With init containers, it should take the max of each individual resource from all init containers
 	expectedResult := makeContainerResource(1000, 500)
 
 	pod := makePodWithResource([]*v1.ResourceList{}, []*v1.ResourceList{&highCpuResource, &highRamResource})

internal/lookout/ui/openapi.sh

@@ -6,3 +6,7 @@
     -g typescript-fetch \
     -i /project/pkg/api/binoculars/api.swagger.json \
     -o /project/internal/lookout/ui/src/openapi/binoculars
+/usr/local/bin/docker-entrypoint.sh generate \
+    -g typescript-fetch \
+    -i /project/pkg/api/schedulerobjects/api.swagger.json \
+    -o /project/internal/lookout/ui/src/openapi/schedulerobjects

internal/lookout/ui/src/components/CodeBlock.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from "react"
+import { ReactNode, useCallback } from "react"
 
 import { Download } from "@mui/icons-material"
 import { IconButton, Skeleton, styled, useColorScheme } from "@mui/material"
@@ -45,6 +45,7 @@ const StyledPre = styled("pre")(({ theme }) => ({
   minHeight: 50,
   display: "flex",
   alignItems: "center",
+  margin: 0,
 }))
 
 const Code = styled("code")({
@@ -72,6 +73,12 @@ const CodeLineNumber = styled("span")({
   },
 })
 
+interface CodeBlockActionProps {
+  title: string
+  onClick: () => void
+  icon: ReactNode
+}
+
 interface CodeBlockLoadingProps {
   loading: true
   code?: undefined | string
@@ -96,13 +103,14 @@ interface CodeBlockNonDownloadbaleProps {
   downloadFileName?: undefined | string
 }
 
-interface CodeBlockbaseProps {
+interface CodeBlockBaseProps {
   showLineNumbers: boolean
   loadingLines?: number
   loadingLineLength?: number
+  additionalActions?: CodeBlockActionProps[]
 }
 
-export type CodeBlockProps = CodeBlockbaseProps &
+export type CodeBlockProps = CodeBlockBaseProps &
   (CodeBlockLoadedProps | CodeBlockLoadingProps) &
   (CodeBlockDownloadbaleProps | CodeBlockNonDownloadbaleProps)
 
@@ -116,6 +124,7 @@ export const CodeBlock = ({
   showLineNumbers,
   loadingLines = DEFAULT_LOADING_LINES,
   loadingLineLength = DEFAULT_LOADING_LINE_LENGTH,
+  additionalActions = [],
 }: CodeBlockProps) => {
   const { colorScheme } = useColorScheme()
 
@@ -177,6 +186,11 @@ export const CodeBlock = ({
             <Download />
           </IconButton>
         )}
+        {additionalActions.map(({ title, onClick, icon }) => (
+          <IconButton key={title} size="small" title={title} onClick={onClick}>
+            {icon}
+          </IconButton>
+        ))}
       </CodeActionsContainer>
       <Highlight
         prism={Prism}