Skip to content

Commit

Permalink
Merge branch 'master' into f/chrisma/remove-lookout
Browse files Browse the repository at this point in the history
  • Loading branch information
JamesMurkin authored Feb 9, 2024
2 parents 3925c66 + 1fd83fd commit e5cb0c1
Show file tree
Hide file tree
Showing 30 changed files with 854 additions and 508 deletions.
97 changes: 97 additions & 0 deletions .github/workflows/check-required.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
name: Check required jobs

# This workflow is triggered when a workflow run for the CI is completed.
# It checks if the "All required checks done" job was actually successful
# (and not just skipped) and creates a check run if that is the case. The
# check run can be used to protect the main branch from being merged if the
# CI is not passing. We need to use a GitHub app token to create the check
# run because otherwise the check suite will be assigned to the first workflow
# run for the CI, which might not be the latest one. See
# https://github.com/orgs/community/discussions/24616#discussioncomment-6088422
# for more details.

on:
workflow_run:
workflows: [CI]

permissions:
actions: read
checks: write

jobs:
required-jobs:
name: Check required jobs
if: ${{ !github.event.repository.fork }}
environment: create-check
runs-on: ubuntu-latest
steps:
- name: Generate an app token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}

- uses: actions/github-script@v7
with:
github-token: ${{ steps.app-token.outputs.token }}
script: |
const ghaAppId = 15368;
const ghaName = 'All required checks done';
const myAppId = ${{ secrets.APP_ID }};
const myName = 'All required checks succeeded';
const owner = context.payload.repository.owner.login;
const repo = context.payload.repository.name;
const sha = context.payload.workflow_run.head_sha;
core.info(`List GitHub Actions check runs for ${sha}.`)
const { data: { check_runs: ghaChecks } } = await github.rest.checks.listForRef({
owner: owner,
repo: repo,
ref: sha,
app_id: ghaAppId,
check_name: ghaName,
});
var newCheck = {
owner: owner,
repo: repo,
name: myName,
head_sha: sha,
status: 'in_progress',
started_at: context.payload.workflow_run.created_at,
output: {
title: 'Not all required checks succeeded',
},
};
core.summary.addHeading('The following required checks have been considered:', 3);
ghaChecks.forEach(check => {
core.summary
.addLink(check.name, check.html_url)
.addCodeBlock(JSON.stringify(check, ['status', 'conclusion', 'started_at', 'completed_at'], 2), 'json');
if (check.status === 'completed' && check.conclusion === 'success') {
newCheck.status = 'completed';
newCheck.conclusion = 'success';
newCheck.started_at = check.started_at;
newCheck.completed_at = check.completed_at;
newCheck.output.title = 'All required checks succeeded';
} else if (check.started_at > newCheck.started_at) {
newCheck.started_at = check.started_at;
}
});
if (ghaChecks.length === 0) {
core.summary.addRaw(`No check runs for ${sha} found.`);
}
newCheck.output.summary = core.summary.stringify();
await core.summary.write();
core.info(`Create own check run for ${sha}: ${JSON.stringify(newCheck, null, 2)}.`)
const { data: { html_url } } = await github.rest.checks.create(newCheck);
await core.summary
.addHeading('Check run created:', 3)
.addLink(myName, html_url)
.addCodeBlock(JSON.stringify(newCheck, null, 2), 'json')
.write();
16 changes: 5 additions & 11 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,23 +35,17 @@ jobs:
build:
if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
uses: ./.github/workflows/build.yml

# Virtual job that can be configured as a required check before a PR can be merged.
# As GitHub considers a check as successful if it is skipped, we need to check its status in
# another workflow (check-required.yml) and create a check there.
all-required-checks-done:
name: All required checks done
if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
needs:
- lint
- test
- build
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
steps:
- uses: actions/github-script@v6
with:
script: |
const results = ${{ toJSON(needs.*.result) }};
if (results.every(res => res === 'success')) {
core.info('All required checks succeeded');
} else {
core.setFailed('Some required checks failed');
}
- run: echo "All required checks done"

2 changes: 1 addition & 1 deletion .mergify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ pull_request_rules:
- "#approved-reviews-by>=2"
- and:
- "#approved-reviews-by>=1"
- "author~=^(JamesMurkin|severinson|d80tb7|carlocamurri|dejanzele|Sharpz7|ClifHouck|robertdavidsmith|theAntiYeti|richscott|suprjinx|zuqq|msumner91|MustafaI)"
- "author~=^(JamesMurkin|severinson|d80tb7|carlocamurri|dejanzele|Sharpz7|ClifHouck|robertdavidsmith|theAntiYeti|richscott|suprjinx|zuqq|msumner91|MustafaI|mijovicmia|masipauskas)"
title:
Two are checks required.
4 changes: 2 additions & 2 deletions .run/Armada Infrastructure Services.run.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<option name="envFilePath" value="" />
<option name="services">
<list>
<option value="lookoutv2-postgres-migration" />
<option value="lookoutv2-migration" />
<option value="scheduler-migration" />
</list>
</option>
Expand All @@ -16,4 +16,4 @@
<option name="RunConfigurationTask" enabled="true" run_configuration_name="Armada Infrastructure Services" run_configuration_type="docker-deploy" />
</method>
</configuration>
</component>
</component>
2 changes: 1 addition & 1 deletion client/python/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "armada_client"
version = "0.2.7"
version = "0.2.8"
description = "Armada gRPC API python client"
readme = "README.md"
requires-python = ">=3.7"
Expand Down
14 changes: 0 additions & 14 deletions deployment/queryapi/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,20 +69,6 @@ spec:
{{- end }}
securityContext:
allowPrivilegeEscalation: false
readinessProbe:
httpGet:
path: /health
port: web
initialDelaySeconds: 5
timeoutSeconds: 5
failureThreshold: 2
livenessProbe:
httpGet:
path: /health
port: web
initialDelaySeconds: 10
timeoutSeconds: 10
failureThreshold: 3
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
Expand Down
4 changes: 3 additions & 1 deletion developer/dependencies/postgres-init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
CREATE USER docker;
CREATE DATABASE postgresv2;
GRANT ALL PRIVILEGES ON DATABASE postgresv2 TO docker;
EOSQL
CREATE DATABASE jobservice;
GRANT ALL PRIVILEGES ON DATABASE jobservice TO docker;
EOSQL
2 changes: 1 addition & 1 deletion e2e/setup/jobservice.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ postgresConfig:
port: 5432
user: postgres
password: psw
dbname: postgres
dbname: jobservice
sslmode: disable
grpcPool:
initialConnections: 5
Expand Down
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
github.com/go-ldap/ldap/v3 v3.4.4
github.com/go-openapi/analysis v0.22.0
github.com/go-openapi/analysis v0.22.2
github.com/go-openapi/jsonreference v0.20.4
github.com/go-openapi/loads v0.21.5
github.com/go-openapi/runtime v0.26.0
github.com/go-openapi/spec v0.20.13
github.com/go-openapi/spec v0.20.14
github.com/go-redis/redis v6.15.9+incompatible
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.3
Expand Down Expand Up @@ -83,7 +83,7 @@ require (
github.com/gogo/status v1.1.1
github.com/golang/mock v1.6.0
github.com/goreleaser/goreleaser v1.15.2
github.com/jackc/pgx/v5 v5.5.0
github.com/jackc/pgx/v5 v5.5.2
github.com/jessevdk/go-flags v1.5.0
github.com/magefile/mage v1.14.0
github.com/minio/highwayhash v1.0.2
Expand Down
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-openapi/analysis v0.22.0 h1:wQ/d07nf78HNj4u+KiSY0sT234IAyePPbMgpUjUJQR0=
github.com/go-openapi/analysis v0.22.0/go.mod h1:acDnkkCI2QxIo8sSIPgmp1wUlRohV7vfGtAIVae73b0=
github.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0=
github.com/go-openapi/analysis v0.22.2/go.mod h1:pDF4UbZsQTo/oNuRfAWWd4dAh4yuYf//LYorPTjrpvo=
github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY=
github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
Expand All @@ -228,8 +228,8 @@ github.com/go-openapi/loads v0.21.5 h1:jDzF4dSoHw6ZFADCGltDb2lE4F6De7aWSpe+IcsRz
github.com/go-openapi/loads v0.21.5/go.mod h1:PxTsnFBoBe+z89riT+wYt3prmSBP6GDAQh2l9H1Flz8=
github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc=
github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ=
github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkIaE=
github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
github.com/go-openapi/strfmt v0.21.10 h1:JIsly3KXZB/Qf4UzvzJpg4OELH/0ASDQsyk//TTBDDk=
github.com/go-openapi/strfmt v0.21.10/go.mod h1:vNDMwbilnl7xKiO/Ve/8H8Bb2JIInBnH+lqiw6QWgis=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
Expand Down Expand Up @@ -475,8 +475,8 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E=
github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw=
github.com/jackc/pgx/v5 v5.5.0 h1:NxstgwndsTRy7eq9/kqYc/BZh5w2hHJV86wjvO+1xPw=
github.com/jackc/pgx/v5 v5.5.0/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jackc/pgx/v5 v5.5.2 h1:iLlpgp4Cp/gC9Xuscl7lFL1PhhW+ZLtXZcrfCt4C3tA=
github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
Expand Down
7 changes: 4 additions & 3 deletions internal/executor/util/kubernetes_objects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ func TestCreatePodFromExecutorApiJob(t *testing.T) {
Job: &armadaevents.SubmitJob{
ObjectMeta: &armadaevents.ObjectMeta{
Labels: map[string]string{},
Annotations: map[string]string{},
Annotations: map[string]string{"runtime_gang_cardinality": "3"},
Namespace: "test-namespace",
},
JobId: jobId,
Expand All @@ -520,8 +520,9 @@ func TestCreatePodFromExecutorApiJob(t *testing.T) {
domain.PodCount: "1",
},
Annotations: map[string]string{
domain.JobSetId: "job-set",
domain.Owner: "user",
domain.JobSetId: "job-set",
domain.Owner: "user",
"runtime_gang_cardinality": "3",
},
},
Spec: v1.PodSpec{
Expand Down
2 changes: 1 addition & 1 deletion internal/jobservice/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (s *JobServiceServer) GetJobStatus(ctx context.Context, opts *js.JobService
FromMessageId: fromMessageId,
}
}
log.Infof("Subscribing %s-%s with messageId %s", opts.Queue, opts.JobSetId, fromMessageId)
log.Infof("Subscribing to queue %s jobset %s messageId %s", opts.Queue, opts.JobSetId, fromMessageId)
} else {
if err := s.jobRepository.UpdateJobSetDb(ctx, opts.Queue, opts.JobSetId, fromMessageId); err != nil {
log.WithFields(requestFields).Warn(err)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@ export interface SidebarTabJobCommandsProps {
}

function getCommandText(job: Job, commandSpec: CommandSpec): string {
templateSettings.interpolate = /{{([\s\S]+?)}}/g
const compiledTemplate = template(commandSpec.template)
return compiledTemplate(job)
try {
templateSettings.interpolate = /{{([\s\S]+?)}}/g
const compiledTemplate = template(commandSpec.template)
return compiledTemplate(job)
} catch (error) {
console.error("Failed to generate command text:", error)
return "" // Return an empty string in case of failure
}
}

export const SidebarTabJobCommands = ({ job, commandSpecs }: SidebarTabJobCommandsProps) => {
Expand Down
2 changes: 2 additions & 0 deletions internal/lookoutv2/repository/aggregates.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ func GetAggregatorsForColumn(queryCol *queryColumn, aggregateType AggregateType,
aggregators[i] = NewStateCountAggregator(queryCol, state)
}
return aggregators, nil
case Min:
return []QueryAggregator{NewSqlFunctionAggregator(queryCol, "MIN")}, nil
default:
return nil, errors.Errorf("cannot determine aggregate type: %v", aggregateType)
}
Expand Down
8 changes: 4 additions & 4 deletions internal/lookoutv2/repository/groupjobs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -469,21 +469,21 @@ func TestGroupJobsWithMaxSubmittedTime(t *testing.T) {
Name: "job-set-1",
Count: 15,
Aggregates: map[string]interface{}{
"submitted": baseTime.Format(time.RFC3339),
"submitted": baseTime.Add(-2 * time.Minute).Format(time.RFC3339),
},
},
{
Name: "job-set-2",
Count: 12,
Aggregates: map[string]interface{}{
"submitted": baseTime.Add(-4 * time.Minute).Format(time.RFC3339),
"submitted": baseTime.Add(-6 * time.Minute).Format(time.RFC3339),
},
},
{
Name: "job-set-3",
Count: 18,
Aggregates: map[string]interface{}{
"submitted": baseTime.Add(-7 * time.Minute).Format(time.RFC3339),
"submitted": baseTime.Add(-9 * time.Minute).Format(time.RFC3339),
},
},
}, result.Groups)
Expand Down Expand Up @@ -949,7 +949,7 @@ func TestGroupJobsComplex(t *testing.T) {
Name: "job-set-1",
Count: 15,
Aggregates: map[string]interface{}{
"submitted": baseTime.Add(3 * time.Minute).Format(time.RFC3339),
"submitted": baseTime.Format(time.RFC3339),
"lastTransitionTime": baseTime.Add(5 * time.Minute).Format(time.RFC3339),
},
},
Expand Down
8 changes: 4 additions & 4 deletions internal/lookoutv2/repository/querybuilder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ func TestQueryBuilder_GroupBySingleAggregate(t *testing.T) {
)
assert.NoError(t, err)
assert.Equal(t, splitByWhitespace(`
SELECT j.jobset, COUNT(*) AS count, MAX(j.submitted) AS submitted
SELECT j.jobset, COUNT(*) AS count, MIN(j.submitted) AS submitted
FROM job AS j
INNER JOIN (
SELECT job_id
Expand Down Expand Up @@ -347,7 +347,7 @@ func TestQueryBuilder_GroupByMultipleAggregates(t *testing.T) {
)
assert.NoError(t, err)
assert.Equal(t, splitByWhitespace(`
SELECT j.jobset, COUNT(*) AS count, AVG(j.last_transition_time_seconds) AS last_transition_time_seconds, MAX(j.submitted) AS submitted
SELECT j.jobset, COUNT(*) AS count, AVG(j.last_transition_time_seconds) AS last_transition_time_seconds, MIN(j.submitted) AS submitted
FROM job AS j
INNER JOIN (
SELECT job_id
Expand Down Expand Up @@ -402,7 +402,7 @@ func TestQueryBuilder_GroupByStateAggregates(t *testing.T) {
SELECT j.jobset,
COUNT(*) AS count,
AVG(j.last_transition_time_seconds) AS last_transition_time_seconds,
MAX(j.submitted) AS submitted,
MIN(j.submitted) AS submitted,
SUM(CASE WHEN j.state = 1 THEN 1 ELSE 0 END) AS state_QUEUED,
SUM(CASE WHEN j.state = 8 THEN 1 ELSE 0 END) AS state_LEASED,
SUM(CASE WHEN j.state = 2 THEN 1 ELSE 0 END) AS state_PENDING,
Expand Down Expand Up @@ -448,7 +448,7 @@ func TestQueryBuilder_GroupByAnnotationMultipleAggregates(t *testing.T) {
)
assert.NoError(t, err)
assert.Equal(t, splitByWhitespace(`
SELECT ual_group.value, COUNT(*) AS count, AVG(j.last_transition_time_seconds) AS last_transition_time_seconds, MAX(j.submitted) AS submitted
SELECT ual_group.value, COUNT(*) AS count, AVG(j.last_transition_time_seconds) AS last_transition_time_seconds, MIN(j.submitted) AS submitted
FROM job AS j
INNER JOIN (
SELECT job_id
Expand Down
3 changes: 2 additions & 1 deletion internal/lookoutv2/repository/tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ const (
Max = 0
Average = 1
StateCounts = 2
Min = 3
)

type LookoutTables struct {
Expand Down Expand Up @@ -139,7 +140,7 @@ func NewTables() *LookoutTables {
stateCol,
}),
groupAggregates: map[string]AggregateType{
submittedCol: Max,
submittedCol: Min,
lastTransitionTimeCol: Average,
stateCol: StateCounts,
},
Expand Down
Loading

0 comments on commit e5cb0c1

Please sign in to comment.