dynamic holodeck ci instance name

Signed-off-by: shiva kumar <[email protected]> unused aws instace and vpcs cleanup unused aws instace and vpcs cleanup Signed-off-by: shiva kumar <[email protected]> unused aws instace and vpcs cleanup Signed-off-by: shiva kumar <[email protected]> unused aws instace and vpcs cleanup Signed-off-by: shiva kumar <[email protected]>
NVIDIA · Dec 18, 2024 · 88a1ff0 · 88a1ff0
1 parent c5b9835
commit 88a1ff0
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 41 deletions.
diff --git a/.github/workflows/awscleanup.yaml b/.github/workflows/awscleanup.yaml
@@ -33,14 +33,6 @@ jobs:
       - name: Identify resources running longer than 4 hours
         id: identify-resources
         run: |
-          # Find EC2 instances with names ci* running longer than 4 hours
-          running_instances=$(aws ec2 describe-instances \
-            --filters Name=instance-state-name,Values=running Name=tag:Name,Values=ci* \
-            --query "Reservations[*].Instances[?LaunchTime<=\`$(date -u -d '4 hours ago' +%Y-%m-%dT%H:%M:%SZ)\`].InstanceId" \
-            --output text | tr -d '\r' | tr '\n' ' ')
-          echo "Found instances: $running_instances"
-          echo "instances=$running_instances" >> $GITHUB_ENV
-
           # Find vpcs with names ci*
           vpcs=$(aws ec2 describe-vpcs \
             --filters "Name=tag:Name,Values=ci*" \
@@ -49,29 +41,28 @@ jobs:
           echo "Found VPCs: $vpcs"
           echo "vpcs=$vpcs" >> $GITHUB_ENV
 
-      - name: Terminate EC2 Instances
-        if: env.instances != ''
-        run: |
-          for instance in $instances; do
-            echo "Terminating instance: $instance"
-            aws ec2 terminate-instances --instance-ids "$instance"
-          done
-
       - name: Clean up VPCs
         if: env.vpcs != ''
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           for vpc in $vpcs; do
-            # Check for EC2 instances attached to VPC
-            instances_in_vpc=$(aws ec2 describe-instances \
-              --filters "Name=vpc-id,Values=$vpc" \
-              --query "Reservations[*].Instances[*].InstanceId" \
-              --output text)
-            # if no instance attached delete it
-            if [ -z "$instances_in_vpc" ]; then
-              scripts/awsvpcscleanup.sh $vpc
-            else
-              echo "EC2 instances are still attached to VPC: $vpc. Skipping deletion."
+
+            GITHUB_REPOSITORY1=gpu-driver-container
+            RUN_ID1="12338981626"
+            JOB_NAME=e2e-tests-nvidiadriver
+            response=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
+              "https://api.github.com/repos/NVIDIA/${GITHUB_REPOSITORY1}/actions/runs/${RUN_ID1}/jobs")
+            echo "SHIVA $response"
+            status=$(echo "$response" | jq -r ".jobs[] | select(.name | test(\"^$JOB_NAME\")) | .status")
+            
+            echo "status=$status"
+            if [[ $status != 'queued' && $status != 'in_progress' ]]; then
+              echo "Holodeck Job status is not in running stage , Delete the dependend resources"
             fi
+
+
+            scripts/awsvpcscleanup.sh $vpc
           done
 
       - name: Post cleanup

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -20,12 +20,12 @@ on:
       - opened
       - synchronize
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
   push:
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
   schedule:
     - cron: '31 11 * * 4'
 

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -21,8 +21,8 @@ on:
       - completed
     branches:
       - "pull-request/[0-9]+"
-      - main
-      - release-*
+      - main-no
+      - release-no
 
 jobs:
   e2e-test:

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -17,12 +17,12 @@ name: Go
 on:
   push:
     branches: 
-      - main
-      - release-*
+      - main-no
+      - release-no
   pull_request:
     branches:
-      - main
-      - release-*
+      - main-no
+      - release-no
 
 jobs:
   build:

diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
@@ -17,14 +17,14 @@ name: Image
 on:
   pull_request:
     branches:
-      - 'main'
-      - 'release-*'
+      - 'main-no'
+      - 'release-no'
   push:
     tags:
       - 'v*.*.*'  
     branches:
-      - 'main'
-      - 'release-*'
+      - 'main-no'
+      - 'release-no'
 
 jobs:
   docker:

diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go
@@ -19,6 +19,7 @@ package aws
 import (
 	"context"
 	"os"
+	"strings"
 
 	"github.com/NVIDIA/holodeck/api/holodeck/v1alpha1"
 	"github.com/NVIDIA/holodeck/internal/logger"
@@ -87,6 +88,20 @@ func New(log *logger.FunLogger, env v1alpha1.Environment, cacheFile string) (*Pr
 	if envRegion := os.Getenv("AWS_REGION"); envRegion != "" {
 		region = envRegion
 	}
+	sha := os.Getenv("GITHUB_SHA")
+	// short sha
+	if len(sha) > 8 {
+		sha = sha[:8]
+	}
+	actor := os.Getenv("GITHUB_ACTOR")
+	branchName := os.Getenv("GITHUB_REF_NAME")
+	repo_name := os.Getenv("GITHUB_REPOSITORY")
+	parts := strings.Split(repo_name, "/")
+	repoName := parts[len(parts)-1]
+	githubRunId := os.Getenv("GITHUB_RUN_ID")
+	githubRunNumber := os.Getenv("GITHUB_RUN_NUMBER")
+	githubJob := os.Getenv("GITHUB_JOB")
+
 	cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region))
 	if err != nil {
 		return nil, err
@@ -100,6 +115,13 @@ func New(log *logger.FunLogger, env v1alpha1.Environment, cacheFile string) (*Pr
 			{Key: aws.String("Name"), Value: aws.String(env.Name)},
 			{Key: aws.String("Project"), Value: aws.String("holodeck")},
 			{Key: aws.String("Environment"), Value: aws.String("cicd")},
+			{Key: aws.String("Sha"), Value: aws.String(sha)},
+			{Key: aws.String("Actor"), Value: aws.String(actor)},
+			{Key: aws.String("BranchName"), Value: aws.String(branchName)},
+			{Key: aws.String("RepoName"), Value: aws.String(repoName)},
+			{Key: aws.String("GithubRunId"), Value: aws.String(githubRunId)},
+			{Key: aws.String("GithubRunNumber"), Value: aws.String(githubRunNumber)},
+			{Key: aws.String("githubJob"), Value: aws.String(githubJob)},
 		},
 		client,
 		r53,

diff --git a/pkg/provider/aws/create.go b/pkg/provider/aws/create.go
@@ -145,7 +145,14 @@ func (p *Provider) createInternetGateway(cache *AWS) error {
 	p.log.Wg.Add(1)
 	go p.log.Loading("Creating Internet Gateway")
 
-	gwInput := &ec2.CreateInternetGatewayInput{}
+	gwInput := &ec2.CreateInternetGatewayInput{
+		TagSpecifications: []types.TagSpecification{
+			{
+				ResourceType: types.ResourceTypeInternetGateway,
+				Tags:         p.Tags,
+			},
+		},
+	}
 	gwOutput, err := p.ec2.CreateInternetGateway(context.TODO(), gwInput)
 	if err != nil {
 		p.fail()
@@ -357,6 +364,17 @@ func (p *Provider) createEC2Instance(cache *AWS) error {
 	}
 	cache.PublicDnsName = *instanceRunning.Reservations[0].Instances[0].PublicDnsName
 
+	// tag network interface
+	instance := instanceOut.Instances[0]
+	networkInterfaceId := *instance.NetworkInterfaces[0].NetworkInterfaceId
+	_, err = p.ec2.CreateTags(context.TODO(), &ec2.CreateTagsInput{
+		Resources: []string{networkInterfaceId},
+		Tags:      p.Tags,
+	})
+	if err != nil {
+		p.fail()
+		return fmt.Errorf("Fail to tag network to instance: %v", err)
+	}
 	p.done()
 	return nil
 }
diff --git a/scripts/awsvpcscleanup.sh b/scripts/awsvpcscleanup.sh
@@ -8,6 +8,17 @@ export vpc=$1
 
 echo "Start Deleting  VPC: $vpc resource"
 
+aws resourcegroupstaggingapi get-resources --tag-filters Key=vpc-id,Values=$vpcid
+
+aws_resources=$(aws resourcegroupstaggingapi get-resources --tag-filters Key=vpc-id,Values=$vpcid)
+
+exit 1
+# Delete Instance
+for instance in $instances; do
+    echo "Terminating instance: $instance"
+    aws ec2 terminate-instances --instance-ids "$instance"
+done
+
 # Delete Internet Gateway
 internet_gateways=$(aws ec2 describe-internet-gateways \
     --filters Name=attachment.vpc-id,Values=$vpc \