Skip to content

Commit

Permalink
testing volume id
Browse files Browse the repository at this point in the history
  • Loading branch information
@lola831
lola831 committed Oct 8, 2024
1 parent bea467e commit 5b8e3bd
Showing 1 changed file with 159 additions and 168 deletions.
327 changes: 159 additions & 168 deletions .github/workflows/run-simulators.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ jobs:
runs-on: ubuntu-latest
concurrency:
group: sim
env:
INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
steps:
- name: Create Volume from Latest Snapshot and Attach to Instance
env:
INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
run: |
# Retrieve the latest snapshot ID
LATEST_SNAPSHOT_ID=$(aws ec2 describe-snapshots --owner-ids self --query 'Snapshots | sort_by(@, &StartTime) | [-1].SnapshotId' --output text)
Expand All @@ -25,9 +25,7 @@ jobs:
# Create a new volume from the latest snapshot
volume_id=$(aws ec2 create-volume --snapshot-id $LATEST_SNAPSHOT_ID --availability-zone $(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[0].Instances[0].Placement.AvailabilityZone" --output text) --volume-type gp2 --query "VolumeId" --output text)
echo "Created volume with ID: $volume_id"
# Store volume_id in GITHUB_ENV
echo "volume_id=$volume_id" >> $GITHUB_ENV
echo "::set-output name=volume_id::$volume_id"
# Wait until the volume is available
aws ec2 wait volume-available --volume-ids $volume_id
Expand All @@ -37,110 +35,105 @@ jobs:
aws ec2 attach-volume --volume-id $volume_id --instance-id $INSTANCE_ID --device /dev/sda1
echo "Volume $volume_id attached to instance $INSTANCE_ID as /dev/sda1"
- name: Start EC2 Instance
env:
INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
run: |
# Get the instance state
instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
# If the machine is stopping wait for it to fully stop
while [ "$instance_state" == "stopping" ]; do
echo "Instance is stopping, waiting for it to fully stop..."
sleep 10
instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
done
# Check if instance state is "stopped"
if [[ "$instance_state" == "stopped" ]]; then
echo "Instance is stopped, starting it..."
aws ec2 start-instances --instance-ids $INSTANCE_ID
elif [[ "$instance_state" == "pending" ]]; then
echo "Instance startup is pending, continuing..."
elif [[ "$instance_state" == "running" ]]; then
echo "Instance is already running..."
exit 0
else
echo "Unknown instance state: $instance_state"
exit 1
fi
# wait for status checks to pass
TIMEOUT=300 # Timeout in seconds
START_TIME=$(date +%s)
END_TIME=$((START_TIME + TIMEOUT))
while true; do
response=$(aws ec2 describe-instance-status --instance-ids $INSTANCE_ID)
system_status=$(echo "$response" | jq -r '.InstanceStatuses[0].SystemStatus.Status')
instance_status=$(echo "$response" | jq -r '.InstanceStatuses[0].InstanceStatus.Status')
if [[ "$system_status" == "ok" && "$instance_status" == "ok" ]]; then
echo "Both SystemStatus and InstanceStatus are 'ok'"
exit 0
fi
CURRENT_TIME=$(date +%s)
if [[ "$CURRENT_TIME" -ge "$END_TIME" ]]; then
echo "Timeout: Both SystemStatus and InstanceStatus have not reached 'ok' state within $TIMEOUT seconds."
exit 1
fi
sleep 10 # Check status every 10 seconds
done
check_simulator_version_updates:
name: check_simulator_version_updates
runs-on: ubuntu-latest
needs: start_ec2_instance
steps:
- name: Check for Simulator Version Updates
env:
PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HOSTNAME: ${{ secrets.SSH_HOST }}
USER_NAME: ${{ secrets.SSH_USERNAME }}
GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
run: |
echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
cd /home/ubuntu/actions/ &&
rm -rf Scenic &&
git clone --branch $(basename "${{ github.ref }}") --single-branch https://[email protected]/BerkeleyLearnVerify/Scenic.git &&
cd Scenic &&
python3 -m venv venv &&
source venv/bin/activate &&
python3 -m pip install -e .[test-full] &&
python3 .github/check_latest_simulators.py
'
check_nvidia_smi:
name: check_nvidia_smi
runs-on: ubuntu-latest
needs: start_ec2_instance
continue-on-error: true
steps:
- name: Check NVIDIA SMI
env:
PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HOSTNAME: ${{ secrets.SSH_HOST}}
USER_NAME: ${{ secrets.SSH_USERNAME}}
run: |
echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
output=$(nvidia-smi)
echo "$output"
if [ -z "$output" ]; then
echo "NVIDIA Driver is not set"
exit 1
fi
'
- name: NVIDIA Driver is not set
if: ${{ failure() }}
run: |
echo "NVIDIA SMI is not working, please run the steps here on the instance:"
echo "https://scenic-lang.atlassian.net/wiki/spaces/KAN/pages/2785287/Setting+Up+AWS+VM?parentProduct=JSW&initialAllowedFeatures=byline-contributors.byline-extensions.page-comments.delete.page-reactions.inline-comments.non-licensed-share&themeState=dark%253Adark%2520light%253Alight%2520spacing%253Aspacing%2520colorMode%253Alight&locale=en-US#Install-NVIDIA-Drivers"
# - name: Start EC2 Instance
# run: |
# # Get the instance state
# instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')

# # If the machine is stopping wait for it to fully stop
# while [ "$instance_state" == "stopping" ]; do
# echo "Instance is stopping, waiting for it to fully stop..."
# sleep 10
# instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
# done

# # Check if instance state is "stopped"
# if [[ "$instance_state" == "stopped" ]]; then
# echo "Instance is stopped, starting it..."
# aws ec2 start-instances --instance-ids $INSTANCE_ID
# elif [[ "$instance_state" == "pending" ]]; then
# echo "Instance startup is pending, continuing..."
# elif [[ "$instance_state" == "running" ]]; then
# echo "Instance is already running..."
# exit 0
# else
# echo "Unknown instance state: $instance_state"
# exit 1
# fi

# # wait for status checks to pass
# TIMEOUT=300 # Timeout in seconds
# START_TIME=$(date +%s)
# END_TIME=$((START_TIME + TIMEOUT))
# while true; do
# response=$(aws ec2 describe-instance-status --instance-ids $INSTANCE_ID)
# system_status=$(echo "$response" | jq -r '.InstanceStatuses[0].SystemStatus.Status')
# instance_status=$(echo "$response" | jq -r '.InstanceStatuses[0].InstanceStatus.Status')

# if [[ "$system_status" == "ok" && "$instance_status" == "ok" ]]; then
# echo "Both SystemStatus and InstanceStatus are 'ok'"
# exit 0
# fi

# CURRENT_TIME=$(date +%s)
# if [[ "$CURRENT_TIME" -ge "$END_TIME" ]]; then
# echo "Timeout: Both SystemStatus and InstanceStatus have not reached 'ok' state within $TIMEOUT seconds."
# exit 1
# fi

# sleep 10 # Check status every 10 seconds
# done

# check_simulator_version_updates:
# name: check_simulator_version_updates
# runs-on: ubuntu-latest
# needs: start_ec2_instance
# steps:
# - name: Check for Simulator Version Updates
# env:
# PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
# HOSTNAME: ${{ secrets.SSH_HOST }}
# USER_NAME: ${{ secrets.SSH_USERNAME }}
# GH_ACCESS_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
# run: |
# echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
# ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
# cd /home/ubuntu/actions/ &&
# rm -rf Scenic &&
# git clone --branch $(basename "${{ github.ref }}") --single-branch https://[email protected]/BerkeleyLearnVerify/Scenic.git &&
# cd Scenic &&
# python3 -m venv venv &&
# source venv/bin/activate &&
# python3 -m pip install -e .[test-full] &&
# python3 .github/check_latest_simulators.py
# '

# check_nvidia_smi:
# name: check_nvidia_smi
# runs-on: ubuntu-latest
# needs: start_ec2_instance
# continue-on-error: true
# steps:
# - name: Check NVIDIA SMI
# env:
# PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
# HOSTNAME: ${{ secrets.SSH_HOST}}
# USER_NAME: ${{ secrets.SSH_USERNAME}}
# run: |
# echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
# ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
# output=$(nvidia-smi)
# echo "$output"
# if [ -z "$output" ]; then
# echo "NVIDIA Driver is not set"
# exit 1
# fi
# '
# - name: NVIDIA Driver is not set
# if: ${{ failure() }}
# run: |
# echo "NVIDIA SMI is not working, please run the steps here on the instance:"
# echo "https://scenic-lang.atlassian.net/wiki/spaces/KAN/pages/2785287/Setting+Up+AWS+VM?parentProduct=JSW&initialAllowedFeatures=byline-contributors.byline-extensions.page-comments.delete.page-reactions.inline-comments.non-licensed-share&themeState=dark%253Adark%2520light%253Alight%2520spacing%253Aspacing%2520colorMode%253Alight&locale=en-US#Install-NVIDIA-Drivers"

# run_carla_simulators:
# name: run_carla_simulators
Expand All @@ -165,65 +158,61 @@ jobs:
# done
# '

run_webots_simulators:
name: run_webots_simulators
runs-on: ubuntu-latest
needs: [check_simulator_version_updates, check_nvidia_smi]
steps:
- name: Run Webots Tests
env:
PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
HOSTNAME: ${{secrets.SSH_HOST}}
USER_NAME: ${{secrets.SSH_USERNAME}}
run: |
echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
Xvfb :99 -screen 0 1024x768x16 &
cd /home/ubuntu/actions/Scenic &&
source venv/bin/activate &&
webots_versions=($(find /software -maxdepth 1 -type d -name 'webots*')) &&
export DISPLAY=:99 &&
for version in "${webots_versions[@]}"; do
echo "============================= Webots $version ============================="
export WEBOTS_ROOT="$version"
pytest tests/simulators/webots
done
kill %1
'
# run_webots_simulators:
# name: run_webots_simulators
# runs-on: ubuntu-latest
# needs: [check_simulator_version_updates, check_nvidia_smi]
# steps:
# - name: Run Webots Tests
# env:
# PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
# HOSTNAME: ${{secrets.SSH_HOST}}
# USER_NAME: ${{secrets.SSH_USERNAME}}
# run: |
# echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
# ssh -o StrictHostKeyChecking=no -i private_key ${USER_NAME}@${HOSTNAME} '
# Xvfb :99 -screen 0 1024x768x16 &
# cd /home/ubuntu/actions/Scenic &&
# source venv/bin/activate &&
# webots_versions=($(find /software -maxdepth 1 -type d -name 'webots*')) &&
# export DISPLAY=:99 &&
# for version in "${webots_versions[@]}"; do
# echo "============================= Webots $version ============================="
# export WEBOTS_ROOT="$version"
# pytest tests/simulators/webots
# done
# kill %1
# '

stop_ec2_instance:
name: stop_ec2_instance
runs-on: ubuntu-latest
needs: [start_ec2_instance, check_simulator_version_updates, check_nvidia_smi, run_webots_simulators]
needs: [start_ec2_instance ]
if: always()
env:
VOLUME_ID: ${{ needs.start_ec2_instance.outputs.volume_id }}
INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
steps:
- name: Stop EC2 Instance
env:
INSTANCE_ID: ${{ secrets.AWS_EC2_INSTANCE_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
run: |
# Get the instance state and stop it if running
instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
if [[ "$instance_state" == "running" ]]; then
echo "Instance is running, stopping it..."
aws ec2 stop-instances --instance-ids $INSTANCE_ID
aws ec2 wait instance-stopped --instance-ids $INSTANCE_ID
echo "Instance has stopped."
elif [[ "$instance_state" == "stopped" ]]; then
echo "Instance is already stopped."
else
echo "Unexpected instance state: $instance_state"
exit 1
fi
# - name: Stop EC2 Instance
# run: |
# # Get the instance state and stop it if running
# instance_state=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].State.Name')
# if [[ "$instance_state" == "running" ]]; then
# echo "Instance is running, stopping it..."
# aws ec2 stop-instances --instance-ids $INSTANCE_ID
# aws ec2 wait instance-stopped --instance-ids $INSTANCE_ID
# echo "Instance has stopped."
# elif [[ "$instance_state" == "stopped" ]]; then
# echo "Instance is already stopped."
# else
# echo "Unexpected instance state: $instance_state"
# exit 1
# fi

- name: Take Snapshot of Volume
env:
VOLUME_ID: ${{ env.volume_id }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
run: |
# Create a snapshot of the volume
echo "Volume ID is: $VOLUME_ID"
Expand All @@ -234,12 +223,14 @@ jobs:
aws ec2 wait snapshot-completed --snapshot-ids $snapshot_id
echo "Snapshot completed."
- name: Detach Volume
run: |
# Detach the volume
aws ec2 detach-volume --volume-id $VOLUME_ID
aws ec2 wait volume-available --volume-ids $VOLUME_ID
echo "Volume $VOLUME_ID detached."
- name: Delete Volume
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
VOLUME_ID: ${{ env.volume_id }}
run: |
# Delete the volume after snapshot is complete
aws ec2 delete-volume --volume-id $VOLUME_ID
Expand Down

0 comments on commit 5b8e3bd

Please sign in to comment.