autoscaler is not adding nodes

[xpillons]

running a non mpi job using “-l select=1:slot_type=execute:ungrouped=true” as a select statement.
The execute node array is not spot
But the autoscaler is not adding a new node

[xpillons@ondemand ~]$ qstat -fx 1651
Job Id: 1651.scheduler
Job_Name = sys-dashboard-sys-codeserver
Job_Owner = [email protected]
job_state = Q
queue = workq
server = scheduler
Checkpoint = u
ctime = Fri Nov 19 09:56:25 2021
Error_Path = ondemand.internal.cloudapp.net:/anfhome/xpillons/ondemand/data
/sys/dashboard/batch_connect/sys/codeserver/output/c1144623-b9b5-44a2-8
5b1-93fa66a0dc14/sys-dashboard-sys-codeserver.e1651
Hold_Types = n
Join_Path = oe
Keep_Files = n
Mail_Points = a
mtime = Fri Nov 19 09:56:25 2021
Output_Path = ondemand.internal.cloudapp.net:/anfhome/xpillons/ondemand/dat
a/sys/dashboard/batch_connect/sys/codeserver/output/c1144623-b9b5-44a2-
85b1-93fa66a0dc14/output.log
Priority = 0
qtime = Fri Nov 19 09:56:25 2021
Rerunable = True
Resource_List.ncpus = 1
Resource_List.nodect = 1
Resource_List.place = scatter:excl
Resource_List.select = 1:slot_type=execute:ungrouped=true
Resource_List.slot_type = execute
Resource_List.ungrouped = false
Resource_List.walltime = 03:00:00
Shell_Path_List = /bin/bash
substate = 10
Variable_List = PBS_O_HOME=/anfhome/xpillons,PBS_O_LANG=C,
PBS_O_LOGNAME=xpillons,
PBS_O_PATH=/var/www/ood/apps/sys/dashboard/tmp/node_modules/yarn/bin:/
opt/ood/ondemand/root/usr/share/gems/2.7/bin:/opt/rh/rh-nodejs12/root/u
sr/bin:/opt/rh/rh-ruby27/root/usr/local/bin:/opt/rh/rh-ruby27/root/usr/
bin:/opt/rh/httpd24/root/usr/bin:/opt/rh/httpd24/root/usr/sbin:/opt/ood
/ondemand/root/usr/bin:/opt/ood/ondemand/root/usr/sbin:/sbin:/bin:/usr/
sbin:/usr/bin,PBS_O_MAIL=/var/mail/root,PBS_O_SHELL=/bin/bash,
PBS_O_WORKDIR=/anfhome/xpillons/ondemand/data/sys/dashboard/batch_conn
ect/sys/codeserver/output/c1144623-b9b5-44a2-85b1-93fa66a0dc14,
PBS_O_SYSTEM=Linux,PBS_O_QUEUE=workq,
PBS_O_HOST=ondemand.internal.cloudapp.net
etime = Fri Nov 19 09:56:25 2021
Submit_arguments = -N sys-dashboard-sys-codeserver -S /bin/bash -o /anfhome
/xpillons/ondemand/data/sys/dashboard/batch_connect/sys/codeserver/outp
ut/c1144623-b9b5-44a2-85b1-93fa66a0dc14/output.log -j oe -l select=1:sl
ot_type=execute:ungrouped=true -l walltime=03:00:00
project = _pbs_project_default

[root@scheduler ~]# azpbs analyze --job-id 1651
NotInAPlacementGroup : Bucket[array=execute vm_size=Standard_F2s_v2 id=ac4fc82f-3d82-4f20-bd6e-67299dcdd388] is not in a placement group
NotInAPlacementGroup : Bucket[array=hb120rs_v2 vm_size=Standard_HB120rs_v2 id=4167f5f8-3a30-46df-92ba-8f77e0b636cd] is not in a placement group
NotInAPlacementGroup : Bucket[array=hb120rs_v3 vm_size=Standard_HB120rs_v3 id=aed72457-3b3c-44a0-ab8f-9ebb1a954fee] is not in a placement group
NotInAPlacementGroup : Bucket[array=hb60rs vm_size=Standard_HB60rs id=53bb8fd0-e62e-468f-b278-93da9e6fef3e] is not in a placement group
NotInAPlacementGroup : Bucket[array=hc44rs vm_size=Standard_HC44rs id=be3124ac-3894-496a-8580-ef06aba68273] is not in a placement group
NotInAPlacementGroup : Bucket[array=viz vm_size=Standard_D8s_v3 id=03e27828-b4bd-4678-a8e5-5886599bc6e5] is not in a placement group
NotInAPlacementGroup : Bucket[array=viz3d vm_size=Standard_NV6 id=ab76e8b1-0632-49a9-be16-93d880341855] is not in a placement group
InvalidOption : Resource[name=ungrouped value='false'] != 'true' for Bucket[array=execute vm_size=Standard_F2s_v2 attr=ungrouped]
InvalidOption : Resource[name=ungrouped value='false'] != 'true' for Bucket[array=execute vm_size=Standard_F2s_v2 attr=ungrouped]
InvalidOption : Resource[name=slot_type value='hb120rs_v2'] != 'execute' for Bucket[array=hb120rs_v2 vm_size=Standard_HB120rs_v2 attr=slot_type]
InvalidOption : Resource[name=slot_type value='hb120rs_v2'] != 'execute' for Bucket[array=hb120rs_v2 vm_size=Standard_HB120rs_v2 attr=slot_type]
InvalidOption : Resource[name=slot_type value='hb120rs_v3'] != 'execute' for Bucket[array=hb120rs_v3 vm_size=Standard_HB120rs_v3 attr=slot_type]
InvalidOption : Resource[name=slot_type value='hb120rs_v3'] != 'execute' for Bucket[array=hb120rs_v3 vm_size=Standard_HB120rs_v3 attr=slot_type]
InvalidOption : Resource[name=slot_type value='hb60rs'] != 'execute' for Bucket[array=hb60rs vm_size=Standard_HB60rs attr=slot_type]
InvalidOption : Resource[name=slot_type value='hb60rs'] != 'execute' for Bucket[array=hb60rs vm_size=Standard_HB60rs attr=slot_type]
InvalidOption : Resource[name=slot_type value='hc44rs'] != 'execute' for Bucket[array=hc44rs vm_size=Standard_HC44rs attr=slot_type]
InvalidOption : Resource[name=slot_type value='hc44rs'] != 'execute' for Bucket[array=hc44rs vm_size=Standard_HC44rs attr=slot_type]
InvalidOption : Resource[name=slot_type value='viz'] != 'execute' for Bucket[array=viz vm_size=Standard_D8s_v3 attr=slot_type]
InvalidOption : Resource[name=slot_type value='viz'] != 'execute' for Bucket[array=viz vm_size=Standard_D8s_v3 attr=slot_type]
InvalidOption : Resource[name=slot_type value='viz3d'] != 'execute' for Bucket[array=viz3d vm_size=Standard_NV6 attr=slot_type]
InvalidOption : Resource[name=slot_type value='viz3d'] != 'execute' for Bucket[array=viz3d vm_size=Standard_NV6 attr=slot_type]

NoCandidatesFound : SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=execute, vm_size=Standard_F2s_v2, pg=None),reason=Bucket[array=execute vm_size=Standard_F2s_v2 id=ac4fc82f-3d82-4f20-bd6e-67299dcdd388] is not
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=hb120rs_v2, vm_size=Standard_HB120rs_v2, pg=None),reason=Bucket[array=hb120rs_v2 vm_size=Standard_HB120rs_v2 id=4167f5f8-3a30-46df-92ba-8f77e0
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=hb120rs_v3, vm_size=Standard_HB120rs_v3, pg=None),reason=Bucket[array=hb120rs_v3 vm_size=Standard_HB120rs_v3 id=aed72457-3b3c-44a0-ab8f-9ebb1a
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=hb60rs, vm_size=Standard_HB60rs, pg=None),reason=Bucket[array=hb60rs vm_size=Standard_HB60rs id=53bb8fd0-e62e-468f-b278-93da9e6fef3e] is not i
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=hc44rs, vm_size=Standard_HC44rs, pg=None),reason=Bucket[array=hc44rs vm_size=Standard_HC44rs id=be3124ac-3894-496a-8580-ef06aba68273] is not i
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=viz, vm_size=Standard_D8s_v3, pg=None),reason=Bucket[array=viz vm_size=Standard_D8s_v3 id=03e27828-b4bd-4678-a8e5-5886599bc6e5] is not in a pl
SatisfiedResult(status=NotInAPlacementGroup, node=NodeBucket(nodearray=viz3d, vm_size=Standard_NV6, pg=None),reason=Bucket[array=viz3d vm_size=Standard_NV6 id=ab76e8b1-0632-49a9-be16-93d880341855] is not in a plac
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=execute, vm_size=Standard_F2s_v2, pg=Standard_F2s_v2_pg0),reason=Resource[name=ungrouped value='false'] != 'true' for Bucket[array=execute vm_size=St
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=execute, vm_size=Standard_F2s_v2, pg=Standard_F2s_v2_pg1),reason=Resource[name=ungrouped value='false'] != 'true' for Bucket[array=execute vm_size=St
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb120rs_v2, vm_size=Standard_HB120rs_v2, pg=Standard_HB120rs_v2_pg0),reason=Resource[name=slot_type value='hb120rs_v2'] != 'execute' for Bucket[array
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb120rs_v2, vm_size=Standard_HB120rs_v2, pg=Standard_HB120rs_v2_pg1),reason=Resource[name=slot_type value='hb120rs_v2'] != 'execute' for Bucket[array
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb120rs_v3, vm_size=Standard_HB120rs_v3, pg=Standard_HB120rs_v3_pg0),reason=Resource[name=slot_type value='hb120rs_v3'] != 'execute' for Bucket[array
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb120rs_v3, vm_size=Standard_HB120rs_v3, pg=Standard_HB120rs_v3_pg1),reason=Resource[name=slot_type value='hb120rs_v3'] != 'execute' for Bucket[array
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb60rs, vm_size=Standard_HB60rs, pg=Standard_HB60rs_pg0),reason=Resource[name=slot_type value='hb60rs'] != 'execute' for Bucket[array=hb60rs vm_size=
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hb60rs, vm_size=Standard_HB60rs, pg=Standard_HB60rs_pg1),reason=Resource[name=slot_type value='hb60rs'] != 'execute' for Bucket[array=hb60rs vm_size=
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hc44rs, vm_size=Standard_HC44rs, pg=Standard_HC44rs_pg0),reason=Resource[name=slot_type value='hc44rs'] != 'execute' for Bucket[array=hc44rs vm_size=
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=hc44rs, vm_size=Standard_HC44rs, pg=Standard_HC44rs_pg1),reason=Resource[name=slot_type value='hc44rs'] != 'execute' for Bucket[array=hc44rs vm_size=
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=viz, vm_size=Standard_D8s_v3, pg=Standard_D8s_v3_pg0),reason=Resource[name=slot_type value='viz'] != 'execute' for Bucket[array=viz vm_size=Standard_
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=viz, vm_size=Standard_D8s_v3, pg=Standard_D8s_v3_pg1),reason=Resource[name=slot_type value='viz'] != 'execute' for Bucket[array=viz vm_size=Standard_
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=viz3d, vm_size=Standard_NV6, pg=Standard_NV6_pg0),reason=Resource[name=slot_type value='viz3d'] != 'execute' for Bucket[array=viz3d vm_size=Standard_
SatisfiedResult(status=InvalidOption, node=NodeBucket(nodearray=viz3d, vm_size=Standard_NV6, pg=Standard_NV6_pg1),reason=Resource[name=slot_type value='viz3d'] != 'execute' for Bucket[array=viz3d vm_size=Standard_

[root@scheduler ~]# azpbs buckets
NODEARRAY PLACEMENT_GROUP VM_SIZE VCPU_COUNT PCPU_COUNT MEMORY AVAILABLE_COUNT NCPUS NGPUS DISK HOST SLOT_TYPE GROUP_ID MEM CCNODEID UNGROUPED
execute Standard_F2s_v2 2 1 4.00g 512 1 0 20.00g execute none 4.00g true
execute Standard_F2s_v2_pg0 Standard_F2s_v2 2 1 4.00g 100 1 0 20.00g execute Standard_F2s_v2_pg0 4.00g false
execute Standard_F2s_v2_pg1 Standard_F2s_v2 2 1 4.00g 100 1 0 20.00g execute Standard_F2s_v2_pg1 4.00g false
hb120rs_v2 Standard_HB120rs_v2 120 120 456.00g 72 120 0 20.00g hb120rs_v2 none 456.00g true
hb120rs_v2 Standard_HB120rs_v2_pg0 Standard_HB120rs_v2 120 120 456.00g 72 120 0 20.00g hb120rs_v2 Standard_HB120rs_v2_pg0 456.00g false
hb120rs_v2 Standard_HB120rs_v2_pg1 Standard_HB120rs_v2 120 120 456.00g 72 120 0 20.00g hb120rs_v2 Standard_HB120rs_v2_pg1 456.00g false
hb120rs_v3 Standard_HB120rs_v3 120 120 448.00g 10 120 0 20.00g hb120rs_v3 none 448.00g true
hb120rs_v3 Standard_HB120rs_v3_pg0 Standard_HB120rs_v3 120 120 448.00g 10 120 0 20.00g hb120rs_v3 Standard_HB120rs_v3_pg0 448.00g false
hb120rs_v3 Standard_HB120rs_v3_pg1 Standard_HB120rs_v3 120 120 448.00g 10 120 0 20.00g hb120rs_v3 Standard_HB120rs_v3_pg1 448.00g false
hb60rs Standard_HB60rs 60 60 228.00g 40 60 0 20.00g hb60rs none 228.00g true
hb60rs Standard_HB60rs_pg0 Standard_HB60rs 60 60 228.00g 40 60 0 20.00g hb60rs Standard_HB60rs_pg0 228.00g false
hb60rs Standard_HB60rs_pg1 Standard_HB60rs 60 60 228.00g 40 60 0 20.00g hb60rs Standard_HB60rs_pg1 228.00g false
hc44rs Standard_HC44rs 44 44 352.00g 40 44 0 20.00g hc44rs none 352.00g true
hc44rs Standard_HC44rs_pg0 Standard_HC44rs 44 44 352.00g 40 44 0 20.00g hc44rs Standard_HC44rs_pg0 352.00g false
hc44rs Standard_HC44rs_pg1 Standard_HC44rs 44 44 352.00g 40 44 0 20.00g hc44rs Standard_HC44rs_pg1 352.00g false
viz Standard_D8s_v3 8 4 32.00g 50 4 0 20.00g viz none 32.00g true
viz Standard_D8s_v3_pg0 Standard_D8s_v3 8 4 32.00g 50 4 0 20.00g viz Standard_D8s_v3_pg0 32.00g false
viz Standard_D8s_v3_pg1 Standard_D8s_v3 8 4 32.00g 50 4 0 20.00g viz Standard_D8s_v3_pg1 32.00g false
viz3d Standard_NV6 6 6 56.00g 10 6 1 20.00g viz3d none 56.00g true
viz3d Standard_NV6_pg0 Standard_NV6 6 6 56.00g 10 6 1 20.00g viz3d Standard_NV6_pg0 56.00g false
viz3d Standard_NV6_pg1 Standard_NV6 6 6 56.00g 10 6 1 20.00g viz3d Standard_NV6_pg1 56.00g false