From 91c5cafb10fe8a26057c0a86c2fbe7bd98d8699c Mon Sep 17 00:00:00 2001 From: Adrien Faure Date: Fri, 24 Nov 2023 16:35:38 +0100 Subject: [PATCH] [oar/kao/quotas] implem job over multiple quotas periods --- .git-blame-ignore-revs | 0 oar/kao/quotas.py | 32 ++++++++-- tests/kao/test_temporal_quotas.py | 102 ++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..e69de29b diff --git a/oar/kao/quotas.py b/oar/kao/quotas.py index d93abcca..5f8eb5df 100644 --- a/oar/kao/quotas.py +++ b/oar/kao/quotas.py @@ -746,15 +746,28 @@ def check(self, job) -> tuple[bool, str, str, int]: return (True, "quotas ok", "", 0) @staticmethod - def check_slots_quotas(slots, sid_left, sid_right, job, job_nb_resources, duration): + def check_slots_quotas( + slots, + sid_left: int, + sid_right: int, + job, + job_nb_resources: int, + duration: int, + ): # loop over slot_set - slots_quotas = Quotas(slots[sid_left].quotas.rules) + slots_quotas: dict[int, Quotas] = {} sid = sid_left while True: slot = slots[sid] + + if slot.quotas_rules_id not in slots_quotas: + slots_quotas[slot.quotas_rules_id] = Quotas(slots[sid].quotas.rules) + + quotas = slots_quotas[slot.quotas_rules_id] + # slot.quotas.show_counters('check_slots_quotas, b e: ' + str(slot.b) + ' ' + str(slot.e)) - slots_quotas.combine(slot.quotas) + quotas.combine(slot.quotas) if sid == sid_right: break @@ -763,11 +776,16 @@ def check_slots_quotas(slots, sid_left, sid_right, job, job_nb_resources, durati if slot.next and ( slot.quotas_rules_id != slots[slot.next].quotas_rules_id ): - return (False, "different quotas rules over job's time", "", 0) + logger.debug("job on two different quotas periods") + + for id, quotas in slots_quotas.items(): + quotas.update(job, job_nb_resources, duration) + res = quotas.check(job) + if not res[0]: + return res - # print('slots b e :' + str(slots[sid_left].b) + " " + str(slots[sid_right].e)) - slots_quotas.update(job, job_nb_resources, duration) - return slots_quotas.check(job) + # return last one that should be a success anyway + return res def set_rules(self, rules_id): """Use for temporal calendar, when rules must be change from default""" diff --git a/tests/kao/test_temporal_quotas.py b/tests/kao/test_temporal_quotas.py index 502f07c8..3566db7d 100644 --- a/tests/kao/test_temporal_quotas.py +++ b/tests/kao/test_temporal_quotas.py @@ -604,3 +604,105 @@ def test_temporal_quotas_window_time_limit_reached(oar_conf): assert j1.res_set == ProcSet(*[(1, 24)]) assert j2.res_set == ProcSet() + + +# Testing jobs over multiple periods +def test_temporal_quots_multi_periods_nb_resources(oar_conf): + config = oar_conf + config["QUOTAS_PERIOD"] = 3 * 7 * 86400 # 3 weeks + Quotas.enabled = True + + rules = { + "periodical": [ + ["* * * *", "quotas_1", "test1"], + ], + "oneshot": [], + "quotas_1": {"*,*,*,/": [24, -1, -1]}, + "quotas_2": {"*,*,*,/": [8, -1, -1]}, + } + + res = ProcSet(*[(1, 32)]) + ResourceSet.default_itvs = ProcSet(*res) + + now = datetime.utcnow() + + now_str = (now + timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M") + then_str = (now + timedelta(minutes=10)).strftime("%Y-%m-%d %H:%M") + + rules["oneshot"].append([now_str, then_str, "quotas_2", "not important"]) + + Quotas.calendar = Calendar(rules, config) + + t0 = now + t1 = t0 + timedelta(seconds=14 * 86400) # - 1 + + ss = SlotSet(Slot(1, 0, 0, ProcSet(*res), int(t0.timestamp()), int(t1.timestamp()))) + all_ss = {"default": ss} + hy = {"node": [ProcSet(*x) for x in [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]]} + + # Job on two quotas period that should pass now + # because it respects the two quotas periods + j = JobPseudo(id=2, queue="default", user="toto", project="") + j.simple_req(("node", 1), 300, res) + + # Job that doesn't pass the oneshot period (and there for should be delayed) + j1 = JobPseudo(id=3, queue="default", user="toto2", project="") + j1.simple_req(("node", 2), 300, res) + + schedule_id_jobs_ct(all_ss, {j.id: j, j1.id: j1}, hy, [j.id, j1.id], 20) + + print(f"job id: {j.id} starts at {datetime.fromtimestamp(j.start_time)}") + + assert int(j.start_time) - int(t0.timestamp()) == 0 + assert int(j1.start_time) - int(t0.timestamp()) > 0 + + +# Testing jobs over multiple periods +def test_temporal_quots_multi_periods_nb_jobs(oar_conf): + config = oar_conf + config["QUOTAS_PERIOD"] = 3 * 7 * 86400 # 3 weeks + Quotas.enabled = True + + rules = { + "periodical": [ + ["* * * *", "quotas_1", "test1"], + ], + "oneshot": [], + "quotas_1": {"*,*,*,/": [-1, 2, -1]}, + "quotas_2": {"*,*,*,/": [-1, 1, -1]}, + } + + res = ProcSet(*[(1, 32)]) + ResourceSet.default_itvs = ProcSet(*res) + + now = datetime.utcnow() + + now_str = (now + timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M") + then_str = (now + timedelta(minutes=10)).strftime("%Y-%m-%d %H:%M") + + rules["oneshot"].append([now_str, then_str, "quotas_2", "not important"]) + + Quotas.calendar = Calendar(rules, config) + + t0 = now + t1 = t0 + timedelta(seconds=14 * 86400) # - 1 + + ss = SlotSet(Slot(1, 0, 0, ProcSet(*res), int(t0.timestamp()), int(t1.timestamp()))) + all_ss = {"default": ss} + hy = {"node": [ProcSet(*x) for x in [[(1, 8)], [(9, 16)], [(17, 24)], [(25, 32)]]]} + + # Job on two quotas period that should pass now + # because it respects the two quotas periods + j = JobPseudo(id=2, queue="default", user="toto", project="") + j.simple_req(("node", 1), 300, res) + + # Job that doesn't pass the oneshot period (and there for should be delayed) + j1 = JobPseudo(id=3, queue="default", user="toto", project="") + j1.simple_req(("node", 2), 300, res) + + schedule_id_jobs_ct(all_ss, {j.id: j, j1.id: j1}, hy, [j.id, j1.id], 20) + + print(f"job id: {j.id} starts at {datetime.fromtimestamp(j.start_time)}") + + assert int(j.start_time) - int(t0.timestamp()) == 0 + assert int(j1.start_time) - int(t0.timestamp()) > 0