Prototype ray task #29
Conversation
|
| return TaskUtil.evaluate_variable_concat_prefix_suffix(context, selector, prefix=prefix, suffix=suffix) | ||
|
|
||
| @staticmethod | ||
| def _load_csv_file( |
There was a problem hiding this comment.
for consistency remove _ ... load_csv_file instead for _load_csv_file
| @@ -241,6 +241,7 @@ def test_calculate_default_page_size_with_statement_page_size(self, generate_tas | |||
| assert size == 100 | |||
| assert mock_statement.page_size == 100 | |||
|
|
|||
| @pytest.mark.skip("Need rework with ray") | |||
There was a problem hiding this comment.
before PR is done needs to be reworked
datamimic_ce/tasks/generate_task.py
Outdated
|
|
||
|
|
||
| def _geniter_single_process_generate(args: tuple) -> dict[str, list]: | ||
| def _geniter_single_process_generate(context: SetupContext | GenIterContext, stmt: GenerateStatement, page_start: int, |
There was a problem hiding this comment.
should not exists anymore ... and we should re-evaluate the naming of the function to make it more clear what they are actually doing
datamimic_ce/tasks/generate_task.py
Outdated
| page_index == len(index_chunk) - 1, | ||
| ) | ||
| # Determine number of Ray workers | ||
| num_workers = int(self.statement.num_process or context.root.num_process or 1) |
There was a problem hiding this comment.
yeah ... this is how it should be ... default is num_process = 1 ... we need to make sure the ray framework is doing what we expect ... last time even when i specified 1 process ... many ray processes were created without any load ... haven't investigated more if this behavior is normal and how to address
No description provided.