You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
When I run the vanilla ManiGaussian using: "bash scripts/train_and_eval_w_geo_sem_dyna.sh ManiGaussian_BC 0,1 12345 ${exp_name}"
The following error occurs:
L_rot: 16.769 x 1.000 | L_grip: 0.535 x 1.000 | L_col: 0.328 x 1.000 | L_rgb: 0.041 x 0.010 | L_embed: 0.780 x 0.0001 | L_dyna: 0.094 x 0.0000 | L_reg: 0.000 x 0.0000 | psnr: 13.879
0%|▎ | 285/100010 [16:30<96:17:34, 3.48s/it]
Error executing job with overrides: ['method=ManiGaussian_BC', 'rlbench.task_name=vanilla_ManiGaussian_0', 'rlbench.demo_path=/home/aida/wangyc/ManiGaussian/data/train_data', 'replay.path=/home/aida/wangyc/ManiGaussian/replay/vanilla_ManiGaussian_0', 'framework.start_seed=0', 'framework.use_wandb=True', 'method.use_wandb=True', 'framework.wandb_group=vanilla_ManiGaussian_0', 'framework.wandb_name=vanilla_ManiGaussian_0', 'ddp.num_devices=2', 'replay.batch_size=1', 'ddp.master_port=12345', 'rlbench.tasks=[close_jar,open_drawer,sweep_to_dustpan_of_size,meat_off_grill,turn_tap,slide_block_to_color_target,put_item_in_drawer,reach_and_drag,push_buttons,stack_blocks]', 'rlbench.demos=20', 'method.neural_renderer.render_freq=2000', 'method.neural_renderer.lambda_embed=0.01', 'method.neural_renderer.lambda_dyna=0.1', 'method.neural_renderer.lambda_reg=0.0', 'method.neural_renderer.foundation_model_name=diffusion', 'method.neural_renderer.use_dynamic_field=True']
Traceback (most recent call last):
File "/home/aida/wangyc/ManiGaussian/train.py", line 96, in main
run_seed_fn.run_seed(
File "/home/aida/wangyc/ManiGaussian/run_seed_fn.py", line 147, in run_seed
train_runner.start()
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/runners/offline_train_runner.py", line 200, in start
batch = self.preprocess_data(data_iter)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/runners/offline_train_runner.py", line 121, in preprocess_data
sampled_batch = next(data_iter) # may raise StopIteration
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/lightning/fabric/wrappers.py", line 296, in iter
for item in self._dataloader:
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 634, in next
data = self._next_data()
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 678, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 41, in fetch
data = next(self.dataset_iter)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/wrappers/pytorch_replay_buffer.py", line 17, in _generator
yield self._replay_buffer.sample_transition_batch(pack_in_dict=True)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/uniform_replay_buffer.py", line 722, in sample_transition_batch
store = self._get_from_disk(
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/uniform_replay_buffer.py", line 391, in _get_from_disk
d = pickle.load(f)
TypeError: setstate() argument 1 must be sequence of length 4, not 2
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.
0%|▎ | 285/100010 [16:50<93:58:23, 3.39s/it]wandb:
Any suggestions for this? Thanks!
The text was updated successfully, but these errors were encountered:
When I run the vanilla ManiGaussian using: "bash scripts/train_and_eval_w_geo_sem_dyna.sh ManiGaussian_BC 0,1 12345 ${exp_name}"
The following error occurs:
L_rot: 16.769 x 1.000 | L_grip: 0.535 x 1.000 | L_col: 0.328 x 1.000 | L_rgb: 0.041 x 0.010 | L_embed: 0.780 x 0.0001 | L_dyna: 0.094 x 0.0000 | L_reg: 0.000 x 0.0000 | psnr: 13.879
0%|▎ | 285/100010 [16:30<96:17:34, 3.48s/it]
Error executing job with overrides: ['method=ManiGaussian_BC', 'rlbench.task_name=vanilla_ManiGaussian_0', 'rlbench.demo_path=/home/aida/wangyc/ManiGaussian/data/train_data', 'replay.path=/home/aida/wangyc/ManiGaussian/replay/vanilla_ManiGaussian_0', 'framework.start_seed=0', 'framework.use_wandb=True', 'method.use_wandb=True', 'framework.wandb_group=vanilla_ManiGaussian_0', 'framework.wandb_name=vanilla_ManiGaussian_0', 'ddp.num_devices=2', 'replay.batch_size=1', 'ddp.master_port=12345', 'rlbench.tasks=[close_jar,open_drawer,sweep_to_dustpan_of_size,meat_off_grill,turn_tap,slide_block_to_color_target,put_item_in_drawer,reach_and_drag,push_buttons,stack_blocks]', 'rlbench.demos=20', 'method.neural_renderer.render_freq=2000', 'method.neural_renderer.lambda_embed=0.01', 'method.neural_renderer.lambda_dyna=0.1', 'method.neural_renderer.lambda_reg=0.0', 'method.neural_renderer.foundation_model_name=diffusion', 'method.neural_renderer.use_dynamic_field=True']
Traceback (most recent call last):
File "/home/aida/wangyc/ManiGaussian/train.py", line 96, in main
run_seed_fn.run_seed(
File "/home/aida/wangyc/ManiGaussian/run_seed_fn.py", line 147, in run_seed
train_runner.start()
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/runners/offline_train_runner.py", line 200, in start
batch = self.preprocess_data(data_iter)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/runners/offline_train_runner.py", line 121, in preprocess_data
sampled_batch = next(data_iter) # may raise StopIteration
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/lightning/fabric/wrappers.py", line 296, in iter
for item in self._dataloader:
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 634, in next
data = self._next_data()
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 678, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/media/aida/data2/anaconda/envs/manigaussian/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 41, in fetch
data = next(self.dataset_iter)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/wrappers/pytorch_replay_buffer.py", line 17, in _generator
yield self._replay_buffer.sample_transition_batch(pack_in_dict=True)
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/uniform_replay_buffer.py", line 722, in sample_transition_batch
store = self._get_from_disk(
File "/home/aida/wangyc/ManiGaussian/third_party/YARR/yarr/replay_buffer/uniform_replay_buffer.py", line 391, in _get_from_disk
d = pickle.load(f)
TypeError: setstate() argument 1 must be sequence of length 4, not 2
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.
0%|▎ | 285/100010 [16:50<93:58:23, 3.39s/it]wandb:
Any suggestions for this? Thanks!
The text was updated successfully, but these errors were encountered: