From e6c55a648cddb7d33187468c22015e719af5012e Mon Sep 17 00:00:00 2001 From: olivame Date: Sun, 8 Feb 2026 09:42:14 +0000 Subject: [PATCH] =?UTF-8?q?=E6=89=80=E6=9C=89case=E7=9A=84baseline?= =?UTF-8?q?=EF=BC=8Camd=E7=89=88=E6=9C=AC=E7=9A=84ground=20truth=E9=83=BD?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 +- case4_run.log | 135 ++++++++++++++++ ckpts/configuration.json | 1 + env.sh | 21 +++ run.log | 150 ++++++++++++++++++ src/unifolm_wma/modules/attention.py | 2 +- unitree_g1_pack_camera/case1/output.log | 144 +++++++++++++++++ unitree_g1_pack_camera/case1/psnr_result.json | 5 + unitree_g1_pack_camera/case2/output.log | 144 +++++++++++++++++ unitree_g1_pack_camera/case2/psnr_result.json | 5 + unitree_g1_pack_camera/case3/output.log | 144 +++++++++++++++++ unitree_g1_pack_camera/case3/psnr_result.json | 5 + unitree_g1_pack_camera/case4/output.log | 144 +++++++++++++++++ unitree_g1_pack_camera/case4/psnr_result.json | 5 + .../case1/output.log | 135 ++++++++++++++++ .../case1/psnr_result.json | 5 + .../case1/psnr_result1.json | 5 + .../case2/output.log | 137 ++++++++++++++++ .../case2/psnr_result.json | 5 + .../case3/output.log | 137 ++++++++++++++++ .../case3/psnr_result.json | 5 + .../case4/output.log | 137 ++++++++++++++++ .../case4/psnr_result.json | 5 + unitree_z1_dual_arm_stackbox/case1/output.log | 134 ++++++++++++++++ .../case1/psnr_result.json | 5 + .../case1/run_world_model_interaction.sh | 2 +- unitree_z1_dual_arm_stackbox/case2/output.log | 134 ++++++++++++++++ .../case2/psnr_result.json | 5 + .../case2/run_world_model_interaction.sh | 2 +- unitree_z1_dual_arm_stackbox/case3/output.log | 134 ++++++++++++++++ .../case3/psnr_result.json | 5 + unitree_z1_dual_arm_stackbox/case4/output.log | 134 ++++++++++++++++ .../case4/psnr_result.json | 5 + .../case1/output.log | 146 +++++++++++++++++ .../case1/psnr_result.json | 5 + .../case1/run_world_model_interaction.sh | 2 +- .../case2/output.log | 146 +++++++++++++++++ .../case2/psnr_result.json | 5 + .../case3/output.log | 146 +++++++++++++++++ .../case3/psnr_result.json | 5 + .../case4/output.log | 146 +++++++++++++++++ .../case4/psnr_result.json | 5 + .../case4/run_world_model_interaction.sh | 2 +- unitree_z1_stackbox/case1/output.log | 149 +++++++++++++++++ unitree_z1_stackbox/case1/psnr_result.json | 5 + .../case1/run_world_model_interaction.sh | 2 +- unitree_z1_stackbox/case2/output.log | 149 +++++++++++++++++ unitree_z1_stackbox/case2/psnr_result.json | 5 + unitree_z1_stackbox/case3/output.log | 149 +++++++++++++++++ unitree_z1_stackbox/case3/psnr_result.json | 5 + unitree_z1_stackbox/case4/output.log | 149 +++++++++++++++++ unitree_z1_stackbox/case4/psnr_result.json | 5 + .../case4/run_world_model_interaction.sh | 2 +- 53 files changed, 3260 insertions(+), 10 deletions(-) create mode 100644 case4_run.log create mode 100644 ckpts/configuration.json create mode 100644 env.sh create mode 100644 run.log create mode 100644 unitree_g1_pack_camera/case1/output.log create mode 100644 unitree_g1_pack_camera/case1/psnr_result.json create mode 100644 unitree_g1_pack_camera/case2/output.log create mode 100644 unitree_g1_pack_camera/case2/psnr_result.json create mode 100644 unitree_g1_pack_camera/case3/output.log create mode 100644 unitree_g1_pack_camera/case3/psnr_result.json create mode 100644 unitree_g1_pack_camera/case4/output.log create mode 100644 unitree_g1_pack_camera/case4/psnr_result.json create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case1/output.log create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result.json create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result1.json create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case2/output.log create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case2/psnr_result.json create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case3/output.log create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case3/psnr_result.json create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case4/output.log create mode 100644 unitree_z1_dual_arm_cleanup_pencils/case4/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox/case1/output.log create mode 100644 unitree_z1_dual_arm_stackbox/case1/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox/case2/output.log create mode 100644 unitree_z1_dual_arm_stackbox/case2/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox/case3/output.log create mode 100644 unitree_z1_dual_arm_stackbox/case3/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox/case4/output.log create mode 100644 unitree_z1_dual_arm_stackbox/case4/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox_v2/case1/output.log create mode 100644 unitree_z1_dual_arm_stackbox_v2/case1/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox_v2/case2/output.log create mode 100644 unitree_z1_dual_arm_stackbox_v2/case2/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox_v2/case3/output.log create mode 100644 unitree_z1_dual_arm_stackbox_v2/case3/psnr_result.json create mode 100644 unitree_z1_dual_arm_stackbox_v2/case4/output.log create mode 100644 unitree_z1_dual_arm_stackbox_v2/case4/psnr_result.json create mode 100644 unitree_z1_stackbox/case1/output.log create mode 100644 unitree_z1_stackbox/case1/psnr_result.json create mode 100644 unitree_z1_stackbox/case2/output.log create mode 100644 unitree_z1_stackbox/case2/psnr_result.json create mode 100644 unitree_z1_stackbox/case3/output.log create mode 100644 unitree_z1_stackbox/case3/psnr_result.json create mode 100644 unitree_z1_stackbox/case4/output.log create mode 100644 unitree_z1_stackbox/case4/psnr_result.json diff --git a/.gitignore b/.gitignore index 2ae054d..87bb685 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,7 @@ coverage.xml *.pot # Django stuff: -*.log + local_settings.py db.sqlite3 @@ -121,7 +121,6 @@ localTest/ fig/ figure/ *.mp4 -*.json Data/ControlVAE.yml Data/Misc Data/Pretrained @@ -129,4 +128,5 @@ Data/utils.py Experiment/checkpoint Experiment/log -*.ckpt \ No newline at end of file +*.ckpt +*.0 \ No newline at end of file diff --git a/case4_run.log b/case4_run.log new file mode 100644 index 0000000..1b7a0fd --- /dev/null +++ b/case4_run.log @@ -0,0 +1,135 @@ +nohup: ignoring input +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:38:45.572744: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:38:45.576864: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:38:45.624825: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:38:45.624883: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:38:45.627150: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:38:45.638316: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:38:45.638803: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:38:46.426363: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/7 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 14%|█▍ | 1/7 [01:38<09:52, 98.73s/it] 29%|██▊ | 2/7 [03:17<08:14, 98.85s/it] 43%|████▎ | 3/7 [04:56<06:35, 98.80s/it] 57%|█████▋ | 4/7 [06:35<04:56, 98.94s/it] 71%|███████▏ | 5/7 [08:14<03:17, 98.93s/it] 86%|████████▌ | 6/7 [09:53<01:38, 98.89s/it] 100%|██████████| 7/7 [11:31<00:00, 98.81s/it] 100%|██████████| 7/7 [11:31<00:00, 98.85s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 12m36.996s +user 11m43.720s +sys 0m46.859s diff --git a/ckpts/configuration.json b/ckpts/configuration.json new file mode 100644 index 0000000..7ecd050 --- /dev/null +++ b/ckpts/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "robotics", "allow_remote": true} \ No newline at end of file diff --git a/env.sh b/env.sh new file mode 100644 index 0000000..8210693 --- /dev/null +++ b/env.sh @@ -0,0 +1,21 @@ +# Note: This script should be sourced, not executed +# Usage: source env.sh +# +# If you need render group permissions, run this first: +# newgrp render +# Then source this script: +# source env.sh + +# Initialize conda +source /mnt/ASC1637/miniconda3/etc/profile.d/conda.sh + +# Activate conda environment +conda activate unifolm-wma-o + +# Set HuggingFace cache directories +export HF_HOME=/mnt/ASC1637/hf_home +export HUGGINGFACE_HUB_CACHE=/mnt/ASC1637/hf_home/hub + +echo "Environment configured successfully" +echo "Conda environment: unifolm-wma-o" +echo "HF_HOME: $HF_HOME" \ No newline at end of file diff --git a/run.log b/run.log new file mode 100644 index 0000000..ac9a398 --- /dev/null +++ b/run.log @@ -0,0 +1,150 @@ +nohup: ignoring input +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:15:49.934949: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:15:49.937974: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:15:49.969069: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:15:49.969100: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:15:49.970909: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:15:49.979005: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:15:49.979255: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:15:50.597743: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/12 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 8%|▊ | 1/12 [01:37<17:51, 97.37s/it] 17%|█▋ | 2/12 [03:14<16:13, 97.31s/it] 25%|██▌ | 3/12 [04:51<14:35, 97.26s/it] 33%|███▎ | 4/12 [06:29<12:58, 97.25s/it] 42%|████▏ | 5/12 [08:06<11:20, 97.24s/it] 50%|█████ | 6/12 [09:43<09:43, 97.24s/it] 58%|█████▊ | 7/12 [11:20<08:06, 97.27s/it] 67%|██████▋ | 8/12 [12:58<06:29, 97.36s/it] 75%|███████▌ | 9/12 [14:36<04:52, 97.49s/it] 83%|████████▎ | 10/12 [16:13<03:15, 97.52s/it] 92%|█████████▏| 11/12 [17:51<01:37, 97.47s/it] 100%|██████████| 12/12 [19:28<00:00, 97.35s/it] 100%|██████████| 12/12 [19:28<00:00, 97.35s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 11: generating actions ... +>>> Step 11: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 20m36.252s +user 18m13.951s +sys 0m58.438s diff --git a/src/unifolm_wma/modules/attention.py b/src/unifolm_wma/modules/attention.py index 7b21317..eab69d4 100644 --- a/src/unifolm_wma/modules/attention.py +++ b/src/unifolm_wma/modules/attention.py @@ -125,7 +125,7 @@ class CrossAttention(nn.Module): context = default(context, x) if self.image_cross_attention and not spatial_self_attn: - assert 1 > 2, ">>> ERROR: should setup xformers and use efficient_forward ..." + # assert 1 > 2, ">>> ERROR: should setup xformers and use efficient_forward ..." context_agent_state = context[:, :self.agent_state_context_len, :] context_agent_action = context[:, self.agent_state_context_len:self. diff --git a/unitree_g1_pack_camera/case1/output.log b/unitree_g1_pack_camera/case1/output.log new file mode 100644 index 0000000..cadf0c6 --- /dev/null +++ b/unitree_g1_pack_camera/case1/output.log @@ -0,0 +1,144 @@ +2026-02-08 05:20:49.828675: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 05:20:49.831563: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:20:49.861366: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 05:20:49.861402: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 05:20:49.862974: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 05:20:49.870402: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:20:49.870647: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 05:20:50.486843: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:38<16:25, 98.56s/it] 18%|█▊ | 2/11 [03:16<14:44, 98.31s/it] 27%|██▋ | 3/11 [04:55<13:06, 98.33s/it] 36%|███▋ | 4/11 [06:36<11:37, 99.66s/it] 45%|████▌ | 5/11 [08:31<10:29, 104.96s/it] 55%|█████▍ | 6/11 [10:10<08:35, 103.07s/it] 64%|██████▎ | 7/11 [11:48<06:46, 101.50s/it] 73%|███████▎ | 8/11 [13:27<05:01, 100.52s/it] 82%|████████▏ | 9/11 [15:05<03:19, 99.79s/it] 91%|█████████ | 10/11 [16:43<01:39, 99.30s/it] 100%|██████████| 11/11 [18:21<00:00, 98.97s/it] 100%|██████████| 11/11 [18:21<00:00, 100.16s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 20m16.253s +user 17m52.621s +sys 1m6.627s diff --git a/unitree_g1_pack_camera/case1/psnr_result.json b/unitree_g1_pack_camera/case1/psnr_result.json new file mode 100644 index 0000000..4c2d316 --- /dev/null +++ b/unitree_g1_pack_camera/case1/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_g1_pack_camera/case1/unitree_g1_pack_camera_case1.mp4", + "pred_video": "unitree_g1_pack_camera/case1/output/inference/unitree_g1_pack_camera_case1_amd.mp4", + "psnr": 16.415668383379177 +} \ No newline at end of file diff --git a/unitree_g1_pack_camera/case2/output.log b/unitree_g1_pack_camera/case2/output.log new file mode 100644 index 0000000..6ad70b6 --- /dev/null +++ b/unitree_g1_pack_camera/case2/output.log @@ -0,0 +1,144 @@ +2026-02-08 05:06:45.806187: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 05:06:45.809295: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:06:45.840950: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 05:06:45.840981: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 05:06:45.842814: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 05:06:45.851049: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:06:45.851316: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 05:06:47.225477: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:37<16:14, 97.41s/it] 18%|█▊ | 2/11 [03:14<14:35, 97.22s/it] 27%|██▋ | 3/11 [04:51<12:58, 97.33s/it] 36%|███▋ | 4/11 [06:29<11:22, 97.47s/it] 45%|████▌ | 5/11 [08:07<09:45, 97.57s/it] 55%|█████▍ | 6/11 [09:45<08:07, 97.59s/it] 64%|██████▎ | 7/11 [11:22<06:30, 97.57s/it] 73%|███████▎ | 8/11 [13:00<04:52, 97.54s/it] 82%|████████▏ | 9/11 [14:37<03:14, 97.50s/it] 91%|█████████ | 10/11 [16:14<01:37, 97.32s/it] 100%|██████████| 11/11 [17:51<00:00, 97.19s/it] 100%|██████████| 11/11 [17:51<00:00, 97.39s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m5.537s +user 16m51.114s +sys 0m52.978s diff --git a/unitree_g1_pack_camera/case2/psnr_result.json b/unitree_g1_pack_camera/case2/psnr_result.json new file mode 100644 index 0000000..4ad761a --- /dev/null +++ b/unitree_g1_pack_camera/case2/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_g1_pack_camera/case2/unitree_g1_pack_camera_case2.mp4", + "pred_video": "unitree_g1_pack_camera/case2/output/inference/unitree_g1_pack_camera_case2_amd.mp4", + "psnr": 19.515250190529375 +} \ No newline at end of file diff --git a/unitree_g1_pack_camera/case3/output.log b/unitree_g1_pack_camera/case3/output.log new file mode 100644 index 0000000..73d1bf0 --- /dev/null +++ b/unitree_g1_pack_camera/case3/output.log @@ -0,0 +1,144 @@ +2026-02-08 05:08:32.803904: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 05:08:32.807010: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:08:32.837936: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 05:08:32.837978: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 05:08:32.839785: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 05:08:32.847835: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:08:32.848223: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 05:08:34.120114: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:39<16:34, 99.46s/it] 18%|█▊ | 2/11 [03:18<14:55, 99.48s/it] 27%|██▋ | 3/11 [04:58<13:16, 99.60s/it] 36%|███▋ | 4/11 [06:38<11:37, 99.69s/it] 45%|████▌ | 5/11 [08:18<09:58, 99.68s/it] 55%|█████▍ | 6/11 [09:57<08:18, 99.66s/it] 64%|██████▎ | 7/11 [11:37<06:38, 99.62s/it] 73%|███████▎ | 8/11 [13:16<04:58, 99.55s/it] 82%|████████▏ | 9/11 [14:56<03:19, 99.50s/it] 91%|█████████ | 10/11 [16:35<01:39, 99.43s/it] 100%|██████████| 11/11 [18:14<00:00, 99.36s/it] 100%|██████████| 11/11 [18:14<00:00, 99.51s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m26.448s +user 17m10.684s +sys 0m56.881s diff --git a/unitree_g1_pack_camera/case3/psnr_result.json b/unitree_g1_pack_camera/case3/psnr_result.json new file mode 100644 index 0000000..e7beb21 --- /dev/null +++ b/unitree_g1_pack_camera/case3/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_g1_pack_camera/case3/unitree_g1_pack_camera_case3.mp4", + "pred_video": "unitree_g1_pack_camera/case3/output/inference/unitree_g1_pack_camera_case3_amd.mp4", + "psnr": 19.429578160315536 +} \ No newline at end of file diff --git a/unitree_g1_pack_camera/case4/output.log b/unitree_g1_pack_camera/case4/output.log new file mode 100644 index 0000000..f34a57b --- /dev/null +++ b/unitree_g1_pack_camera/case4/output.log @@ -0,0 +1,144 @@ +2026-02-08 05:29:19.728303: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 05:29:19.731620: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:29:19.761276: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 05:29:19.761301: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 05:29:19.762880: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 05:29:19.770578: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 05:29:19.771072: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 05:29:21.043661: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:37<16:18, 97.81s/it] 18%|█▊ | 2/11 [03:15<14:38, 97.56s/it] 27%|██▋ | 3/11 [04:52<12:59, 97.48s/it] 36%|███▋ | 4/11 [06:29<11:21, 97.38s/it] 45%|████▌ | 5/11 [08:06<09:43, 97.28s/it] 55%|█████▍ | 6/11 [09:44<08:06, 97.35s/it] 64%|██████▎ | 7/11 [11:21<06:29, 97.36s/it] 73%|███████▎ | 8/11 [12:59<04:52, 97.38s/it] 82%|████████▏ | 9/11 [14:36<03:14, 97.39s/it] 91%|█████████ | 10/11 [16:14<01:37, 97.42s/it] 100%|██████████| 11/11 [17:51<00:00, 97.42s/it] 100%|██████████| 11/11 [17:51<00:00, 97.41s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m38.883s +user 16m50.169s +sys 1m4.995s diff --git a/unitree_g1_pack_camera/case4/psnr_result.json b/unitree_g1_pack_camera/case4/psnr_result.json new file mode 100644 index 0000000..e895201 --- /dev/null +++ b/unitree_g1_pack_camera/case4/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_g1_pack_camera/case4/unitree_g1_pack_camera_case4.mp4", + "pred_video": "unitree_g1_pack_camera/case4/output/inference/unitree_g1_pack_camera_case4_amd.mp4", + "psnr": 17.80386833747375 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_cleanup_pencils/case1/output.log b/unitree_z1_dual_arm_cleanup_pencils/case1/output.log new file mode 100644 index 0000000..77ba312 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case1/output.log @@ -0,0 +1,135 @@ +2026-02-08 09:20:29.036523: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 09:20:29.301726: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 09:20:29.656318: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 09:20:29.656367: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 09:20:29.662840: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 09:20:29.718736: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 09:20:29.718991: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 09:20:31.661239: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/8 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 12%|█▎ | 1/8 [01:40<11:43, 100.53s/it] 25%|██▌ | 2/8 [03:20<10:01, 100.30s/it] 38%|███▊ | 3/8 [05:00<08:21, 100.22s/it] 50%|█████ | 4/8 [06:41<06:41, 100.29s/it] 62%|██████▎ | 5/8 [08:21<05:01, 100.38s/it] 75%|███████▌ | 6/8 [10:02<03:20, 100.41s/it] 88%|████████▊ | 7/8 [11:42<01:40, 100.37s/it] 100%|██████████| 8/8 [13:23<00:00, 100.44s/it] 100%|██████████| 8/8 [13:23<00:00, 100.39s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 14m41.368s +user 13m8.275s +sys 0m48.945s diff --git a/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result.json b/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result.json new file mode 100644 index 0000000..d71c636 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_cleanup_pencils/case1/unitree_z1_dual_arm_cleanup_pencils_case1.mp4", + "pred_video": "unitree_z1_dual_arm_cleanup_pencils/case1/output/inference/unitree_z1_dual_arm_cleanup_pencils_case1_amd.mp4", + "psnr": 19.586376345676264 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result1.json b/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result1.json new file mode 100644 index 0000000..6f778e6 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case1/psnr_result1.json @@ -0,0 +1,5 @@ +{ + "gt_video": "/mnt/ASC1637/unifolm-world-model-action/unitree_z1_dual_arm_cleanup_pencils/case1/output/inference/unitree_z1_dual_arm_cleanup_pencils_case1_amd.mp4", + "pred_video": "/mnt/ASC1637/unifolm-world-model-action/unitree_z1_dual_arm_cleanup_pencils/case1/output/inference/0_full_fs4.mp4", + "psnr": 44.83864567508593 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_cleanup_pencils/case2/output.log b/unitree_z1_dual_arm_cleanup_pencils/case2/output.log new file mode 100644 index 0000000..295cfb2 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case2/output.log @@ -0,0 +1,137 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 06:59:34.465946: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 06:59:34.469367: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 06:59:34.500805: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 06:59:34.500837: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 06:59:34.502917: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 06:59:34.511434: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 06:59:34.511678: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 06:59:35.478194: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/8 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 12%|█▎ | 1/8 [01:37<11:23, 97.57s/it] 25%|██▌ | 2/8 [03:14<09:44, 97.48s/it] 38%|███▊ | 3/8 [04:52<08:07, 97.47s/it] 50%|█████ | 4/8 [06:29<06:29, 97.49s/it] 62%|██████▎ | 5/8 [08:07<04:52, 97.42s/it] 75%|███████▌ | 6/8 [09:44<03:14, 97.32s/it] 88%|████████▊ | 7/8 [11:21<01:37, 97.34s/it] 100%|██████████| 8/8 [12:59<00:00, 97.36s/it] 100%|██████████| 8/8 [12:59<00:00, 97.40s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 14m25.691s +user 12m47.050s +sys 0m51.550s diff --git a/unitree_z1_dual_arm_cleanup_pencils/case2/psnr_result.json b/unitree_z1_dual_arm_cleanup_pencils/case2/psnr_result.json new file mode 100644 index 0000000..b427406 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case2/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_cleanup_pencils/case2/unitree_z1_dual_arm_cleanup_pencils_case2.mp4", + "pred_video": "unitree_z1_dual_arm_cleanup_pencils/case2/output/inference/unitree_z1_dual_arm_cleanup_pencils_case2_amd.mp4", + "psnr": 20.484298972158296 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_cleanup_pencils/case3/output.log b/unitree_z1_dual_arm_cleanup_pencils/case3/output.log new file mode 100644 index 0000000..e7dc987 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case3/output.log @@ -0,0 +1,137 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:18:52.629976: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:18:52.633025: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:18:52.663985: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:18:52.664018: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:18:52.665837: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:18:52.673889: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:18:52.674218: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:18:53.298338: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/8 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 12%|█▎ | 1/8 [01:40<11:43, 100.54s/it] 25%|██▌ | 2/8 [03:20<10:02, 100.36s/it] 38%|███▊ | 3/8 [05:01<08:21, 100.32s/it] 50%|█████ | 4/8 [06:41<06:41, 100.36s/it] 62%|██████▎ | 5/8 [08:21<05:00, 100.30s/it] 75%|███████▌ | 6/8 [10:01<03:20, 100.28s/it] 88%|████████▊ | 7/8 [11:42<01:40, 100.34s/it] 100%|██████████| 8/8 [13:22<00:00, 100.36s/it] 100%|██████████| 8/8 [13:22<00:00, 100.34s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 14m26.030s +user 13m2.487s +sys 0m43.472s diff --git a/unitree_z1_dual_arm_cleanup_pencils/case3/psnr_result.json b/unitree_z1_dual_arm_cleanup_pencils/case3/psnr_result.json new file mode 100644 index 0000000..deb9f3b --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case3/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_cleanup_pencils/case3/unitree_z1_dual_arm_cleanup_pencils_case3.mp4", + "pred_video": "unitree_z1_dual_arm_cleanup_pencils/case3/output/inference/unitree_z1_dual_arm_cleanup_pencils_case3_amd.mp4", + "psnr": 21.20205061239349 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_cleanup_pencils/case4/output.log b/unitree_z1_dual_arm_cleanup_pencils/case4/output.log new file mode 100644 index 0000000..0ea728b --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case4/output.log @@ -0,0 +1,137 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:22:15.333099: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:22:15.336215: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:22:15.366489: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:22:15.366522: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:22:15.368294: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:22:15.376202: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:22:15.376444: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:22:15.995383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/8 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 12%|█▎ | 1/8 [01:37<11:23, 97.68s/it] 25%|██▌ | 2/8 [03:15<09:47, 97.83s/it] 38%|███▊ | 3/8 [04:53<08:09, 97.91s/it] 50%|█████ | 4/8 [06:31<06:32, 98.03s/it] 62%|██████▎ | 5/8 [08:10<04:54, 98.11s/it] 75%|███████▌ | 6/8 [09:48<03:16, 98.18s/it] 88%|████████▊ | 7/8 [11:26<01:38, 98.24s/it] 100%|██████████| 8/8 [13:04<00:00, 98.16s/it] 100%|██████████| 8/8 [13:04<00:00, 98.09s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 14m8.258s +user 13m1.587s +sys 0m47.389s diff --git a/unitree_z1_dual_arm_cleanup_pencils/case4/psnr_result.json b/unitree_z1_dual_arm_cleanup_pencils/case4/psnr_result.json new file mode 100644 index 0000000..8e4f5b5 --- /dev/null +++ b/unitree_z1_dual_arm_cleanup_pencils/case4/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_cleanup_pencils/case4/unitree_z1_dual_arm_cleanup_pencils_case4.mp4", + "pred_video": "unitree_z1_dual_arm_cleanup_pencils/case4/output/inference/unitree_z1_dual_arm_cleanup_pencils_case4_amd.mp4", + "psnr": 21.130122583788612 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox/case1/output.log b/unitree_z1_dual_arm_stackbox/case1/output.log new file mode 100644 index 0000000..8d12770 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case1/output.log @@ -0,0 +1,134 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:24:40.357099: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:24:40.360365: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:24:40.391744: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:24:40.391772: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:24:40.393608: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:24:40.401837: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:24:40.402077: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:24:41.022382: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/7 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 14%|█▍ | 1/7 [01:41<10:09, 101.63s/it] 29%|██▊ | 2/7 [03:20<08:18, 99.78s/it] 43%|████▎ | 3/7 [04:58<06:36, 99.24s/it] 57%|█████▋ | 4/7 [06:37<04:57, 99.05s/it] 71%|███████▏ | 5/7 [08:16<03:17, 98.90s/it] 86%|████████▌ | 6/7 [09:54<01:38, 98.80s/it] 100%|██████████| 7/7 [11:33<00:00, 98.70s/it] 100%|██████████| 7/7 [11:33<00:00, 99.03s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 13m3.558s +user 11m47.297s +sys 1m10.848s diff --git a/unitree_z1_dual_arm_stackbox/case1/psnr_result.json b/unitree_z1_dual_arm_stackbox/case1/psnr_result.json new file mode 100644 index 0000000..01d5deb --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case1/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox/case1/unitree_z1_dual_arm_stackbox_case1.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox/case1/output/inference/unitree_z1_dual_arm_stackbox_case1_amd.mp4", + "psnr": 21.258130518117493 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox/case1/run_world_model_interaction.sh b/unitree_z1_dual_arm_stackbox/case1/run_world_model_interaction.sh index 0d9ed4c..07c12f8 100644 --- a/unitree_z1_dual_arm_stackbox/case1/run_world_model_interaction.sh +++ b/unitree_z1_dual_arm_stackbox/case1/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_dual_arm_stackbox/case1" dataset="unitree_z1_dual_arm_stackbox" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=7 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \ diff --git a/unitree_z1_dual_arm_stackbox/case2/output.log b/unitree_z1_dual_arm_stackbox/case2/output.log new file mode 100644 index 0000000..3351455 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case2/output.log @@ -0,0 +1,134 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:25:18.653033: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:25:18.656060: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:25:18.687077: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:25:18.687119: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:25:18.688915: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:25:18.697008: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:25:18.697255: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:25:19.338303: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/7 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 14%|█▍ | 1/7 [01:39<09:56, 99.35s/it] 29%|██▊ | 2/7 [03:18<08:17, 99.50s/it] 43%|████▎ | 3/7 [04:58<06:38, 99.54s/it] 57%|█████▋ | 4/7 [06:38<04:58, 99.52s/it] 71%|███████▏ | 5/7 [08:17<03:19, 99.55s/it] 86%|████████▌ | 6/7 [09:57<01:39, 99.53s/it] 100%|██████████| 7/7 [11:36<00:00, 99.50s/it] 100%|██████████| 7/7 [11:36<00:00, 99.51s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 13m28.231s +user 12m40.856s +sys 1m40.352s diff --git a/unitree_z1_dual_arm_stackbox/case2/psnr_result.json b/unitree_z1_dual_arm_stackbox/case2/psnr_result.json new file mode 100644 index 0000000..4d8ef1e --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case2/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox/case2/unitree_z1_dual_arm_stackbox_case2.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox/case2/output/inference/unitree_z1_dual_arm_stackbox_case2_amd.mp4", + "psnr": 23.878153424077645 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox/case2/run_world_model_interaction.sh b/unitree_z1_dual_arm_stackbox/case2/run_world_model_interaction.sh index 7b6d005..a75c901 100644 --- a/unitree_z1_dual_arm_stackbox/case2/run_world_model_interaction.sh +++ b/unitree_z1_dual_arm_stackbox/case2/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_dual_arm_stackbox/case2" dataset="unitree_z1_dual_arm_stackbox" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=6 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \ diff --git a/unitree_z1_dual_arm_stackbox/case3/output.log b/unitree_z1_dual_arm_stackbox/case3/output.log new file mode 100644 index 0000000..e2f3b64 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case3/output.log @@ -0,0 +1,134 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:35:33.682231: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:35:33.685275: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:35:33.716682: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:35:33.716728: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:35:33.718523: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:35:33.726756: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:35:33.727105: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:35:34.356722: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/7 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 14%|█▍ | 1/7 [01:41<10:06, 101.02s/it] 29%|██▊ | 2/7 [03:23<08:29, 101.84s/it] 43%|████▎ | 3/7 [05:04<06:45, 101.43s/it] 57%|█████▋ | 4/7 [06:45<05:04, 101.42s/it] 71%|███████▏ | 5/7 [08:27<03:22, 101.40s/it] 86%|████████▌ | 6/7 [10:08<01:41, 101.39s/it] 100%|██████████| 7/7 [11:49<00:00, 101.33s/it] 100%|██████████| 7/7 [11:49<00:00, 101.39s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 12m57.093s +user 11m59.943s +sys 0m58.711s diff --git a/unitree_z1_dual_arm_stackbox/case3/psnr_result.json b/unitree_z1_dual_arm_stackbox/case3/psnr_result.json new file mode 100644 index 0000000..5cb47fb --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case3/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox/case3/unitree_z1_dual_arm_stackbox_case3.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox/case3/output/inference/unitree_z1_dual_arm_stackbox_case3_amd.mp4", + "psnr": 25.400458754751128 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox/case4/output.log b/unitree_z1_dual_arm_stackbox/case4/output.log new file mode 100644 index 0000000..bd0b25a --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case4/output.log @@ -0,0 +1,134 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:38:45.572744: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:38:45.576864: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:38:45.624825: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:38:45.624883: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:38:45.627150: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:38:45.638316: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:38:45.638803: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:38:46.426363: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/7 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 14%|█▍ | 1/7 [01:38<09:52, 98.73s/it] 29%|██▊ | 2/7 [03:17<08:14, 98.85s/it] 43%|████▎ | 3/7 [04:56<06:35, 98.80s/it] 57%|█████▋ | 4/7 [06:35<04:56, 98.94s/it] 71%|███████▏ | 5/7 [08:14<03:17, 98.93s/it] 86%|████████▌ | 6/7 [09:53<01:38, 98.89s/it] 100%|██████████| 7/7 [11:31<00:00, 98.81s/it] 100%|██████████| 7/7 [11:31<00:00, 98.85s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 12m36.996s +user 11m43.720s +sys 0m46.859s diff --git a/unitree_z1_dual_arm_stackbox/case4/psnr_result.json b/unitree_z1_dual_arm_stackbox/case4/psnr_result.json new file mode 100644 index 0000000..b976bf1 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox/case4/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox/case4/unitree_z1_dual_arm_stackbox_case4.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox/case4/output/inference/unitree_z1_dual_arm_stackbox_case4_amd.mp4", + "psnr": 24.098958457373858 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox_v2/case1/output.log b/unitree_z1_dual_arm_stackbox_v2/case1/output.log new file mode 100644 index 0000000..e360207 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case1/output.log @@ -0,0 +1,146 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:51:23.961486: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:51:24.200063: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:51:24.522299: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:51:24.522350: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:51:24.528237: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:51:24.579400: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:51:24.579644: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:51:25.781311: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:38<16:20, 98.04s/it] 18%|█▊ | 2/11 [03:15<14:40, 97.81s/it] 27%|██▋ | 3/11 [04:53<13:01, 97.72s/it] 36%|███▋ | 4/11 [06:31<11:24, 97.71s/it] 45%|████▌ | 5/11 [08:08<09:46, 97.71s/it] 55%|█████▍ | 6/11 [09:46<08:08, 97.65s/it] 64%|██████▎ | 7/11 [11:23<06:30, 97.65s/it] 73%|███████▎ | 8/11 [13:02<04:54, 98.09s/it] 82%|████████▏ | 9/11 [14:40<03:15, 97.83s/it] 91%|█████████ | 10/11 [16:17<01:37, 97.73s/it] 100%|██████████| 11/11 [17:55<00:00, 97.64s/it] 100%|██████████| 11/11 [17:55<00:00, 97.74s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m33.920s +user 16m44.564s +sys 0m45.115s diff --git a/unitree_z1_dual_arm_stackbox_v2/case1/psnr_result.json b/unitree_z1_dual_arm_stackbox_v2/case1/psnr_result.json new file mode 100644 index 0000000..d6645d8 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case1/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox_v2/case1/unitree_z1_dual_arm_stackbox_v2_case1.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox_v2/case1/output/inference/unitree_z1_dual_arm_stackbox_v2_case1_amd.mp4", + "psnr": 18.126776535969576 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox_v2/case1/run_world_model_interaction.sh b/unitree_z1_dual_arm_stackbox_v2/case1/run_world_model_interaction.sh index bdcbbff..9cd5402 100644 --- a/unitree_z1_dual_arm_stackbox_v2/case1/run_world_model_interaction.sh +++ b/unitree_z1_dual_arm_stackbox_v2/case1/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_dual_arm_stackbox_v2/case1" dataset="unitree_z1_dual_arm_stackbox_v2" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=7 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \ diff --git a/unitree_z1_dual_arm_stackbox_v2/case2/output.log b/unitree_z1_dual_arm_stackbox_v2/case2/output.log new file mode 100644 index 0000000..c8a347f --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case2/output.log @@ -0,0 +1,146 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:56:31.144789: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:56:31.148256: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:56:31.178870: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:56:31.178898: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:56:31.180683: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:56:31.188800: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:56:31.189142: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:56:31.810098: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:40<16:41, 100.16s/it] 18%|█▊ | 2/11 [03:20<15:04, 100.47s/it] 27%|██▋ | 3/11 [05:01<13:24, 100.62s/it] 36%|███▋ | 4/11 [06:42<11:44, 100.69s/it] 45%|████▌ | 5/11 [08:22<10:02, 100.48s/it] 55%|█████▍ | 6/11 [10:02<08:21, 100.33s/it] 64%|██████▎ | 7/11 [11:42<06:40, 100.23s/it] 73%|███████▎ | 8/11 [13:22<05:00, 100.23s/it] 82%|████████▏ | 9/11 [15:03<03:20, 100.23s/it] 91%|█████████ | 10/11 [16:43<01:40, 100.33s/it] 100%|██████████| 11/11 [18:24<00:00, 100.41s/it] 100%|██████████| 11/11 [18:24<00:00, 100.39s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m39.936s +user 17m50.106s +sys 1m16.084s diff --git a/unitree_z1_dual_arm_stackbox_v2/case2/psnr_result.json b/unitree_z1_dual_arm_stackbox_v2/case2/psnr_result.json new file mode 100644 index 0000000..ff123e0 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case2/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox_v2/case2/unitree_z1_dual_arm_stackbox_v2_case2.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox_v2/case2/output/inference/unitree_z1_dual_arm_stackbox_v2_case2_amd.mp4", + "psnr": 19.38130614773096 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox_v2/case3/output.log b/unitree_z1_dual_arm_stackbox_v2/case3/output.log new file mode 100644 index 0000000..3f4ce97 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case3/output.log @@ -0,0 +1,146 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 07:56:04.467082: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 07:56:04.470145: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:56:04.502248: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 07:56:04.502277: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 07:56:04.504088: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 07:56:04.512557: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 07:56:04.512830: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 07:56:05.259641: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:38<16:20, 98.03s/it] 18%|█▊ | 2/11 [03:16<14:43, 98.19s/it] 27%|██▋ | 3/11 [04:55<13:08, 98.54s/it] 36%|███▋ | 4/11 [06:33<11:29, 98.52s/it] 45%|████▌ | 5/11 [08:11<09:50, 98.38s/it] 55%|█████▍ | 6/11 [09:49<08:10, 98.11s/it] 64%|██████▎ | 7/11 [11:27<06:31, 97.97s/it] 73%|███████▎ | 8/11 [13:04<04:53, 97.83s/it] 82%|████████▏ | 9/11 [14:42<03:15, 97.72s/it] 91%|█████████ | 10/11 [16:19<01:37, 97.71s/it] 100%|██████████| 11/11 [17:57<00:00, 97.74s/it] 100%|██████████| 11/11 [17:57<00:00, 97.97s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m10.550s +user 17m29.662s +sys 1m3.432s diff --git a/unitree_z1_dual_arm_stackbox_v2/case3/psnr_result.json b/unitree_z1_dual_arm_stackbox_v2/case3/psnr_result.json new file mode 100644 index 0000000..d40e8d2 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case3/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox_v2/case3/unitree_z1_dual_arm_stackbox_v2_case3.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox_v2/case3/output/inference/unitree_z1_dual_arm_stackbox_v2_case3_amd.mp4", + "psnr": 18.74462122425683 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox_v2/case4/output.log b/unitree_z1_dual_arm_stackbox_v2/case4/output.log new file mode 100644 index 0000000..dfd26f8 --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case4/output.log @@ -0,0 +1,146 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:04:16.104516: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:04:16.109112: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:04:16.138703: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:04:16.138737: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:04:16.140302: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:04:16.147672: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:04:16.147903: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:04:17.363218: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/11 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 9%|▉ | 1/11 [01:39<16:32, 99.26s/it] 18%|█▊ | 2/11 [03:17<14:49, 98.81s/it] 27%|██▋ | 3/11 [04:56<13:10, 98.76s/it] 36%|███▋ | 4/11 [06:35<11:31, 98.80s/it] 45%|████▌ | 5/11 [08:14<09:53, 98.85s/it] 55%|█████▍ | 6/11 [09:53<08:14, 98.87s/it] 64%|██████▎ | 7/11 [11:31<06:34, 98.68s/it] 73%|███████▎ | 8/11 [13:09<04:55, 98.49s/it] 82%|████████▏ | 9/11 [14:47<03:16, 98.38s/it] 91%|█████████ | 10/11 [16:25<01:38, 98.29s/it] 100%|██████████| 11/11 [18:03<00:00, 98.26s/it] 100%|██████████| 11/11 [18:03<00:00, 98.54s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 19m41.120s +user 16m53.778s +sys 0m53.898s diff --git a/unitree_z1_dual_arm_stackbox_v2/case4/psnr_result.json b/unitree_z1_dual_arm_stackbox_v2/case4/psnr_result.json new file mode 100644 index 0000000..64967ec --- /dev/null +++ b/unitree_z1_dual_arm_stackbox_v2/case4/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_dual_arm_stackbox_v2/case4/unitree_z1_dual_arm_stackbox_v2_case4.mp4", + "pred_video": "unitree_z1_dual_arm_stackbox_v2/case4/output/inference/unitree_z1_dual_arm_stackbox_v2_case4_amd.mp4", + "psnr": 19.526448380726254 +} \ No newline at end of file diff --git a/unitree_z1_dual_arm_stackbox_v2/case4/run_world_model_interaction.sh b/unitree_z1_dual_arm_stackbox_v2/case4/run_world_model_interaction.sh index 370c1c3..c8a5d77 100644 --- a/unitree_z1_dual_arm_stackbox_v2/case4/run_world_model_interaction.sh +++ b/unitree_z1_dual_arm_stackbox_v2/case4/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_dual_arm_stackbox_v2/case4" dataset="unitree_z1_dual_arm_stackbox_v2" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=6 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \ diff --git a/unitree_z1_stackbox/case1/output.log b/unitree_z1_stackbox/case1/output.log new file mode 100644 index 0000000..9df9505 --- /dev/null +++ b/unitree_z1_stackbox/case1/output.log @@ -0,0 +1,149 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:12:47.424053: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:12:47.427280: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:12:47.458253: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:12:47.458288: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:12:47.462758: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:12:47.518283: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:12:47.518566: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:12:48.593011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/12 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 8%|▊ | 1/12 [01:38<18:08, 98.94s/it] 17%|█▋ | 2/12 [03:18<16:30, 99.01s/it] 25%|██▌ | 3/12 [04:57<14:51, 99.07s/it] 33%|███▎ | 4/12 [06:36<13:12, 99.04s/it] 42%|████▏ | 5/12 [08:15<11:33, 99.00s/it] 50%|█████ | 6/12 [09:54<09:54, 99.10s/it] 58%|█████▊ | 7/12 [11:33<08:14, 99.00s/it] 67%|██████▋ | 8/12 [13:13<06:38, 99.58s/it] 75%|███████▌ | 9/12 [14:54<04:59, 99.88s/it] 83%|████████▎ | 10/12 [16:33<03:19, 99.58s/it] 92%|█████████▏| 11/12 [18:12<01:39, 99.39s/it] 100%|██████████| 12/12 [19:51<00:00, 99.25s/it] 100%|██████████| 12/12 [19:51<00:00, 99.28s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 11: generating actions ... +>>> Step 11: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 21m30.786s +user 18m14.269s +sys 0m57.007s diff --git a/unitree_z1_stackbox/case1/psnr_result.json b/unitree_z1_stackbox/case1/psnr_result.json new file mode 100644 index 0000000..fd9374d --- /dev/null +++ b/unitree_z1_stackbox/case1/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_stackbox/case1/unitree_z1_stackbox_case1.mp4", + "pred_video": "unitree_z1_stackbox/case1/output/inference/unitree_z1_stackbox_case1_amd.mp4", + "psnr": 19.81391789862606 +} \ No newline at end of file diff --git a/unitree_z1_stackbox/case1/run_world_model_interaction.sh b/unitree_z1_stackbox/case1/run_world_model_interaction.sh index 73d9132..2547c58 100644 --- a/unitree_z1_stackbox/case1/run_world_model_interaction.sh +++ b/unitree_z1_stackbox/case1/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_stackbox/case1" dataset="unitree_z1_stackbox" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=5 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \ diff --git a/unitree_z1_stackbox/case2/output.log b/unitree_z1_stackbox/case2/output.log new file mode 100644 index 0000000..410213c --- /dev/null +++ b/unitree_z1_stackbox/case2/output.log @@ -0,0 +1,149 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:15:49.934949: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:15:49.937974: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:15:49.969069: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:15:49.969100: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:15:49.970909: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:15:49.979005: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:15:49.979255: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:15:50.597743: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/12 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 8%|▊ | 1/12 [01:37<17:51, 97.37s/it] 17%|█▋ | 2/12 [03:14<16:13, 97.31s/it] 25%|██▌ | 3/12 [04:51<14:35, 97.26s/it] 33%|███▎ | 4/12 [06:29<12:58, 97.25s/it] 42%|████▏ | 5/12 [08:06<11:20, 97.24s/it] 50%|█████ | 6/12 [09:43<09:43, 97.24s/it] 58%|█████▊ | 7/12 [11:20<08:06, 97.27s/it] 67%|██████▋ | 8/12 [12:58<06:29, 97.36s/it] 75%|███████▌ | 9/12 [14:36<04:52, 97.49s/it] 83%|████████▎ | 10/12 [16:13<03:15, 97.52s/it] 92%|█████████▏| 11/12 [17:51<01:37, 97.47s/it] 100%|██████████| 12/12 [19:28<00:00, 97.35s/it] 100%|██████████| 12/12 [19:28<00:00, 97.35s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 11: generating actions ... +>>> Step 11: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 20m36.252s +user 18m13.951s +sys 0m58.438s diff --git a/unitree_z1_stackbox/case2/psnr_result.json b/unitree_z1_stackbox/case2/psnr_result.json new file mode 100644 index 0000000..f3f5b50 --- /dev/null +++ b/unitree_z1_stackbox/case2/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_stackbox/case2/unitree_z1_stackbox_case2.mp4", + "pred_video": "unitree_z1_stackbox/case2/output/inference/unitree_z1_stackbox_case2_amd.mp4", + "psnr": 21.083821459054743 +} \ No newline at end of file diff --git a/unitree_z1_stackbox/case3/output.log b/unitree_z1_stackbox/case3/output.log new file mode 100644 index 0000000..b317f57 --- /dev/null +++ b/unitree_z1_stackbox/case3/output.log @@ -0,0 +1,149 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:16:22.299521: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:16:22.302545: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:16:22.335354: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:16:22.335389: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:16:22.337179: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:16:22.345296: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:16:22.345548: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:16:23.008743: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +[rank: 0] Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/12 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 8%|▊ | 1/12 [01:39<18:16, 99.64s/it] 17%|█▋ | 2/12 [03:19<16:35, 99.56s/it] 25%|██▌ | 3/12 [04:58<14:55, 99.53s/it] 33%|███▎ | 4/12 [06:38<13:16, 99.53s/it] 42%|████▏ | 5/12 [08:17<11:36, 99.54s/it] 50%|█████ | 6/12 [09:57<09:57, 99.57s/it] 58%|█████▊ | 7/12 [11:37<08:18, 99.66s/it] 67%|██████▋ | 8/12 [13:17<06:39, 99.83s/it] 75%|███████▌ | 9/12 [14:57<04:59, 99.93s/it] 83%|████████▎ | 10/12 [16:37<03:19, 99.97s/it] 92%|█████████▏| 11/12 [18:17<01:39, 99.85s/it] 100%|██████████| 12/12 [19:56<00:00, 99.71s/it] 100%|██████████| 12/12 [19:56<00:00, 99.71s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 11: generating actions ... +>>> Step 11: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 21m2.951s +user 18m44.789s +sys 1m4.895s diff --git a/unitree_z1_stackbox/case3/psnr_result.json b/unitree_z1_stackbox/case3/psnr_result.json new file mode 100644 index 0000000..2f54211 --- /dev/null +++ b/unitree_z1_stackbox/case3/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_stackbox/case3/unitree_z1_stackbox_case3.mp4", + "pred_video": "unitree_z1_stackbox/case3/output/inference/unitree_z1_stackbox_case3_amd.mp4", + "psnr": 21.322784880212172 +} \ No newline at end of file diff --git a/unitree_z1_stackbox/case4/output.log b/unitree_z1_stackbox/case4/output.log new file mode 100644 index 0000000..c9a3afc --- /dev/null +++ b/unitree_z1_stackbox/case4/output.log @@ -0,0 +1,149 @@ +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/lightning_fabric/__init__.py:29: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. + __import__("pkg_resources").declare_namespace(__name__) +2026-02-08 08:25:54.657305: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-02-08 08:25:54.660628: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:25:54.691237: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered +2026-02-08 08:25:54.691275: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered +2026-02-08 08:25:54.693046: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered +2026-02-08 08:25:54.701142: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used. +2026-02-08 08:25:54.701413: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +2026-02-08 08:25:55.801367: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT +Global seed set to 123 +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/kornia/feature/lightglue.py:44: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) +INFO:mainlogger:LatentVisualDiffusion: Running in v-prediction mode +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +INFO:unifolm_wma.models.diffusion_head.conditional_unet1d:number of parameters: 5.010531e+08 +AE working on z of shape (1, 4, 32, 32) = 4096 dimensions. +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): hf-mirror.com:443 +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/miniconda3/envs/unifolm-wma-o/lib/python3.10/site-packages/open_clip/factory.py:88: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + checkpoint = torch.load(checkpoint_path, map_location=map_location) +INFO:root:Loaded ViT-H-14 model config. +DEBUG:urllib3.connectionpool:https://hf-mirror.com:443 "HEAD /laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin HTTP/1.1" 302 0 +INFO:root:Loading pretrained ViT-H-14 weights (laion2b_s32b_b79k). +/mnt/ASC1637/unifolm-world-model-action/scripts/evaluation/world_model_interaction.py:86: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + state_dict = torch.load(ckpt, map_location="cpu") +>>> model checkpoint loaded. +>>> Load pre-trained model ... +INFO:root:***** Configing Data ***** +>>> unitree_z1_stackbox: 1 data samples loaded. +>>> unitree_z1_stackbox: data stats loaded. +>>> unitree_z1_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox: data stats loaded. +>>> unitree_z1_dual_arm_stackbox: normalizer initiated. +>>> unitree_z1_dual_arm_stackbox_v2: 1 data samples loaded. +>>> unitree_z1_dual_arm_stackbox_v2: data stats loaded. +>>> unitree_z1_dual_arm_stackbox_v2: normalizer initiated. +>>> unitree_z1_dual_arm_cleanup_pencils: 1 data samples loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: data stats loaded. +>>> unitree_z1_dual_arm_cleanup_pencils: normalizer initiated. +>>> unitree_g1_pack_camera: 1 data samples loaded. +>>> unitree_g1_pack_camera: data stats loaded. +>>> unitree_g1_pack_camera: normalizer initiated. +>>> Dataset is successfully loaded ... +>>> Generate 16 frames under each generation ... +DEBUG:h5py._conv:Creating converter from 3 to 5 +DEBUG:PIL.PngImagePlugin:STREAM b'IHDR' 16 13 +DEBUG:PIL.PngImagePlugin:STREAM b'pHYs' 41 9 +DEBUG:PIL.PngImagePlugin:STREAM b'IDAT' 62 4096 + 0%| | 0/12 [00:00>> Step 0: generating actions ... +>>> Step 0: interacting with world model ... +DEBUG:PIL.Image:Importing BlpImagePlugin +DEBUG:PIL.Image:Importing BmpImagePlugin +DEBUG:PIL.Image:Importing BufrStubImagePlugin +DEBUG:PIL.Image:Importing CurImagePlugin +DEBUG:PIL.Image:Importing DcxImagePlugin +DEBUG:PIL.Image:Importing DdsImagePlugin +DEBUG:PIL.Image:Importing EpsImagePlugin +DEBUG:PIL.Image:Importing FitsImagePlugin +DEBUG:PIL.Image:Importing FitsStubImagePlugin +DEBUG:PIL.Image:Importing FliImagePlugin +DEBUG:PIL.Image:Importing FpxImagePlugin +DEBUG:PIL.Image:Image: failed to import FpxImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing FtexImagePlugin +DEBUG:PIL.Image:Importing GbrImagePlugin +DEBUG:PIL.Image:Importing GifImagePlugin +DEBUG:PIL.Image:Importing GribStubImagePlugin +DEBUG:PIL.Image:Importing Hdf5StubImagePlugin +DEBUG:PIL.Image:Importing IcnsImagePlugin +DEBUG:PIL.Image:Importing IcoImagePlugin +DEBUG:PIL.Image:Importing ImImagePlugin +DEBUG:PIL.Image:Importing ImtImagePlugin +DEBUG:PIL.Image:Importing IptcImagePlugin +DEBUG:PIL.Image:Importing JpegImagePlugin +DEBUG:PIL.Image:Importing Jpeg2KImagePlugin +DEBUG:PIL.Image:Importing McIdasImagePlugin +DEBUG:PIL.Image:Importing MicImagePlugin +DEBUG:PIL.Image:Image: failed to import MicImagePlugin: No module named 'olefile' +DEBUG:PIL.Image:Importing MpegImagePlugin +DEBUG:PIL.Image:Importing MpoImagePlugin +DEBUG:PIL.Image:Importing MspImagePlugin +DEBUG:PIL.Image:Importing PalmImagePlugin +DEBUG:PIL.Image:Importing PcdImagePlugin +DEBUG:PIL.Image:Importing PcxImagePlugin +DEBUG:PIL.Image:Importing PdfImagePlugin +DEBUG:PIL.Image:Importing PixarImagePlugin +DEBUG:PIL.Image:Importing PngImagePlugin +DEBUG:PIL.Image:Importing PpmImagePlugin +DEBUG:PIL.Image:Importing PsdImagePlugin +DEBUG:PIL.Image:Importing QoiImagePlugin +DEBUG:PIL.Image:Importing SgiImagePlugin +DEBUG:PIL.Image:Importing SpiderImagePlugin +DEBUG:PIL.Image:Importing SunImagePlugin +DEBUG:PIL.Image:Importing TgaImagePlugin +DEBUG:PIL.Image:Importing TiffImagePlugin +DEBUG:PIL.Image:Importing WebPImagePlugin +DEBUG:PIL.Image:Importing WmfImagePlugin +DEBUG:PIL.Image:Importing XbmImagePlugin +DEBUG:PIL.Image:Importing XpmImagePlugin +DEBUG:PIL.Image:Importing XVThumbImagePlugin + 8%|▊ | 1/12 [01:37<17:51, 97.38s/it] 17%|█▋ | 2/12 [03:14<16:12, 97.24s/it] 25%|██▌ | 3/12 [04:51<14:35, 97.28s/it] 33%|███▎ | 4/12 [06:29<12:59, 97.40s/it] 42%|████▏ | 5/12 [08:06<11:21, 97.30s/it] 50%|█████ | 6/12 [09:43<09:43, 97.17s/it] 58%|█████▊ | 7/12 [11:20<08:05, 97.07s/it] 67%|██████▋ | 8/12 [12:57<06:28, 97.02s/it] 75%|███████▌ | 9/12 [14:34<04:50, 96.98s/it] 83%|████████▎ | 10/12 [16:11<03:14, 97.00s/it] 92%|█████████▏| 11/12 [17:48<01:37, 97.06s/it] 100%|██████████| 12/12 [19:25<00:00, 97.13s/it] 100%|██████████| 12/12 [19:25<00:00, 97.14s/it] +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 1: generating actions ... +>>> Step 1: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 2: generating actions ... +>>> Step 2: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 3: generating actions ... +>>> Step 3: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 4: generating actions ... +>>> Step 4: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 5: generating actions ... +>>> Step 5: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 6: generating actions ... +>>> Step 6: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 7: generating actions ... +>>> Step 7: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 8: generating actions ... +>>> Step 8: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 9: generating actions ... +>>> Step 9: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 10: generating actions ... +>>> Step 10: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> +>>> Step 11: generating actions ... +>>> Step 11: interacting with world model ... +>>>>>>>>>>>>>>>>>>>>>>>> + +real 21m23.164s +user 18m27.701s +sys 1m28.356s diff --git a/unitree_z1_stackbox/case4/psnr_result.json b/unitree_z1_stackbox/case4/psnr_result.json new file mode 100644 index 0000000..e596062 --- /dev/null +++ b/unitree_z1_stackbox/case4/psnr_result.json @@ -0,0 +1,5 @@ +{ + "gt_video": "unitree_z1_stackbox/case4/unitree_z1_stackbox_case4.mp4", + "pred_video": "unitree_z1_stackbox/case4/output/inference/unitree_z1_stackbox_case4_amd.mp4", + "psnr": 25.32928948331741 +} \ No newline at end of file diff --git a/unitree_z1_stackbox/case4/run_world_model_interaction.sh b/unitree_z1_stackbox/case4/run_world_model_interaction.sh index 054b175..3560fbd 100644 --- a/unitree_z1_stackbox/case4/run_world_model_interaction.sh +++ b/unitree_z1_stackbox/case4/run_world_model_interaction.sh @@ -2,7 +2,7 @@ res_dir="unitree_z1_stackbox/case4" dataset="unitree_z1_stackbox" { - time CUDA_VISIBLE_DEVICES=0 python3 scripts/evaluation/world_model_interaction.py \ + time CUDA_VISIBLE_DEVICES=7 python3 scripts/evaluation/world_model_interaction.py \ --seed 123 \ --ckpt_path ckpts/unifolm_wma_dual.ckpt \ --config configs/inference/world_model_interaction.yaml \