save

aaa_helpers.txt

@@ -104,6 +104,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_pre_cape_f" > nohup/iccv
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128" > nohup/iccv/pyr/iccv_pyr_32_64_128.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skirt.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin_2.out 2>&1 &
+nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin_skirt.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_reg_2.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_curv.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_curv_skirt.out 2>&1 &
@@ -117,6 +118,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256" > nohup/iccv
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skirt_2.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skin.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv.out 2>&1 &
+nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv_skirt.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_reg.out 2>&1 &
 
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512.out 2>&1 &
@@ -143,6 +145,8 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_reg > nohu
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_skirt.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv > nohup/iccv/evaluate/iccv_pyr_32_64_curv.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_curv_skirt.out 2>&1 &
+nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_10p > nohup/iccv/evaluate/iccv_pyr_32_64_10p.out 2>&1 &
+nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_25p > nohup/iccv/evaluate/iccv_pyr_32_64_25p.out 2>&1 &
 
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt.out 2>&1 &

calculate_error.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_25p import config
+from configs.iccv.config_abl_pyr_32_64_25p import config
 
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 # import tensorflow as tf
@@ -65,7 +65,7 @@ if __name__ == '__main__':
     garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item()
 
 
-    output_folder = '/home/workspace/local-data/results/iccv/' + checkpoint_name + '-'
+    output_folder = '/home/workspace/local-data_128/results/iccv/' + checkpoint_name + '-'
     # check if there is a folder that starts with this name using glob
     folders = glob.glob(output_folder + '*')
     assert len(folders) == 1, f'There should be only one checkpoint that starts with this name, but found {len(folders)}'

configs/iccv/config_abl_pyr_32_64_10p.py

@@ -1,6 +1,6 @@
 config = dict()
 
-config['gpu'] = '4'
+config['gpu'] = '0'
 config['name'] = 'iccv_pyr_32_64_10p'
 config['notes'] = ''
 config['tags'] = ['pyr', 'iccv']
@@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = ''
 
 config['train_dir'] = '/home/workspace/local-data_127/train'
 config['valid_dir'] = '/home/workspace/local-data_127/valid'
-config['test_dir'] = '/home/workspace/local-data_127/test_ready'
+config['test_dir'] = '/home/workspace/local-data/test_ready'
 
 config['frames_file'] = 'frames_10p.txt'
 

configs/iccv/config_abl_pyr_32_64_128_256_512.py

@@ -1,6 +1,6 @@
 config = dict()
 
-config['gpu'] = '0'
+config['gpu'] = '1'
 config['name'] = 'iccv_pyr_32_64_128_256_512'
 config['notes'] = ''
 config['tags'] = ['pyr', 'iccv']
@@ -22,7 +22,7 @@ config['use_offsets'] =               False
 config['use_norm_loss'] =             True
 config['use_curvature'] =             True
 
-config['pretrained_model'] = ''
+config['pretrained_model'] = 'checkpoints/iccv/iccv_pyr_32_64_128_256_512'
 config['pretrained_model_iteration'] = ''
 
 config['pyramid'] = dict()
@@ -32,13 +32,15 @@ config['pyramid']['pretrained_pyramid_iteration'] = '90000'
 config['pyramid']['final_upscale_enhancer'] = ''
 
 config['train_dir'] = '/home/workspace/local-data/train'
-config['valid_dir'] = '/home/workspace/local-data/valid'
+config['valid_dir'] = '/home/workspace/local-data/train'
 config['test_dir'] = '/home/workspace/local-data/test_ready'
+config['frames_file'] = 'frames_00761_0.txt'
+config['frames_file_valid'] = 'frames_00761_0.txt'
 config['batch_size'] = 4
 config['is_skirt'] = False
 config['is_cape'] = False
 
-config['dataset_in_memory'] = False
+config['dataset_in_memory'] = True
 config['resolution'] = None
 
 config['smpl_res'] = 14475

configs/iccv/config_abl_pyr_32_64_128_256_curv_skirt.py

@@ -0,0 +1,187 @@
+config = dict()
+
+config['gpu'] = '1'
+config['name'] = 'iccv_pyr_32_64_128_256_curv_skirt'
+config['notes'] = ''
+config['tags'] = ['pyr', 'iccv', 'curv', 'skirt']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              True # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             False
+
+config['pretrained_model'] = ''
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64, 128, 256]
+config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_128_curv_skirt'
+config['pyramid']['pretrained_pyramid_iteration'] = ''
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data_128/train_skirt'
+config['valid_dir'] = '/home/workspace/local-data/valid_skirt'
+config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2'
+config['batch_size'] = 4
+config['is_skirt'] = True
+config['is_cape'] = False
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.0001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'

configs/iccv/config_abl_pyr_32_64_128_curv_skirt.py

@@ -94,11 +94,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -107,13 +106,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -122,7 +120,7 @@ for res in config['pyramid']['resolutions']:
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
     level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
     level['decoder_depths'] = level['encoder_depths']
     level['decoder_dims'] = level['encoder_dims']
     level['avg_garment_path'] = ''

configs/iccv/config_abl_pyr_32_64_128_reg_skirt.py

@@ -94,11 +94,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -107,13 +106,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -122,7 +120,7 @@ for res in config['pyramid']['resolutions']:
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
     level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
     level['decoder_depths'] = level['encoder_depths']
     level['decoder_dims'] = level['encoder_dims']
     level['avg_garment_path'] = ''

configs/iccv/config_abl_pyr_32_64_128_skin_skirt.py

@@ -1,6 +1,6 @@
 config = dict()
 
-config['gpu'] = '5'
+config['gpu'] = '0'
 config['name'] = 'iccv_pyr_32_64_128_skin_skirt'
 config['notes'] = ''
 config['tags'] = ['pyr', 'iccv']
@@ -31,7 +31,7 @@ config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_skin_
 config['pyramid']['final_upscale_enhancer'] = ''
 
 config['train_dir'] = '/home/workspace/local-data_128/train_skirt'
-config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt'
+config['valid_dir'] = '/home/workspace/local-data/valid_skirt'
 config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2'
 config['batch_size'] = 4
 config['is_skirt'] = True
@@ -100,11 +100,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -113,13 +112,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -128,7 +126,7 @@ for res in config['pyramid']['resolutions']:
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
     level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
     level['decoder_depths'] = level['encoder_depths']
     level['decoder_dims'] = level['encoder_dims']
     level['avg_garment_path'] = ''

configs/iccv/config_abl_pyr_32_64_25p.py

@@ -1,6 +1,6 @@
 config = dict()
 
-config['gpu'] = '5'
+config['gpu'] = '1'
 config['name'] = 'iccv_pyr_32_64_25p'
 config['notes'] = ''
 config['tags'] = ['pyr', 'iccv']
@@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = ''
 
 config['train_dir'] = '/home/workspace/local-data_127/train'
 config['valid_dir'] = '/home/workspace/local-data_127/valid'
-config['test_dir'] = '/home/workspace/local-data_127/test_ready'
+config['test_dir'] = '/home/workspace/local-data/test_ready'
 
 config['frames_file'] = 'frames_25p.txt'
 

configs/iccv/config_abl_pyr_32_64_curv_skirt.py

@@ -86,7 +86,6 @@ config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt'
 config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt'
 config['pyramid']['freeze_encoders'] = True
 
-
 config['pyramid']['levels'] = []
 for res in config['pyramid']['resolutions']:
   if res == 512:
@@ -94,11 +93,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -107,13 +105,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -121,13 +118,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 128
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [16, 9, 9, 9]
+    level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [64, 128, 128, 64]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
-    level['decoder_input'] = (4, 2, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [16, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 128, level['latent_dim']]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 64:

configs/iccv/config_abl_pyr_32_64_reg_skirt.py

@@ -86,7 +86,6 @@ config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt'
 config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt'
 config['pyramid']['freeze_encoders'] = True
 
-
 config['pyramid']['levels'] = []
 for res in config['pyramid']['resolutions']:
   if res == 512:
@@ -94,11 +93,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -107,13 +105,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -121,13 +118,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 128
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [16, 9, 9, 9]
+    level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [64, 128, 128, 64]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
-    level['decoder_input'] = (4, 2, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [16, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 128, level['latent_dim']]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 64:

configs/iccv/config_abl_pyr_32_64_skin_skirt copy.py

@@ -0,0 +1,193 @@
+config = dict()
+
+config['gpu'] = '2'
+config['name'] = 'iccv_pyr_32_64_skin_skirt'
+config['notes'] = ''
+config['tags'] = ['pyr', 'iccv']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              False # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             True
+
+config['pretrained_model'] = ''
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64]
+config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_skin_skirt'
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data_128/train_skirt'
+config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt'
+config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2'
+config['batch_size'] = 4
+config['is_skirt'] = True
+config['is_cape'] = False
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.0001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+if config['use_skinning']:
+  config['signature_dynamic']['uv_unposed'] = (512, 256, 3)
+  config['signature_dynamic']['mesh_unposed'] = (config['smpl_res'], 3,)
+else:
+  config['signature_dynamic']['uv_posed'] = (512, 256, 3)
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      if config['use_skinning']:
+        config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      else:
+        config['signature_dynamic']['uv_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'

configs/iccv/config_abl_pyr_32_64_skin_skirt.py

@@ -100,11 +100,10 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 512
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 7
-    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
-    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
     level['upscale_enhancer'] = ''
     config['pyramid']['levels'].append(level)
@@ -113,13 +112,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 256
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_depths'] = [3, 3, 9, 3]
-    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
-    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 64, 16]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 128:
@@ -127,13 +125,12 @@ for res in config['pyramid']['resolutions']:
     level['resolution'] = 128
     level['latent_dim'] = 64 * config['bottleneck_scale']
     level['kernel_size'] = 5
-    level['encoder_depths'] = [16, 9, 9, 9]
+    level['encoder_depths'] = [9, 9, 9]
-    level['encoder_dims'] = [64, 128, 128, 64]
+    level['encoder_dims'] = [64, 128, level['latent_dim']]
-    level['decoder_input'] = (4, 2, 128)
+    level['decoder_depths'] = level['encoder_depths']
-    level['decoder_depths'] = [16, 9, 9, 9]
+    level['decoder_dims'] = level['encoder_dims']
-    level['decoder_dims'] = [256, 128, 128, level['latent_dim']]
     level['avg_garment_path'] = ''
-    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2'
     level['upscale_enhancer_freeze'] = True
     config['pyramid']['levels'].append(level)
   elif res == 64:

curvature_analysis.py

@@ -338,7 +338,7 @@ def visualize_neighbourhoods(folder, frame):
     write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_25.obj', verts, faces.astype(np.int32), colors_binned_all[1].astype(np.int32))
     write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_100.obj', verts, faces.astype(np.int32), colors_binned_all[2].astype(np.int32))
     write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_mesh.obj', verts, faces.astype(np.int32), np.ones((verts.shape[0], 3), dtype=np.int32) * 200)
-
+    save_stl(f'examples/curvature_analysis/{id}_{frame_nr}_mesh.stl', verts, faces.astype(np.int32))
 
     fig.show()
     return fig
@@ -372,10 +372,10 @@ if __name__ == '__main__':
     i = 0
     while True:
         frame = np.random.choice(frames)
+        print(frame)
         # frame = '03265_193'
         # frame = '03102_49'
         # fig, dice = visualize_curvature_labels(folder, results_folder, frame)
         # wandb.log({f'{frame}-dice:{dice}': fig}, step=i)
         visualize_neighbourhoods(folder, frame)
         i += 1
-        print(frame)

cvae_pyramid.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_64_25p import config
+from configs.iccv.config_abl_pyr_32_64_128_256_512 import config
 
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 import tensorflow as tf
@@ -730,12 +730,12 @@ def generate_results():
 @tf.function
 def speed_step(levels_input, conds):
 
-    return model.sample([levels_input, conds], training=False)
+    return model.sample([levels_input, conds])
 
 
 
 if __name__ == '__main__':
-    init_wandb()
+    # init_wandb()
     parser = argparse.ArgumentParser(description='')
     parser.add_argument('--name', type=str, default='lhunor')
     args = vars(parser.parse_args())
@@ -846,52 +846,52 @@ if __name__ == '__main__':
     for batch in dataset_train:
         batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic']))
 
-        # levels_input, conds = group_inputs_for_levels(batch, config)
+        levels_input, conds = group_inputs_for_levels(batch, config)
 
-        # start = time.perf_counter()
+        start = time.perf_counter()
-        # out = speed_step(levels_input, conds)
+        out = speed_step(levels_input, conds)
-        # t = time.perf_counter() - start
+        t = time.perf_counter() - start
-        # print(t)
+        print(t)
-        # times.append(t)
+        times.append(t)
 
-        # if step > 200:
+        if step > 200:
-        #     break
+            break
 
 
-        train_loss = train_step(batch)
+        # train_loss = train_step(batch)
-        train_accumulator.update(train_loss)
+        # train_accumulator.update(train_loss)
 
-        if step % 25 == 0:
+        # if step % 25 == 0:
-            data_valid = valid_iterator.get_next()
+        #     data_valid = valid_iterator.get_next()
-            data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
+        #     data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
-            valid_loss = valid_step(data_valid)
+        #     valid_loss = valid_step(data_valid)
-            valid_accumulator.update(valid_loss)
+        #     valid_accumulator.update(valid_loss)
 
-        if train_loss['train/euclid_error'] < 0.015:
+        # if train_loss['train/euclid_error'] < 0.015:
-            if balancing_activated_step == 0:
+        #     if balancing_activated_step == 0:
-                balancing_activated_step = step
+        #         balancing_activated_step = step
-            balancing_coeff = balancing_coeff + 1 / 20000
+        #     balancing_coeff = balancing_coeff + 1 / 20000
 
-        if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
+        # if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
-            to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
+        #     to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
-            if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
+        #     if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
-                train_figs = plot_everything(batch, 'train')
+        #         train_figs = plot_everything(batch, 'train')
-                valid_figs = plot_everything(data_valid, 'valid')
+        #         valid_figs = plot_everything(data_valid, 'valid')
-                to_log = { **to_log, **train_figs, **valid_figs }
+        #         to_log = { **to_log, **train_figs, **valid_figs }
-            if step == 0:
+        #     if step == 0:
-                to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
+        #         to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
 
-            to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
+        #     to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
                 
-            wandb.log(to_log, step=step)
+        #     wandb.log(to_log, step=step)
-            train_accumulator.reset()
+        #     train_accumulator.reset()
-            valid_accumulator.reset()
+        #     valid_accumulator.reset()
 
-        if step % 5000 == 0:
+        # if step % 5000 == 0:
-            manager.save(checkpoint_number=step)
+        #     manager.save(checkpoint_number=step)
 
-        print(step)
+        # print(step)
 
         step += 1
-    # print('mean:')
+    print('mean:')
-    # print(np.mean(times[1:]))
+    print(np.mean(times[10:]))

evaluate_ae_pyramid.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_64_128_256_512_curv import config
+from configs.iccv.config_abl_pyr_32_64_10p import config
 
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 import tensorflow as tf

get_error.py

@@ -7,9 +7,9 @@ if __name__ == '__main__':
     garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item()
     folder = '/home/workspace/data/results/errors/iccv'
 
-    checkpoint_name = 'iccv_pyr_32_64_128_256_512'
+    checkpoint_name = 'iccv_pyr_32_64_25p'
     checkpoint_name_skirt = 'iccv_pyr_32_64_128_256_512_skirt'
-    iteration = '120000'
+    iteration = '155000'
     iteration_skirt : str = '115000'
 
     # checkpoint_name = 'abl_skin'

models/pyramid_level.py

@@ -101,6 +101,7 @@ class PyramidLevelNew(tf.keras.Model):
     def __init__(self, config):
         super(PyramidLevelNew, self).__init__()
         self.size = config['resolution']
+        self.latent_dim=config['latent_dim']
 
         self.encoder = VariationalEncoderNew(config['encoder_depths'], config['encoder_dims'], kernel_size=config['kernel_size'], latent_dim=config['latent_dim'])
         self.decoder = DecoderNew(config['decoder_depths'], config['decoder_dims'], config['kernel_size'])
@@ -126,13 +127,12 @@ class PyramidLevelNew(tf.keras.Model):
 
     def sample(self, z_cond, uv_mask, noise=None):
         if noise is None:
-            eps = tf.random.normal(shape=(uv_mask.shape[0], self.latent_dim))
+            eps = tf.random.normal(shape=(uv_mask.shape[0], 8, 4, self.latent_dim))
         else:
             eps = noise
 
         z = tf.concat([eps, z_cond], axis=-1)
-
+        x = self.decoder(z, training=False)
-        x = self.decoder(z)
         
         if self.avg_garment is not None:
             x = x + self.avg_garment[tf.newaxis,:,:,:]

models/pyramid_model_optimized.py

@@ -320,6 +320,87 @@ class PyramidNew(tf.keras.Model):
 
         return z_cond
 
+    def sample(self, inputs):
+        levels_input, conds = inputs
+        uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2]
+        z_cond = self.encode_conds(uv_static, uv_body, uv_curv, training=False)
+    
+        pred = [None, None, None, None, None]
+        for i in range(len(self.levels)):
+            pred[i] = self.levels[i].sample(z_cond, levels_input[i][-2])
+        
+        pred_1 = pred[0]
+        pred_2 = pred[1]
+        pred_3 = pred[2]
+        pred_4 = pred[3]
+        pred_5 = pred[4]
+
+        pred_1 = self.levels[0].enhance(pred_1, levels_input[0][-2], training=False)
+        pred_1 = tf.image.resize(pred_1, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_1 = pred_1 * levels_input[1][-2][:,:,:,tf.newaxis]
+        
+        pred_2 = pred_1 + pred_2
+        pred_2 = self.levels[1].enhance(pred_2, levels_input[1][-2], training=False)
+        pred_2 = tf.image.resize(pred_2, (self.levels[2].size, self.levels[2].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_2 = pred_2 * levels_input[2][-2][:,:,:,tf.newaxis]
+
+        pred_3 = pred_2 + pred_3
+        pred_3 = self.levels[2].enhance(pred_3, levels_input[2][-2], training=False)
+        pred_3 = tf.image.resize(pred_3, (self.levels[3].size, self.levels[3].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_3 = pred_3 * levels_input[3][-2][:,:,:,tf.newaxis]
+
+        pred_4 = pred_3 + pred_4
+        pred_4 = self.levels[3].enhance(pred_4, levels_input[3][-2], training=False)
+        pred_4 = tf.image.resize(pred_4, (self.levels[4].size, self.levels[4].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_4 = pred_4 * levels_input[4][-2][:,:,:,tf.newaxis]
+
+        pred_5 = pred_4 + pred_5
+        pred_5 = pred_5 * levels_input[4][-2][:,:,:,tf.newaxis]
+
+        return pred_5
+
+
+    def sample_seq(self, inputs):
+        levels_input, conds = inputs
+        uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2]
+        z_cond = self.encode_conds(uv_static, uv_body, uv_curv, training=False)
+    
+        # level 1
+        pred_1 = self.levels[0].sample(z_cond, levels_input[0][-2])
+        pred_1 = self.levels[0].enhance(pred_1, levels_input[0][-2], training=False)
+        pred_1 = tf.image.resize(pred_1, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_1 = pred_1 * levels_input[1][-2][:,:,:,tf.newaxis]
+
+        # level 2
+        pred_2 = self.levels[1].sample(z_cond, levels_input[1][-2])
+        pred_2 = pred_1 + pred_2
+        pred_2 = self.levels[1].enhance(pred_2, levels_input[1][-2], training=False)
+        pred_2 = tf.image.resize(pred_2, (self.levels[2].size, self.levels[2].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_2 = pred_2 * levels_input[2][-2][:,:,:,tf.newaxis]
+
+        # level 3
+        pred_3 = self.levels[2].sample(z_cond, levels_input[2][-2])
+        pred_3 = pred_2 + pred_3
+        pred_3 = self.levels[2].enhance(pred_3, levels_input[2][-2], training=False)
+        pred_3 = tf.image.resize(pred_3, (self.levels[3].size, self.levels[3].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_3 = pred_3 * levels_input[3][-2][:,:,:,tf.newaxis]
+
+        # level 4
+        pred_4 = self.levels[3].sample(z_cond, levels_input[3][-2])
+        pred_4 = pred_3 + pred_4
+        pred_4 = self.levels[3].enhance(pred_4, levels_input[3][-2], training=False)
+        pred_4 = tf.image.resize(pred_4, (self.levels[4].size, self.levels[4].size//2), tf.image.ResizeMethod.BILINEAR)
+        pred_4 = pred_4 * levels_input[4][-2][:,:,:,tf.newaxis]
+
+        # level 5
+        pred_5 = self.levels[4].sample(z_cond, levels_input[4][-2])
+        pred_5 = pred_5 * levels_input[4][-2][:,:,:,tf.newaxis]
+
+        pred_5 = pred_4 + pred_5
+
+        return pred_5
+    
+
     def call(self, inputs, training):
         levels_input, conds = inputs
         uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2]

plot_ablation.py

@@ -1,5 +1,5 @@
 import os
-os.environ["CUDA_VISIBLE_DEVICES"] = "6"
+os.environ["CUDA_VISIBLE_DEVICES"] = "1"
 import sys
 import time
 import cv2
@@ -128,7 +128,11 @@ def get_experiment_plots(folder, frames, experiments, skirt_mods, bodies, faces,
         frame_out = []
         id = frame.split('_')[0]
         for exp in experiments:
-            garment = np.load(os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy'))
+            path = os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy')
+            if not os.path.exists(path):
+                path = os.path.join('/home/workspace/local-data_128/results/iccv', exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy')
+                assert os.path.exists(path), path
+            garment = np.load(path)
             if exp == 'abl_orient':
                 garment = deorient(garment, frame, skirt_mods[id] == '_skirt')
             garment = postprocess(garment, bodies[frame], id, skirt_mods[id] == '_skirt')
@@ -190,11 +194,11 @@ if __name__ == '__main__':
 
     faces = get_faces()
 
-    frames = ['00079_135']
+    frames = ['00079_135', '00505_23', '00688_25', '01693_128', '00408_34']
     # frames = [ '00075_4', '00475_273', '00761_295', '00793_3', '00993_220', '01134_112', '01278_108', '00718_38', '00951_34', '00962_60','01065_38','01191_83','01240_168']
-    # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')]
+    rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')]
-    # random.shuffle(rand_frames)
+    random.shuffle(rand_frames)
-    # rand_frames = rand_frames[:25]
+    rand_frames = rand_frames[:25]
     # frames.extend(rand_frames)
     # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape')]
     # random.shuffle(rand_frames)
@@ -207,34 +211,38 @@ if __name__ == '__main__':
     cameras = {frame: camera_1 for frame in frames}
 
     # abl_all replace abl_norm maybe
-    experiments = [['iccv_pyr_32', '335000'], ['iccv_pyr_32_skin', '330000'], ['iccv_pyr_32_curv', '155000'], ['iccv_pyr_32_reg', '150000']] # , 'pyra_32_res'
+    # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_256_512_curv', '80000'], ['iccv_pyr_32_64_128_256_512_skin', '115000'], ['iccv_pyr_32_64_128_256_512_reg', '75000']] # , 'pyra_32_res'
+    # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_256_curv', '80000'], ['iccv_pyr_32_64_128_256_skin', '145000'], ['iccv_pyr_32_64_128_256_reg', '35000']] # , 'pyra_32_res'
+    # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_curv', '20000'], ['iccv_pyr_32_64_128_skin_2', '20000'], ['iccv_pyr_32_64_128_reg_2', '80000']] # , 'pyra_32_res'
+    experiments = [['abl_all_3', '130000']] # , 'pyra_32_res'
     skirt_mods = { frame.split('_')[0]: ('_skirt' if os.path.exists('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape/' + frame + '.npy') else '') for frame in frames }
 
     bodies = get_bodies(frames, skirt_mods)
-    # gts = get_gt_image(frames, bodies, faces, cameras, skirt_mods)
-    gts = get_gt_meshes(frames, bodies, faces, cameras, skirt_mods)
 
     results_folder_base = '/home/workspace/data/results/iccv/'
 
-    output_folder = os.path.join(results_folder_base, 'ablation_plots')
+    output_folder = os.path.join('examples/ablation/img_3')
+    output_folder_stl = os.path.join('examples/ablation/stl_3')
     os.makedirs(output_folder, exist_ok=True)
+    os.makedirs(output_folder_stl, exist_ok=True)
 
+    # gts_img = get_gt_image(frames, bodies, faces, cameras, skirt_mods)
     # images = get_experiment_plots(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras)
+    # rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts_img, images)]
+    # for frame, row in zip(frames, rows):
+    #     custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png'))
+
+    gts = get_gt_meshes(frames, bodies, faces, cameras, skirt_mods)
     meshes = get_experiment_meshes(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras)
     for mesh, frame in zip(meshes, frames):
         for exp, (garment, body, faces) in zip(experiments, mesh):
-            save_stl(f'examples/{frame}_{exp[0]}_garment.stl', garment, faces)
+            save_stl(f'{output_folder_stl}/{frame}_{exp[0]}_garment.stl', garment, faces)
-            save_stl(f'examples/{frame}_{exp[0]}_body.stl', body, F)
+            save_stl(f'{output_folder_stl}/{frame}_{exp[0]}_body.stl', body, F)
-
     for mesh, frame in zip(gts, frames):
         garment, body, faces = mesh
-        save_stl(f'examples/{frame}_gt_garment.stl', garment, faces)
+        save_stl(f'{output_folder_stl}/{frame}_gt_garment.stl', garment, faces)
-        save_stl(f'examples/{frame}_gt_body.stl', body, F)
+        save_stl(f'{output_folder_stl}/{frame}_gt_body.stl', body, F)
 
-    # rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts, images)]
-
-    # for frame, row in zip(frames, rows):
-    #     custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png'))
 
     # final = np.concatenate(rows, axis=0)