save

aaa_helpers.txt

@@ -127,12 +127,14 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skin" > nohup
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv_skirt.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_reg.out 2>&1 &
+nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_seq_2_fine" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_seq_2_fine.out 2>&1 &
 
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_skirt.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_skin.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_curv.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_reg.out 2>&1 &
+nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_fine" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_fine.out 2>&1 &
 
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32 > nohup/iccv/evaluate/iccv_pyr_32.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_skirt > nohup/iccv/evaluate/iccv_pyr_32_skirt.out 2>&1 &
@@ -164,6 +166,7 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv >
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_curv_skirt.out 2>&1 &
 
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256 > nohup/iccv/evaluate/iccv_pyr_32_64_128_256.out 2>&1 &
+nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_seq > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_seq.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_skirt.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_skin > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_skin.out 2>&1 &
 nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_curv > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_curv.out 2>&1 &

calculate_error.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config
+from configs.iccv.config_abl_pyr_32_64_128_seq import config
 
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 import tensorflow as tf
@@ -58,7 +58,7 @@ if __name__ == '__main__':
     # smpl = SMPL(use_one_hot=config['use_one_hot'])
     skirt_modifier = '_skirt' if config['is_skirt'] else ''
 
-    error_type = 'collision' # 'euclidean' 'collision' edge normal
+    error_type = 'euclidean' # 'euclidean' 'collision' edge normal
 
     folder = config['test_dir']
 
@@ -92,9 +92,9 @@ if __name__ == '__main__':
     frames = [frame for frame in frames if not frame.endswith('_fixed_coll.npy')]
     # frames = frames[:100]
 
-    print('Reading bodies...')
+    # print('Reading bodies...')
-    bodies_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_posed_body_no_orient'
+    # bodies_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_posed_body_no_orient'
-    bodies = {frame : np.load(bodies_folder + '/' + frame[:-1] + 'z')['mesh_posed_body_no_orient'] for frame in tqdm(frames) }
+    # bodies = {frame : np.load(bodies_folder + '/' + frame[:-1] + 'z')['mesh_posed_body_no_orient'] for frame in tqdm(frames) }
 
     if config['remove_global_orientation']:
         gt_name = 'mesh_posed_no_orient'

configs/iccv/config_abl_pyr_32_64_128_256_seq.py

@@ -0,0 +1,187 @@
+config = dict()
+
+config['gpu'] = '1'
+config['name'] = 'iccv_pyr_32_64_128_256_seq_2_fine'
+config['notes'] = ''
+config['tags'] = ['pyr', 'iccv']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              True # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             True
+
+config['pretrained_model'] = 'checkpoints/iccv/iccv_pyr_32_64_128_256_seq_2'
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64, 128, 256]
+config['pyramid']['pretrained_pyramid'] = ''
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data/train'
+config['valid_dir'] = '/home/workspace/local-data/valid'
+config['test_dir'] = '/home/workspace/local-data/test_ready'
+
+config['batch_size'] = 4
+config['is_skirt'] = False
+config['is_cape'] = False
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.0001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'

configs/iccv/config_abl_pyr_32_64_128_seq.py

@@ -0,0 +1,187 @@
+config = dict()
+
+config['gpu'] = '1'
+config['name'] = 'iccv_pyr_32_64_128_seq_2'
+config['notes'] = ''
+config['tags'] = ['pyr', 'iccv']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              True # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             True
+
+config['pretrained_model'] = ''
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64, 128]
+config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_seq_2'
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data/train'
+config['valid_dir'] = '/home/workspace/local-data/valid'
+config['test_dir'] = '/home/workspace/local-data/test_ready'
+
+config['batch_size'] = 4
+config['is_skirt'] = False
+config['is_cape'] = False
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.0001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'

configs/iccv/config_abl_pyr_32_seq.py

@@ -0,0 +1,187 @@
+config = dict()
+
+config['gpu'] = '4'
+config['name'] = 'iccv_pyr_32_seq_2'
+config['notes'] = ''
+config['tags'] = ['pyr', 'iccv']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              True # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             True
+
+config['pretrained_model'] = ''
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64]
+config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32'
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data/train'
+config['valid_dir'] = '/home/workspace/local-data/valid'
+config['test_dir'] = '/home/workspace/local-data/test_ready'
+
+config['batch_size'] = 4
+config['is_skirt'] = False
+config['is_cape'] = False
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.0001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 3, 9, 3]
+    level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [64, 128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'

cvae_pyramid.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_tshirt import config
+from configs.iccv.config_abl_pyr_32_64_128_256_seq import config
 
 os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
@@ -221,20 +221,104 @@ def train_step(batch):
         out_levels = model([levels_input, conds], training=True)
 
         if config['use_skinning']:
-            mesh_orig, uv_orig = batch['mesh_unposed'], batch['uv_unposed']
+            mesh_orig = batch['mesh_unposed']
         else:
-            mesh_orig, uv_orig = batch['mesh_posed'], batch['uv_posed']
+            mesh_orig = batch['mesh_posed']
 
         output = dict()
         reconstr_loss_total = 0
         kl_total = 0
 
-        #for i, res in enumerate(config['pyramid']['resolutions']):
+        for i, res in enumerate(config['pyramid']['resolutions']):
-        i = -1
+        # i = -1
-        res = config['pyramid']['resolutions'][-1]
+            res = config['pyramid']['resolutions'][i] if i != len(config['pyramid']['resolutions']) - 1 else 512
+            res_modifier = '_' + str(res) if res != 512 else ''
+            pred, mean, logvar = out_levels[i]
+
+            if config['use_skinning']:
+                uv_orig = batch['uv_unposed' + res_modifier]
+            else:
+                uv_orig = batch['uv_posed' + res_modifier]
+
+            mesh_pred = uv2mesh(pred)
+
+            if config['use_skinning']:
+                shape = batch['pose_shape'][:, 72:]
+                gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
+                shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
+
+                mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
+                mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
+            else:
+                mesh_pred_posed = mesh_pred
+
+            # is the output already masked?
+
+            e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
+            c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
+            c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio)
+            l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1) * batch['uv_mask' + res_modifier], axis=[1, 2]))
+            # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
+            # ms_ssim = tf.reduce_mean(tf.image.ssim(pred + 1, batch['uv_unposed'] + 1, max_val=3))
+            l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
+            n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'])
+            kl = kl_loss(mean, logvar)
+
+            if config['use_extra_losses']:
+                reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (l2_loss + l2_mesh_loss)
+                if not config['is_cape']:
+                    reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (e_loss + c_loss) * config['reg_loss_coeff'] 
+            else:
+                reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (l2_loss)
+
+            if config['use_norm_loss'] and step > 2000:
+                reconstr_loss_total = reconstr_loss_total + n_loss * config['reg_loss_coeff']
+            
+            kl_total += kl
+
+            mean = tf.reduce_mean(mean)
+            logvar = tf.reduce_mean(logvar)
+
+            metric = tf.reduce_sum(tf.math.sqrt(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (mesh_orig - mesh_pred) ** 2, axis=-1))) / tf.reduce_sum(batch['mesh_mask'])
+
+            output = { **output, **{ 'train/edge_loss' + res_modifier: e_loss, 'train/collision_loss' + res_modifier: c_loss,'train/euclid_error' + res_modifier: metric, 
+                                    'train/mean' + res_modifier: mean, 'train/logvar' + res_modifier: logvar, 'train/kl_loss' + res_modifier: kl,
+                                    'train/var' + res_modifier: np.e ** logvar, 'train/l2_loss' + res_modifier: l2_loss, 'train/normal_loss' + res_modifier: n_loss,
+                                    'train/euclid_error' + res_modifier: metric, 'train/l2_mesh_loss' + res_modifier: l2_mesh_loss } } # 'train/ms_ssim' + res_modifier: ms_ssim,
+
+        loss = reconstr_loss_total + balancing_coeff * kl_total
+
+    gradients = tape.gradient(loss, model.trainable_variables)
+    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
+    return { **{ 'train/loss': loss, 'train/reconstr_loss_total': reconstr_loss_total, 'train/kl_total': kl_total }, **output }
+
+
+@tf.function
+def valid_step(batch):
+    levels_input, conds = group_inputs_for_levels(batch, config)
+    out_levels = model([levels_input, conds], training=False)
+
+
+    if config['use_skinning']:
+        mesh_orig = batch['mesh_unposed']
+    else:
+        mesh_orig = batch['mesh_posed']
+
+    output = dict()
+    reconstr_loss_total = 0
+    kl_total = 0
+
+    for i, res in enumerate(config['pyramid']['resolutions']):
+    # i = -1
+        res = config['pyramid']['resolutions'][i] if i != len(config['pyramid']['resolutions']) - 1 else 512
         res_modifier = '_' + str(res) if res != 512 else ''
         pred, mean, logvar = out_levels[i]
 
+        if config['use_skinning']:
+            uv_orig = batch['uv_unposed' + res_modifier]
+        else:
+            uv_orig = batch['uv_posed' + res_modifier]
+
         mesh_pred = uv2mesh(pred)
 
         if config['use_skinning']:
@@ -246,13 +330,12 @@ def train_step(batch):
             mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
         else:
             mesh_pred_posed = mesh_pred
-
         # is the output already masked?
 
         e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
         c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
         c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio)
-        l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2]))
+        l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1) * batch['uv_mask' + res_modifier], axis=[1, 2]))
         # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
         # ms_ssim = tf.reduce_mean(tf.image.ssim(pred + 1, batch['uv_unposed'] + 1, max_val=3))
         l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
@@ -276,83 +359,10 @@ def train_step(batch):
 
         metric = tf.reduce_sum(tf.math.sqrt(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (mesh_orig - mesh_pred) ** 2, axis=-1))) / tf.reduce_sum(batch['mesh_mask'])
 
-        output = { **output, **{ 'train/edge_loss' + res_modifier: e_loss, 'train/collision_loss' + res_modifier: c_loss,'train/euclid_error' + res_modifier: metric, 
+        output = { **output, **{ 'valid/edge_loss' + res_modifier: e_loss, 'valid/collision_loss' + res_modifier: c_loss,'valid/euclid_error' + res_modifier: metric, 
-                                 'train/mean' + res_modifier: mean, 'train/logvar' + res_modifier: logvar, 'train/kl_loss' + res_modifier: kl,
+                                    'valid/mean' + res_modifier: mean, 'valid/logvar' + res_modifier: logvar, 'valid/kl_loss' + res_modifier: kl,
-                                 'train/var' + res_modifier: np.e ** logvar, 'train/l2_loss' + res_modifier: l2_loss, 'train/normal_loss': n_loss,
+                                    'valid/var' + res_modifier: np.e ** logvar, 'valid/l2_loss' + res_modifier: l2_loss, 'valid/normal_loss' + res_modifier: n_loss,
-                                 'train/euclid_error': metric, 'train/l2_mesh_loss': l2_mesh_loss } } # 'train/ms_ssim' + res_modifier: ms_ssim,
+                                    'valid/euclid_error' + res_modifier: metric, 'valid/l2_mesh_loss' + res_modifier: l2_mesh_loss } } # 'valid/ms_ssim' + res_modifier: ms_ssim,
-
-        loss = reconstr_loss_total + balancing_coeff * kl_total
-
-    gradients = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
-    return { **{ 'train/loss': loss, 'train/reconstr_loss_total': reconstr_loss_total, 'train/kl_total': kl_total }, **output }
-
-
-@tf.function
-def valid_step(batch):
-    levels_input, conds = group_inputs_for_levels(batch, config)
-    out_levels = model([levels_input, conds], training=False)
-
-
-    if config['use_skinning']:
-        mesh_orig, uv_orig = batch['mesh_unposed'], batch['uv_unposed']
-    else:
-        mesh_orig, uv_orig = batch['mesh_posed'], batch['uv_posed']
-
-    output = dict()
-    reconstr_loss_total = 0
-    kl_total = 0
-
-    #for i, res in enumerate(config['pyramid']['resolutions']):
-    i = -1
-    res = config['pyramid']['resolutions'][-1]
-    res_modifier = '_' + str(res) if res != 512 else ''
-    pred, mean, logvar = out_levels[i]
-
-    mesh_pred = uv2mesh(pred)
-
-    if config['use_skinning']:
-        shape = batch['pose_shape'][:, 72:]
-        gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
-        shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
-
-        mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
-        mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
-    else:
-        mesh_pred_posed = mesh_pred
-    # is the output already masked?
-
-    e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
-    c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
-    c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio)
-    l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2]))
-    # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
-    # ms_ssim = tf.reduce_mean(tf.image.ssim(pred + 1, batch['uv_unposed'] + 1, max_val=3))
-    l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
-    n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'])
-    kl = kl_loss(mean, logvar)
-
-    if config['use_extra_losses']:
-        reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (l2_loss + l2_mesh_loss)
-        if not config['is_cape']:
-            reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (e_loss + c_loss) * config['reg_loss_coeff'] 
-    else:
-        reconstr_loss_total = reconstr_loss_total + config['pyramid']['loss_weights'][i] * (l2_loss)
-    
-    if config['use_norm_loss'] and step > 2000:
-        reconstr_loss_total = reconstr_loss_total + n_loss * config['reg_loss_coeff']
-        
-    kl_total += kl
-
-    mean = tf.reduce_mean(mean)
-    logvar = tf.reduce_mean(logvar)
-
-    metric = tf.reduce_sum(tf.math.sqrt(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (mesh_orig - mesh_pred) ** 2, axis=-1))) / tf.reduce_sum(batch['mesh_mask'])
-
-    output = { **output, **{ 'valid/edge_loss' + res_modifier: e_loss, 'valid/collision_loss' + res_modifier: c_loss,'valid/euclid_error' + res_modifier: metric, 
-                                 'valid/mean' + res_modifier: mean, 'valid/logvar' + res_modifier: logvar, 'valid/kl_loss' + res_modifier: kl,
-                                'valid/var' + res_modifier: np.e ** logvar, 'valid/l2_loss' + res_modifier: l2_loss, 'valid/normal_loss': n_loss,
-                                'valid/euclid_error': metric, 'valid/l2_mesh_loss': l2_mesh_loss } } # 'valid/ms_ssim' + res_modifier: ms_ssim,
 
     loss = reconstr_loss_total + balancing_coeff * kl_total
 
@@ -852,8 +862,8 @@ if __name__ == '__main__':
     #     load_model(model.upscale_enhancer, config['pyramid']['final_upscale_enhancer'])
     #     model.upscale_enhancer.trainable = False
 
-    for i in range(len(model.levels)-1):
+    # for i in range(len(model.levels)-1):
-        model.levels[i].trainable = False
+    #     model.levels[i].trainable = False
 
     # load_model(model.cond_encoder, config['pyramid']['pretrained_pyramid'])
     # load_model(model.levels[0], config['pyramid']['pretrained_pyramid'])

evaluate_ae_pyramid.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config
+from configs.iccv.config_abl_pyr_32_64_128_seq import config
 
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 import tensorflow as tf

get_error.py

@@ -8,11 +8,11 @@ if __name__ == '__main__':
     garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item()
     folder = '/home/workspace/data/results/errors/iccv'
 
-    checkpoint_name = 'iccv_pyr_32_64_128_skin'
+    checkpoint_name = 'iccv_pyr_32_64_128_256_seq_2'
     checkpoint_name_skirt = 'iccv_pyr_32_64_128_skin_skirt'
-    iteration = '75' + '000'
+    iteration = '185' + '000'
     iteration_skirt : str = '155' + '000'
-    error_type = 'collision' # 'euclidean' 'collision' edge normal
+    error_type = 'euclidean' # 'euclidean' 'collision' edge normal
 
 
     errors = []
@@ -21,31 +21,31 @@ if __name__ == '__main__':
     per_garment_errors_skirt  = { 'Tshirt': [], 'Dress': [], 'Skirt': [], 'Top': [] }
 
 
-    # with open(os.path.join(folder, f'{checkpoint_name}-{iteration}-{error_type}.csv'), 'r') as f:
+    with open(os.path.join(folder, f'{checkpoint_name}-{iteration}-{error_type}.csv'), 'r') as f:
-    #     csv_reader = csv.reader(f, delimiter=',')
-    #     for row in csv_reader:
-    #         per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1]))
-
-    # print('\nNon-skirt:')
-    # for key in per_garment_errors.keys():
-    #     avg = np.mean(per_garment_errors[key])
-    #     errors.append(avg)
-    #     print(key + ': ' + str(avg))
-
-    # print('\nAvg non-skirt: ' + str(np.mean(errors)))
-
-    with open(os.path.join(folder, f'{checkpoint_name_skirt}-{iteration_skirt}-{error_type}.csv'), 'r') as f:
         csv_reader = csv.reader(f, delimiter=',')
         for row in csv_reader:
-            per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1]))
+            per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1]))
 
-    print('\nSkirt:')
+    print('\nNon-skirt:')
-    for key in per_garment_errors_skirt.keys():
+    for key in per_garment_errors.keys():
-        avg = np.mean(per_garment_errors_skirt[key])
+        avg = np.mean(per_garment_errors[key])
-        errors_skirt.append(avg)
+        errors.append(avg)
         print(key + ': ' + str(avg))
 
-    print('Avg skirt: ' + str(np.mean(errors_skirt)))
+    print('\nAvg non-skirt: ' + str(np.mean(errors)))
+
+    # with open(os.path.join(folder, f'{checkpoint_name_skirt}-{iteration_skirt}-{error_type}.csv'), 'r') as f:
+    #     csv_reader = csv.reader(f, delimiter=',')
+    #     for row in csv_reader:
+    #         per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1]))
+
+    # print('\nSkirt:')
+    # for key in per_garment_errors_skirt.keys():
+    #     avg = np.mean(per_garment_errors_skirt[key])
+    #     errors_skirt.append(avg)
+    #     print(key + ': ' + str(avg))
+
+    # print('Avg skirt: ' + str(np.mean(errors_skirt)))
 
     # cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt'])
     # cum_avg_top = np.mean(per_garment_errors['Top'] + per_garment_errors_skirt['Top'])

models/pyramid_level.py

@@ -155,15 +155,18 @@ class PyramidLevelNew(tf.keras.Model):
         return out
 
 
-    def call(self, inputs, z_cond, training=None):
+    def call(self, inputs, z_cond, prev_latent=None, training=None):
         uv_unposed, uv_template, uv_body_posed, uv_curv, uv_mask, uv_mask_high = inputs
 
         x = tf.concat([uv_unposed, uv_template, uv_body_posed, uv_curv], axis=-1)
         mean, logvar = self.encoder(x, training=training)
         z = self.reparameterize(mean, logvar, training)
-        z = self.reshape(z)
+        z_latent = self.reshape(z)
     
-        z = tf.concat([z, z_cond], axis=-1)
+        if prev_latent is not None:
+            z_latent = tf.concat([z_latent, prev_latent], axis=-1)
+            
+        z = tf.concat([z_latent, z_cond], axis=-1)
         x = self.decoder(z, training=training)
 
         if self.avg_garment is not None:
@@ -177,4 +180,4 @@ class PyramidLevelNew(tf.keras.Model):
         
         # x = x * uv_mask_high[:,:,:,tf.newaxis]
 
-        return x, mean, logvar
+        return x, mean, logvar, z_latent

models/pyramid_model_optimized.py

@@ -12,244 +12,6 @@ from models.pyramid_level import PyramidLevel, PyramidLevelNew
 from models.upscaler import UpscalerBasic, UpscalerBasicConv, UpscalerBasicNew
 
 
-class Pyramid(tf.keras.Model):
-    
-    def __init__(self, config):
-        super(Pyramid, self).__init__()
-
-        self.static_enc : AEEncoder = AEEncoder(config['cond_ae']['latent_dim'], config['cond_ae']['encoder_depths'], config['cond_ae']['encoder_dims'], config['cond_ae']['bottleneck_size'], config['cond_ae']['kernel_size'])
-        if config['pyramid']['static_model'] != '':
-            load_model(self.static_enc, config['pyramid']['static_model'] + '_enc')
-
-        self.body_enc : AEEncoder = AEEncoder(config['cond_ae_body']['latent_dim'], config['cond_ae_body']['encoder_depths'], config['cond_ae_body']['encoder_dims'], config['cond_ae_body']['bottleneck_size'], config['cond_ae_body']['kernel_size'])
-        if config['pyramid']['body_model'] != '':
-            load_model(self.body_enc, config['pyramid']['body_model'] + '_enc')
-
-        self.curv_enc : AEEncoder = AEEncoder(config['cond_ae']['latent_dim'], config['cond_ae']['encoder_depths'], config['cond_ae']['encoder_dims'], config['cond_ae']['bottleneck_size'], config['cond_ae']['kernel_size'])
-        if config['pyramid']['curvature_model'] != '':
-            load_model(self.curv_enc, config['pyramid']['curvature_model'] + '_enc')
-
-        if config['pyramid']['freeze_encoders']:
-            self.static_enc.trainable = False
-            self.body_enc.trainable = False
-            self.curv_enc.trainable = False
-
-        self.cond_encoder = tf.keras.Sequential([tf.keras.layers.Dense(512, activation='relu'),
-                                                 tf.keras.layers.Dense(256, activation='relu'),
-                                                 tf.keras.layers.Dense(128, activation='relu'),
-                                                 tf.keras.layers.Dense(config['pyramid']['latent_dim'], activation='tanh'),])
-
-        self.levels = [PyramidLevel({ **config['pyramid']['levels'][i], **{ 'cond_ae': config['cond_ae']} }) for i in range(len(config['pyramid']['levels']))]
-
-        # if config['is_cape']:
-        #     if len(config['pyramid']['resolutions']) == 2:
-        #         self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        #     if len(config['pyramid']['resolutions']) == 3:
-        #         self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        #         self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        #     if len(config['pyramid']['resolutions']) == 4:
-        #         self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        #         self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        #         self.levels[-3].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        # else:
-        if len(config['pyramid']['resolutions']) == 2:
-            self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        if len(config['pyramid']['resolutions']) == 3:
-            self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-            self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        if len(config['pyramid']['resolutions']) == 4:
-            self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 10000)])
-            self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-            self.levels[-3].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        if len(config['pyramid']['resolutions']) == 5:
-            self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100000)])
-            self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 10000)])
-            self.levels[-3].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-            self.levels[-4].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-
-        self.add_noise = config['pyramid']['add_noise']
-        self.noise_amount = config['pyramid']['noise_amount']
-        self.use_offsets = config['use_offsets']
- 
-        self.upscale_enhancer = None
-        if config['pyramid']['final_upscale_enhancer'] != '':
-            k_size = config['pyramid']['final_upscale_enhancer_kernel_size'] if 'final_upscale_enhancer_kernel_size' in config['pyramid'].keys() else ((3 if config['pyramid']['resolutions'][-1] <= 64 else 5))
-            self.upscale_enhancer = UpscalerBasicConv(low_res=config['pyramid']['resolutions'][-1], kernel_size=k_size)
-            load_model(self.upscale_enhancer, config['pyramid']['final_upscale_enhancer'])
-            self.upscale_enhancer.trainable = False
-
-    def enhance(self, low, mask, training):
-        x = self.upscale_enhancer([low, mask], training=training)
-        return x
-
-    def encode_conds(self, uv_template, uv_body_posed, uv_curv, training):
-        z_static = self.static_enc(uv_template, training)
-        z_body = self.body_enc(uv_body_posed, training)
-        z_curv = self.curv_enc(uv_curv, training)
-
-        z_cond = tf.concat([z_static, z_body, z_curv], axis=-1)
-        z_cond = self.cond_encoder(z_cond, training=training)
-
-        return z_cond
-
-    def call(self, inputs, training):
-        levels_input, conds = inputs
-        uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2]
-
-        z_cond = self.encode_conds(uv_static, uv_body, uv_curv, training)
-
-        if self.add_noise == True and training == True:
-            z_cond = z_cond + (tf.random.uniform(z_cond.shape, maxval=self.noise_amount * 2) - self.noise_amount)
-
-        out_levels = []
-        pred, mean, logvar = self.levels[0](levels_input[0], z_cond, training=training)
-        if self.use_offsets:
-            pred = pred + levels_input[0][2] # uv_body_posed_32
-        if 1 != len(self.levels):
-            pred = self.levels[0].enhance(pred, levels_input[0][-2], training=training)  # levels_input[i][-2]=uv_mask_res
-            pred = tf.image.resize(pred, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.LANCZOS5)
-            out_levels.append([pred, mean, logvar])
-            pred = pred * levels_input[1][-2][:,:,:,tf.newaxis]
-        else:
-            pred = self.enhance(pred, levels_input[0][-2], training=training)
-            pred = tf.image.resize(pred, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-            pred = pred * levels_input[-1][-1][:,:,:,tf.newaxis] # full res uv mask
-            out_levels.append([pred, mean, logvar])
-
-        for i in range(1, len(self.levels)):
-            pred_level, mean, logvar = self.levels[i](levels_input[i], z_cond, training=training)
-            pred = pred + pred_level
-            if i != len(self.levels) - 1:
-                pred = self.levels[i].enhance(pred, levels_input[i][-2], training=training)
-                out_levels.append([pred, mean, logvar])
-                pred = tf.image.resize(pred, (self.levels[i+1].size, self.levels[i+1].size//2), tf.image.ResizeMethod.LANCZOS5)
-                pred = pred * levels_input[i+1][-2][:,:,:,tf.newaxis]
-            else:
-                if self.upscale_enhancer is not None:
-                    pred = self.enhance(pred, levels_input[-1][-2], training=training)
-                    pred = tf.image.resize(pred, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-                pred = pred * levels_input[-1][-1][:,:,:,tf.newaxis] # full res uv mask
-                out_levels.append([pred, mean, logvar])
-
-        return out_levels
-
-        # # 32
-        # pred_0, mean_0, logvar_0 = self.levels[0](levels_input[0], z_cond, training=training)
-        # pred_0 = self.enhance(pred_0, levels_input[0][-2], training=training)
-        # pred_0 = tf.image.resize(pred_0, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-        # pred_0 = pred_0 * levels_input[-1][-1][:,:,:,tf.newaxis] # full res uv mask
-        # return [ [pred_0, mean_0, logvar_0] ]
-
-        # # 32 64
-        # pred_0, mean_0, logvar_0 = self.levels[0](levels_input[0], z_cond, training=training)
-        # pred_0 = self.levels[0].enhance(pred_0, levels_input[0][-2], training=training)  # levels_input[i][-2]=uv_mask_res
-        # pred_0 = tf.image.resize(pred_0, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.LANCZOS5)
-        # pred = pred_0 * levels_input[1][-2][:,:,:,tf.newaxis]
-
-        # pred_1, mean_1, logvar_1 = self.levels[1](levels_input[1], z_cond, training=training)
-        # pred_1 = pred + pred_1
-        # pred_1 = self.enhance(pred_1, levels_input[-1][-2], training=training)
-        # pred_1 = tf.image.resize(pred_1, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-        # pred_1 = pred_1 * levels_input[-1][-1][:,:,:,tf.newaxis] # full res uv mask
-
-        # return [ [pred_0, mean_0, logvar_0], [pred_1, mean_1, logvar_1] ]
-
-
-# class Pyramid(tf.keras.Model):
-    
-#     def __init__(self, config):
-#         super(Pyramid, self).__init__()
-
-#         self.static_enc : AEEncoder = AEEncoder(config['cond_ae']['latent_dim'], config['cond_ae']['encoder_depths'], config['cond_ae']['encoder_dims'], config['cond_ae']['bottleneck_size'], config['cond_ae']['kernel_size'])
-#         if config['pyramid']['static_model'] != '':
-#             load_model(self.static_enc, config['pyramid']['static_model'] + '_enc')
-
-#         self.body_enc : AEEncoder = AEEncoder(config['cond_ae']['latent_dim'], config['cond_ae']['encoder_depths'], config['cond_ae']['encoder_dims'], config['cond_ae']['bottleneck_size'], config['cond_ae']['kernel_size'])
-#         if config['pyramid']['body_model'] != '':
-#             load_model(self.body_enc, config['pyramid']['body_model'] + '_enc')
-
-#         self.curv_enc : AEEncoder = AEEncoder(config['cond_ae']['latent_dim'], config['cond_ae']['encoder_depths'], config['cond_ae']['encoder_dims'], config['cond_ae']['bottleneck_size'], config['cond_ae']['kernel_size'])
-#         if config['pyramid']['curvature_model'] != '':
-#             load_model(self.curv_enc, config['pyramid']['curvature_model'] + '_enc')
-
-#         if config['pyramid']['freeze_encoders']:
-#             self.static_enc.trainable = False
-#             self.body_enc.trainable = False
-#             self.curv_enc.trainable = False
-
-#         self.cond_encoder = tf.keras.Sequential([tf.keras.layers.Dense(512, activation='relu'),
-#                                                  tf.keras.layers.Dense(256, activation='relu'),
-#                                                  tf.keras.layers.Dense(128, activation='relu'),
-#                                                  tf.keras.layers.Dense(config['pyramid']['latent_dim']),])
-
-#         self.levels = [PyramidLevel({ **config['pyramid']['levels'][i], **{ 'cond_ae': config['cond_ae']} }) for i in range(len(config['pyramid']['levels']))]
-
- 
-#         self.upscale_enhancer = None
-#         if config['pyramid']['final_upscale_enhancer'] != '':
-#             self.upscale_enhancer = UpscalerBasicNew(low_res=config['pyramid']['resolutions'][-1])
-#             load_model(self.upscale_enhancer, config['pyramid']['final_upscale_enhancer'])
-#             self.upscale_enhancer.trainable = False
-
-#     def enhance(self, low, mask, training):
-#         x = self.upscale_enhancer([low, mask], training=training)
-#         return x
-
-#     def encode_conds(self, uv_template, uv_body_posed, uv_curv):
-#         z_static = self.static_enc(uv_template)
-#         z_body = self.body_enc(uv_body_posed)
-#         z_curv = self.curv_enc(uv_curv)
-
-#         z_cond = tf.concat([z_static, z_body, z_curv], axis=-1)
-#         z_cond = self.cond_encoder(z_cond)
-
-#         return z_cond
-
-#     def call(self, inputs, training):
-#         levels_input, conds = inputs
-#         uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2]
-#         z_cond = self.encode_conds(uv_static, uv_body, uv_curv)
-
-#         out_levels = []
-#         pred_0, mean_0, logvar_0 = self.levels[0](levels_input[0], z_cond, training=training)
-#         pred_0 = self.levels[0].enhance(pred_0, levels_input[0][-2], training=training)  # levels_input[i][-2]=uv_mask_res
-#         # pred_0 = self.enhance(pred_0, levels_input[0][-2], training=training)
-            
-#         ##################    
-#         # if more than 1 level
-#         # x = tf.image.resize(pred_0, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.LANCZOS5)
-#         # x = x * levels_input[1][-2][:,:,:,tf.newaxis]
-
-#         # if only 1 level
-#         pred_0 = tf.image.resize(pred_0, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-#         ##################
-
-
-#         # pred_1, mean_1, logvar_1 = self.levels[1](levels_input[1], z_cond, training=training)
-#         # pred_1 = pred_0 + pred_1
-#         # pred_1 = self.levels[1].enhance(pred, levels_input[1][-2], training=training)
-
-#         # for i in range(1, len(self.levels)):
-#         #     pred_level, mean, logvar = self.levels[i](levels_input[i], z_cond, training=training)
-#         #     pred = pred + pred_level
-#         #     if i != len(self.levels) - 1:
-#         #         pred = self.levels[i].enhance(pred, levels_input[i][-2], training=training)
-#         #         out_levels.append([pred, mean, logvar])
-#         #         pred = tf.image.resize(pred, (self.levels[i+1].size, self.levels[i+1].size//2), tf.image.ResizeMethod.LANCZOS5)
-#         #         pred = pred * levels_input[i+1][-2][:,:,:,tf.newaxis]
-#         #     else:
-#         #         if self.upscale_enhancer is not None:
-#         #             pred = self.enhance(pred, levels_input[-1][-2], training=training)
-#         #             pred = tf.image.resize(pred, (512, 256), tf.image.ResizeMethod.LANCZOS5)
-#         #         out_levels.append([pred, mean, logvar])
-#         #         pred = pred * levels_input[-1][-1][:,:,:,tf.newaxis] # full res uv mask
-
-#         out_levels = [
-#             [pred_0, mean_0, logvar_0]
-#         ]
-
-#         return out_levels
-
 
 class PyramidNew(tf.keras.Model):
     
@@ -278,21 +40,6 @@ class PyramidNew(tf.keras.Model):
         self.levels = [PyramidLevelNew({ **config['pyramid']['levels'][i] }) for i in range(len(config['pyramid']['levels']))]
 
 
-        # if len(config['pyramid']['resolutions']) == 2:
-        #     self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        # if len(config['pyramid']['resolutions']) == 3:
-        #     self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-        #     self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        # if len(config['pyramid']['resolutions']) == 4:
-        #     self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 10000)])
-        #     self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-        #     self.levels[-3].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-        # if len(config['pyramid']['resolutions']) == 5:
-        #     self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100000)])
-        #     self.levels[-2].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 10000)])
-        #     self.levels[-3].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 1000)])
-        #     self.levels[-4].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)])
-
         self.add_noise = config['pyramid']['add_noise']
         self.noise_amount = config['pyramid']['noise_amount']
         self.use_offsets = config['use_offsets']
@@ -411,13 +158,13 @@ class PyramidNew(tf.keras.Model):
             z_cond = z_cond + (tf.random.uniform(z_cond.shape, maxval=self.noise_amount * 2) - self.noise_amount)
 
         out_levels = []
-        pred, mean, logvar = self.levels[0](levels_input[0], z_cond, training=training)
+        pred, mean, logvar, z_latent = self.levels[0](levels_input[0], z_cond, training=training)
         if self.use_offsets:
             pred = pred + levels_input[0][2] # uv_body_posed_32
         if 1 != len(self.levels):
             pred = self.levels[0].enhance(pred, levels_input[0][-2], training=training)  # levels_input[i][-2]=uv_mask_res
-            pred = tf.image.resize(pred, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
             out_levels.append([pred, mean, logvar])
+            pred = tf.image.resize(pred, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
             pred = pred * levels_input[1][-2][:,:,:,tf.newaxis]
         else:
             # pred = self.enhance(pred, levels_input[0][-2], training=training)
@@ -429,7 +176,7 @@ class PyramidNew(tf.keras.Model):
         for i in range(1, len(self.levels)):
             input = levels_input[i]
             input[0] = input[0] - pred
-            pred_level, mean, logvar = self.levels[i](input, z_cond, training=training)
+            pred_level, mean, logvar, z_latent = self.levels[i](input, z_cond, z_latent, training=training)
             pred = pred + pred_level
             if i != len(self.levels) - 1:
                 pred = self.levels[i].enhance(pred, levels_input[i][-2], training=training)