diff --git a/aaa_helpers.txt b/aaa_helpers.txt index 4994f34..703babd 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -93,6 +93,10 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skirt" > nohup/ic nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_reg.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_128_cape_m_00032_SL" > nohup/iccv/pyr/iccv_pyr_64_128_cape_m_00032_SL.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_128_cape_f_03223_SL" > nohup/iccv/pyr/iccv_pyr_64_128_cape_f_03223_SL.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_128_cape_f_03223_SS" > nohup/iccv/pyr/iccv_pyr_64_128_cape_f_03223_SS.out 2>&1 & + nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256" > nohup/iccv/pyr/iccv_pyr_32_64_128_256.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skirt.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index 9f13451..70f8fe0 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_skin import config +from configs.iccv.config_abl_pyr_32_64_128 import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] # import tensorflow as tf diff --git a/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SL.py b/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SL.py new file mode 100644 index 0000000..bae3f24 --- /dev/null +++ b/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SL.py @@ -0,0 +1,191 @@ +config = dict() + +config['gpu'] = '3' +config['name'] = 'iccv_pyr_64_128_cape_f_03223_SL' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'cape_f'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_64_cape_f_03223_SL' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/cape_processed/train/female' +config['valid_dir'] = '/home/workspace/local-data_127/cape_processed/train/female' +config['test_dir'] = '/home/workspace/local-data_127/cape_processed/train/female' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = True +config['frames_file'] = 'frames_03223_SL_train_2.txt' +config['frames_file_valid'] = 'frames_03223_SL_test_2.txt' +config['frames_file_test'] = 'frames_03223_SL_test_2.txt' + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_f' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_f' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_f' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_cape_f'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_cape_f'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_cape_f'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SS.py b/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SS.py new file mode 100644 index 0000000..f331881 --- /dev/null +++ b/configs/iccv/config_abl_pyr_64_128_cape_f_03223_SS.py @@ -0,0 +1,191 @@ +config = dict() + +config['gpu'] = '1' +config['name'] = 'iccv_pyr_64_128_cape_f_03223_SS' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'cape_f'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_64_cape_f_03223_SS' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/cape_processed/train/female' +config['valid_dir'] = '/home/workspace/local-data/cape_processed/train/female' +config['test_dir'] = '/home/workspace/local-data/cape_processed/train/female' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = True +config['frames_file'] = 'frames_03223_SS_train_2.txt' +config['frames_file_valid'] = 'frames_03223_SS_test_2.txt' +config['frames_file_test'] = 'frames_03223_SS_test_2.txt' + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_f' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_f' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_f' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_cape_f'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_cape_f'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_cape_f'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_64_128_cape_m_00032_SL.py b/configs/iccv/config_abl_pyr_64_128_cape_m_00032_SL.py new file mode 100644 index 0000000..3f24b13 --- /dev/null +++ b/configs/iccv/config_abl_pyr_64_128_cape_m_00032_SL.py @@ -0,0 +1,192 @@ +config = dict() + +config['gpu'] = '1' +config['name'] = 'iccv_pyr_64_128_cape_m_00032_SL' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'cape_m'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_64_cape_m_00032_SL' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' +config['valid_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' +config['test_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = True + +config['frames_file'] = 'frames_00032_SL_train_2.txt' +config['frames_file_valid'] = 'frames_00032_SL_test_2.txt' +config['frames_file_test'] = 'frames_00032_SL_test_2.txt' + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_m' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_m' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_m' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_cape_m'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_cape_m'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_cape_m'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_64_cape_m_00032_SL.py b/configs/iccv/config_abl_pyr_64_cape_m_00032_SL.py index 56369c0..4f1deb8 100644 --- a/configs/iccv/config_abl_pyr_64_cape_m_00032_SL.py +++ b/configs/iccv/config_abl_pyr_64_cape_m_00032_SL.py @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' config['valid_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' -config['test_dir'] = '/home/workspace/local-data_127/cape_processed/train/male' +config['test_dir'] = '/home/workspace/local-data/cape_processed/train/male' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = True diff --git a/cvae_pyramid.py b/cvae_pyramid.py index e1a7811..ff932e9 100644 --- a/cvae_pyramid.py +++ b/cvae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_64_pre_cape_f import config +from configs.iccv.config_abl_pyr_64_128_cape_f_03223_SS import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/evaluate_cape_unif_pyr.py b/evaluate_cape_unif_pyr.py new file mode 100644 index 0000000..70a8b2c --- /dev/null +++ b/evaluate_cape_unif_pyr.py @@ -0,0 +1,191 @@ +import os +import sys +from configs.iccv.config_abl_pyr_64_cape_m_00032_SL import config + +os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] +import tensorflow as tf + +gpus = tf.config.experimental.list_physical_devices('GPU') +if gpus: + try: + # Currently, memory growth needs to be the same across GPUs + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + logical_gpus = tf.config.experimental.list_logical_devices('GPU') + print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") + except RuntimeError as e: + # Memory growth must be set before GPUs have been initialized + print(e) + +import glob +import time +import argparse +import wandb +import tqdm +import random +import numpy as np +import plotly.graph_objects as go +import plotly.express as ex +from PIL import Image +from postprocessing import fix_collision + +if config['is_skirt']: + from losses import edge_loss_skirt as edge_loss +else: + from losses import edge_loss + + + +from losses import collision_ratio, kl_loss #, concept_loss +# from losses import collision_penalty as collision_penalty +from losses import collision_loss as collision_penalty +from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision +from dataset import DatasetFactory, StaticDataset +from custom_utils.smpl import SMPL +from models.pyramid_model_optimized import PyramidNew as Pyramid +from custom_utils.supersmpl_io import writeOBJ +from unif_metric import chamfer_and_score + + +uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_adaptive=False, is_smpl_orig=config['is_cape']) + +def group_inputs_for_levels(batch): + levels_input = [] + for res in config['pyramid']['resolutions']: + res_modifier = '_' + str(res) if res != 512 else '' + levels_input.append([batch['uv_unposed' + res_modifier], batch['uv_static' + res_modifier], batch['uv_body_posed' + res_modifier], batch['uv_curvature' + res_modifier], batch['uv_mask' + res_modifier], batch['uv_mask']]) + conds = [batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature']] + return levels_input, conds + + +@tf.function +def step(batch): + levels_input, conds = group_inputs_for_levels(batch) + out_levels = model([levels_input, conds], training=False) + pred = uv2mesh(out_levels[-1][0]) + + mesh_pred = uv2mesh(pred) + + # if config['use_skinning']: + # shape = batch['pose_shape'][:, 72:] + # gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32) + # shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]) + + # mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] + # mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped) + # else: + # mesh_pred_posed = mesh_pred + + # mesh_pred_posed_fixed = fix_collision(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask']) + + # return mesh_pred_posed, None #mesh_pred_posed_fixed + return mesh_pred + + +if __name__ == '__main__': + # init_wandb() + parser = argparse.ArgumentParser(description='') + parser.add_argument('--name', type=str) + args = vars(parser.parse_args()) + + skirt_modifier = '_skirt' if config['is_skirt'] else '' + + folder = config['test_dir'] + smpl = SMPL(use_one_hot=False, orig_smpl=config['is_cape']) + model = Pyramid(config) + + # str_split = config['name'].split('_') + # test_idx = str_split.index('test') + # checkpoint_name = '_'.join(str_split[:test_idx]) + checkpoint_name = config['name'] + + if config['is_cape']: + F = np.load("custom_utils/smpl_faces_orig.npy").astype(np.int32) + else: + F = np.load("custom_utils/smpl_faces" + skirt_modifier + ".npy").astype(np.int32) + + + # ids = os.listdir(os.path.join(folder, 'pose_shape')) + # ids = [id.split('_')[0] for id in ids] + # ids = set(ids) + + iteration = '' + if iteration == '': + iteration = load_model(model, 'checkpoints/iccv/' + checkpoint_name, return_iteration=True) + else: + load_model(model, 'checkpoints/iccv/' + checkpoint_name, iteration=iteration) + + + output_folder = '/home/workspace/data/results/' + checkpoint_name + '-' + iteration + + + # os.makedirs(output_folder, exist_ok=True) + + # if config['use_skinning']: + # config['signature_dynamic'].pop('mesh_posed') + # config['signature_dynamic'].pop('mesh_unposed') + # config['signature_dynamic'].pop('mesh_body_posed') + dataset_resolutions = config['pyramid']['resolutions'].copy() + if 512 not in dataset_resolutions: + dataset_resolutions.append(512) + factory = DatasetFactory() + # dataset_test = factory.get_dataset(folder=config['test_dir'], + # batch_size=config['batch_size'], + # signature=config['signature_dynamic'], + # resolutions=[config['resolution']], + # in_memory=False, + # do_repeat=False, + # use_no_orient=config['remove_global_orientation'], + # frames_file=config['frames_file_test'] if 'frames_file_test' in config.keys() else None, + # is_cape=config['is_cape']) + # static_dataset_test = StaticDataset(folder=config['test_dir'], + # signature=config['signature_static'], + # resolutions=[config['resolution']], + # in_memory=True, + # is_cape=config['is_cape']) + + dataset_test = factory.get_dataset(folder=config['test_dir'], + batch_size=config['batch_size'], + signature=config['signature_dynamic'], + resolutions=dataset_resolutions, + in_memory=False, + do_repeat=False, + use_no_orient=True, + frames_file=config['frames_file_test'] if 'frames_file_test' in config.keys() else None, + is_cape=config['is_cape']) + static_dataset_test = StaticDataset(folder=config['test_dir'], + signature=config['signature_static'], + resolutions=dataset_resolutions, + in_memory=True, + is_cape=config['is_cape']) + # faces_folder = '/home/workspace/data/results/support/faces' + ('_skirt' if config['is_skirt'] else '') + # ids = os.listdir(faces_folder) + # ids = [id.split('.')[0] for id in ids] + # faces = {id : np.load(faces_folder + '/' + id + '.npy') for id in ids } + errors = [] + euclid = [] + + for batch in tqdm.tqdm(dataset_test): + batch = static_dataset_test.expand_batch(batch_to_dict(batch, config['signature_dynamic'])) + + mesh_pred = step(batch) + # mesh_pred_posed, mesh_pred_posed_fixed = mesh_pred_posed.numpy(), mesh_pred_posed_fixed.numpy() + + mask = batch['mesh_mask'].numpy().astype(bool) + + for i in range(mask.shape[0]): + id, frame_nr = batch['id'][i].numpy().decode("utf-8"), batch['frame_nr'][i].numpy().decode("utf-8") + x = chamfer_and_score(mesh_pred.numpy()[i, mask[i], :], batch['mesh_unposed'].numpy()[i, mask[i], :]) + y = np.mean(np.sqrt(np.sum((mesh_pred.numpy()[i, mask[i], :] - batch['mesh_unposed'].numpy()[i, mask[i], :])**2, axis=-1))) + errors.append(x[1]) + euclid.append(y) + + # writeOBJ(os.path.join(output_folder, id + '_' + frame_nr + '.obj'), mesh_pred_posed[i, mask[i], :], faces[id]) + # np.save(os.path.join(output_folder, id + '_' + frame_nr + '.npy'), mesh_pred_posed[i, mask[i], :]) + # np.save(os.path.join(output_folder, id + '_' + frame_nr + '_fixed_coll.npy'), mesh_pred_posed_fixed[i, mask[i], :]) + # writeOBJ(os.path.join(output_folder, id + '_' + frame_nr + '_fixed_coll.obj'), mesh_pred_posed_fixed[i, mask[i], :], faces[id]) + + print(np.mean(errors)) + print(np.mean(euclid)) + print() +