diff --git a/aaa_helpers.txt b/aaa_helpers.txt index 932e274..d14a195 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -44,7 +44,7 @@ nohup python cvae.py --name="lhunor_abl_skin_var" > nohup/iccv/ablation/abl_skin nohup python evaluate_ae.py --name abl_all > nohup/iccv/evaluate/abl_all.out 2>&1 & nohup python evaluate_ae.py --name abl_all_skirt > nohup/iccv/evaluate/abl_all_skirt.out 2>&1 & -nohup python cvae_curv.py --name="lhunor_cvae_curv" > nohup/iccv/ablation/cvae_curv.out 2>&1 & +nohup python cvae_curv.py --name="lhunor_cvae_curv" > nohup/iccv/ablation/cvae_curv_3.out 2>&1 & nohup python upscale_test.py --name="iccv_32_2" > nohup/iccv/upscale/iccv_32_2.out 2>&1 & nohup python upscale_test.py --name="iccv_64_2" > nohup/iccv/upscale/iccv_64_2.out 2>&1 & @@ -86,6 +86,8 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_curv" > nohup/iccv/py nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_curv_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_reg" > nohup/iccv/pyr/iccv_pyr_32_64_reg.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_reg_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_reg_skirt.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_10p" > nohup/iccv/pyr/iccv_pyr_32_64_10p.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_25p" > nohup/iccv/pyr/iccv_pyr_32_64_25p.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f" > nohup/iccv/pyr/iccv_pyr_64_cape_f.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_m" > nohup/iccv/pyr/iccv_pyr_64_cape_m.out 2>&1 & @@ -121,6 +123,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512" > nohup/ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_skin.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_curv.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512_reg.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32 > nohup/iccv/evaluate/iccv_pyr_32.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_skirt > nohup/iccv/evaluate/iccv_pyr_32_skirt.out 2>&1 & @@ -130,6 +133,8 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_reg > nohup/i nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_reg_skirt > nohup/iccv/evaluate/iccv_pyr_32_reg_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_curv > nohup/iccv/evaluate/iccv_pyr_32_curv.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_curv_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_10p > nohup/iccv/evaluate/iccv_pyr_32_10p.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_25p > nohup/iccv/evaluate/iccv_pyr_32_25p.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64 > nohup/iccv/evaluate/iccv_pyr_32_64.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_skin > nohup/iccv/evaluate/iccv_pyr_32_64_skin.out 2>&1 & @@ -148,9 +153,15 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv_s nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256 > nohup/iccv/evaluate/iccv_pyr_32_64_128_256.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_skin > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_skin.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_curv > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_curv.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_reg > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_reg.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_512 > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_512.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_512_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_512_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_512_reg > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_512_reg.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_512_skin > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_512_skin.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_512_curv > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_512_curv.out 2>&1 & nohup python cvae.py --name="lhunor_cape_f_03223_SL" > nohup/iccv/ablation/abl_all_cape_f_03223_SL.out 2>&1 & nohup python cvae.py --name="lhunor_cape_f_03223_SS" > nohup/iccv/ablation/abl_all_cape_f_03223_SS.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index 2dcd1d3..62f7622 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_curv_skirt import config +from configs.iccv.config_abl_pyr_32_25p import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] # import tensorflow as tf diff --git a/configs/iccv/config_abl_all_skirt_var.py b/configs/iccv/config_abl_all_skirt_var.py index bd74c10..b06d065 100644 --- a/configs/iccv/config_abl_all_skirt_var.py +++ b/configs/iccv/config_abl_all_skirt_var.py @@ -19,7 +19,7 @@ ablation_count = [config['use_skinning'], config['remove_global_orientation'], c assert ablation_count == 1 or ablation_count == 0 config['gpu'] = '0' -config['name'] = 'abl_all_skirt_var_2' +config['name'] = 'abl_all_skirt_var' config['notes'] = '' config['tags'] = ['iccv', 'abl_all', 'skirt'] diff --git a/configs/iccv/config_abl_pyr_32_10p.py b/configs/iccv/config_abl_pyr_32_10p.py index df06227..046cc12 100644 --- a/configs/iccv/config_abl_pyr_32_10p.py +++ b/configs/iccv/config_abl_pyr_32_10p.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '4' +config['gpu'] = '2' config['name'] = 'iccv_pyr_32_10p' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_25p.py b/configs/iccv/config_abl_pyr_32_25p.py index b740e00..868ff3a 100644 --- a/configs/iccv/config_abl_pyr_32_25p.py +++ b/configs/iccv/config_abl_pyr_32_25p.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '5' +config['gpu'] = '2' config['name'] = 'iccv_pyr_32_25p' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_64_10p.py b/configs/iccv/config_abl_pyr_32_64_10p.py new file mode 100644 index 0000000..dc2d547 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_10p.py @@ -0,0 +1,189 @@ +config = dict() + +config['gpu'] = '4' +config['name'] = 'iccv_pyr_32_64_10p' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_10p' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/train' +config['valid_dir'] = '/home/workspace/local-data_127/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' + +config['frames_file'] = 'frames_10p.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_10p' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_10p' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_10p' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py index 7199e28..eec6410 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '3' +config['gpu'] = '2' config['name'] = 'iccv_pyr_32_64_128_256_512_curv' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'curv'] @@ -33,7 +33,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data_127/train' config['valid_dir'] = '/home/workspace/local-data_127/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py new file mode 100644 index 0000000..8ee9402 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py @@ -0,0 +1,187 @@ +config = dict() + +config['gpu'] = '0' +config['name'] = 'iccv_pyr_32_64_128_256_512_reg' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'reg'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = False # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = False +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128, 256, 512] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_128_256_reg' +config['pyramid']['pretrained_pyramid_iteration'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/train' +config['valid_dir'] = '/home/workspace/local-data_127/valid' +config['test_dir'] = '/home/workspace/local-data/test_ready' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py index 3a30602..59a3ccf 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '4' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_128_256_512_skin' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -33,7 +33,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_curv.py b/configs/iccv/config_abl_pyr_32_64_128_256_curv.py index b3b452f..edf1c2b 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_curv.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_curv.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '3' +config['gpu'] = '2' config['name'] = 'iccv_pyr_32_64_128_256_curv' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'curv'] diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_reg.py b/configs/iccv/config_abl_pyr_32_64_128_256_reg.py index 1255cc2..094565d 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_reg.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_reg.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '1' +config['gpu'] = '5' config['name'] = 'iccv_pyr_32_64_128_256_reg' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'reg'] diff --git a/configs/iccv/config_abl_pyr_32_64_25p.py b/configs/iccv/config_abl_pyr_32_64_25p.py new file mode 100644 index 0000000..9966141 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_25p.py @@ -0,0 +1,189 @@ +config = dict() + +config['gpu'] = '5' +config['name'] = 'iccv_pyr_32_64_25p' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_25p' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/train' +config['valid_dir'] = '/home/workspace/local-data_127/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' + +config['frames_file'] = 'frames_25p.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_25p' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_25p' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_25p' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_cvae_curv.py b/configs/iccv/config_cvae_curv.py index 76368d0..ab12b04 100644 --- a/configs/iccv/config_cvae_curv.py +++ b/configs/iccv/config_cvae_curv.py @@ -19,7 +19,7 @@ ablation_count = [config['use_skinning'], config['remove_global_orientation'], c assert ablation_count == 1 or ablation_count == 0 config['gpu'] = '0' -config['name'] = 'cvae_curv' +config['name'] = 'cvae_curv_3' config['notes'] = '' config['tags'] = ['iccv', 'cvae_curv'] @@ -50,12 +50,13 @@ if config['is_cape']: config['latent_dim'] = 128 -config['pretrained_model'] = '' +config['pretrained_model'] = 'checkpoints/iccv/cvae_curv' +config['pretrained_model_iteration'] = '' ############### config['lr'] = 0.0001# 0.0001 ############### -config['starting_balancing_coeff'] = 0.01 +config['starting_balancing_coeff'] = 1.0 config['signature_dynamic'] = { 'id': (), 'frame_nr': (), 'pose_shape': (72 + 11,), diff --git a/curvature_analysis.py b/curvature_analysis.py new file mode 100644 index 0000000..6b08b86 --- /dev/null +++ b/curvature_analysis.py @@ -0,0 +1,297 @@ +import os +import sys +os.environ["CUDA_VISIBLE_DEVICES"] = '4' + + +import tqdm +import argparse +import random +import string +import time +import numpy as np +import trimesh as tm +import cmapy as cm +import scipy as sp +import scipy.io as sio +import plotly.graph_objects as go +from pyntcloud import PyntCloud +from scipy.sparse import csr_matrix, coo_matrix +from scipy.sparse.csgraph import floyd_warshall, dijkstra +from plotly.subplots import make_subplots +from custom_utils.custom_utils import _maskFaces +from postprocessing import fix_collision + +from custom_utils.supersmpl_io import readPC2, readPC2Frame, writePC2, readOBJ +from custom_utils.custom_utils import _maskFaces, save_stl +from auto_encoder_curv import dice_dense + + +def write_obj(file, vertices, faces, colors): + with open(file, 'w') as file: + for v, c in zip(vertices, colors): + line = 'v ' + ' '.join([str(_) for _ in v]) + ' ' + ' '.join([str(_) for _ in c]) + '\n' + file.write(line) + for f in faces: + line = 'f ' + ' '.join([str(_) for _ in f]) + '\n' + file.write(line) + + +def postprocess(mesh, faces, mask=None, body=None, do_smoothing=True): + if body is not None: + mesh_fixed = fix_collision(mesh, body[np.newaxis,:,:], mask[np.newaxis,:])[0].numpy()[mask, :] + else: + mesh_fixed = mesh + + if do_smoothing: + m = tm.Trimesh(mesh_fixed, faces, process=False, validate=False) + outliers = np.array([len(v) for v in m.vertex_neighbors]) == 0 + outliers = np.nonzero(outliers)[0] + outliers_n_neighbours = get_closest_n_points(mesh_fixed, outliers, 2) + outliers_faces = np.concatenate((outliers[:, np.newaxis], outliers_n_neighbours), axis=-1) + faces_fixed = np.concatenate((faces, outliers_faces), axis=0) + m = tm.Trimesh(mesh_fixed, faces_fixed, process=False, validate=False) + tm.smoothing.filter_humphrey(m, iterations=1, alpha=0.6, beta=0.4) + + mesh = m.vertices + else: + mesh = mesh_fixed + + return mesh + +def export_to_ply(mesh, file): + ply = tm.exchange.ply.export_ply(mesh, encoding='binary', vertex_normal=True, include_attributes=True) + output_file = open(file, "wb+") + output_file.write(ply) + output_file.close() + + +def get_closest_n_points(verts, idx, n): + dist = sp.spatial.distance.cdist(verts[idx, :], verts) + dist[dist == 0] = 1e9 + idx_sort = np.argsort(dist, axis=-1) + closest = idx_sort[:, :n] + return closest + + +def calculate_dist(folder, neighbourhood): + out = dict() + frames = sorted(os.listdir(os.path.join(folder, 'mesh_posed')))[int(args.start):int(args.end)] + + for frame in tqdm.tqdm(frames): + id = frame.split('_')[0] + if id in out.keys(): + continue + + mask = np.load(os.path.join(folder, 'mesh_mask', f'{id}.npy')).astype(bool) + k_neighbors = np.argsort(dist[mask, :][:, mask], axis=-1)[:, :neighbourhood] #cloud.get_neighbors(k=neighbourhood) + out[id] = k_neighbors + + return out + + +def histedges_equalN(x, nbin): + npt = len(x) + return np.interp(np.linspace(0, npt, nbin + 1), + np.arange(npt), + np.sort(x)) + + +def get_bins(values, nr_of_bins): + flattened = [] + for c in values: + flattened.extend(c.flatten()) + flattened = np.array(flattened) + bins = histedges_equalN(flattened, nr_of_bins) + return bins + + +def generate_curvatures(v, faces, mask, neighbourhood=25, nr_of_labels=6): + # id = frame.split('_')[0] + # frame_nr = int(frame.split('_')[1].split('.')[0]) + + # mask = np.load(os.path.join(folder, 'mesh_mask', f'{id}.npy')).astype(bool) + # _, faces = _maskFaces(np.zeros((14475,3)), F, mask) + # v = np.load(os.path.join(folder, 'mesh_posed', f'{id}_{frame_nr}.npz'))['mesh_posed'][mask,:] + + m = tm.Trimesh(v, faces, process=False, validate=False) + + outliers = np.array([len(v) for v in m.vertex_neighbors]) == 0 + outliers = np.nonzero(outliers)[0] + outliers_n_neighbours = get_closest_n_points(v, outliers, 2) + outliers_faces = np.concatenate((outliers[:, np.newaxis], outliers_n_neighbours), axis=-1) + faces = np.concatenate((faces, outliers_faces), axis=0) + + verts = v + + m = tm.Trimesh(verts, faces, process=False, validate=False) + export_to_ply(m, temp_file) + + cloud = PyntCloud.from_file(temp_file) + + k_neighbors = np.argsort(dist[mask, :][:, mask], axis=-1)[:, :neighbourhood] #cloud.get_neighbors(k=neighbourhood) + #k_neighbors = k_neighbors_cache[id] # np.argsort(dist[mask, :][:, mask], axis=-1)[:, :neighbourhood] #cloud.get_neighbors(k=neighbourhood) + ev = cloud.add_scalar_field("eigen_values", k_neighbors=k_neighbors) + cloud.add_scalar_field("curvature", ev=ev) + curvature = np.array(cloud.points["curvature(" + str(neighbourhood+1) + ")"]) + + # bins = np.load(f'custom_data/train_bins.npy') # get_bins(curvature, nr_of_labels) + bins = get_bins(curvature, nr_of_labels) + labels = np.linspace(0, bins.shape[0] - 2, bins.shape[0] - 1) + + frame_labels = np.zeros((curvature.shape[0]), dtype=np.float32) + for j in range(bins.shape[0] - 2): + frame_labels[(curvature >= bins[j]) * (curvature < bins[j+1])] = labels[j] + frame_labels[curvature >= bins[-2]] = labels[-1] + + return frame_labels + +import tensorflow as tf + + +def visualize_curvature_labels(folder, results_folder, frame): + id = frame.split('_')[0] + frame_nr = frame.split('_')[1].split('.')[0] + + mask = np.load(os.path.join(folder, 'mesh_mask', f'{id}.npy')).astype(bool) + _, faces = _maskFaces(np.zeros((14475,3)), F, mask) + verts = np.load(os.path.join(folder, 'mesh_posed', f'{id}_{frame_nr}.npz'))['mesh_posed'][mask,:] + pred = np.load(os.path.join(results_folder, frame + '.npy')) + pred = postprocess(pred, faces) + + curvature_labels_gt = generate_curvatures(verts, faces, mask, 25) + curvature_labels_pred = generate_curvatures(pred, faces, mask, 25) + # dice = dice_dense(curvature_labels_pred[None,:], curvature_labels_gt[None,:]) + dice = dice_dense(tf.one_hot(curvature_labels_pred, 7)[None,:], tf.one_hot(curvature_labels_gt, 7)[None,:]).numpy() + colors_binned_all = [] + + for curvature_labels in [curvature_labels_gt, curvature_labels_pred]: + colors = np.array([cm.color('jet', 1 - i) for i in np.linspace(0, 1, int(curvature_labels.max()+1))]) + colors_binned = np.zeros((curvature_labels.shape[0], 3)) + for i in range(int(curvature_labels.max())+1): + # print(np.count_nonzero(curvature_labels == i)) + colors_binned[curvature_labels == i, :] = colors[i] + colors_binned_all.append(colors_binned) + + fig = make_subplots(rows=1, cols=3, start_cell="top-left", specs = [ [{ 'type': 'scene' } for _ in range(3)] ]) + + fig.add_trace(go.Mesh3d( + # 8 vertices of a cube + x=-pred[:,0], + y=pred[:,2], + z=pred[:,1], + # i, j and k give the vertices of triangles + i = faces[:,0], + j = faces[:,1], + k = faces[:,2], + vertexcolor=np.ones((verts.shape[0],3))*np.array([255,200,200])[None,:] + ), + row=1, col=2) + + fig.add_trace(go.Mesh3d( + # 8 vertices of a cube + x=verts[:,0], + y=verts[:,1], + z=verts[:,2], + # i, j and k give the vertices of triangles + i = faces[:,0], + j = faces[:,1], + k = faces[:,2], + vertexcolor=colors_binned_all[0] + ), + row=1, col=1) + + fig.add_trace(go.Mesh3d( + # 8 vertices of a cube + x=-pred[:,0], + y=pred[:,2], + z=pred[:,1], + # i, j and k give the vertices of triangles + i = faces[:,0], + j = faces[:,1], + k = faces[:,2], + vertexcolor=colors_binned_all[1], + text=f'dice: {dice}' + ), + row=1, col=3) + + fig.update_layout(scene_aspectmode='manual', scene_aspectratio=dict(x=1, y=1, z=1), + scene2_aspectmode='manual', scene2_aspectratio=dict(x=1, y=1, z=1), + scene3_aspectmode='manual', scene3_aspectratio=dict(x=1, y=1, z=1), + # scene4_aspectmode='manual', scene4_aspectratio=dict(x=1, y=1, z=1), + # scene5_aspectmode='manual', scene5_aspectratio=dict(x=1, y=1, z=1), + ) + helper = dict(xaxis = dict(range=[-1,1], showbackground=False, showticklabels=False, showaxeslabels=False, showspikes=False, visible=False), + yaxis = dict(range=[-1,1], showbackground=False, showticklabels=False, showaxeslabels=False, showspikes=False, visible=False), + zaxis = dict(range=[-1,1], showbackground=False, showticklabels=False, showaxeslabels=False, showspikes=False, visible=False),) + + fig.update_layout(scene = helper, + scene2 = helper, + scene3 = helper, + # scene4 = helper, + # scene5 = helper, + ) + + + # meshes = [go.Mesh3d( + # x=x[:,0], + # y=x[:,1], + # z=x[:,2], + # i = faces[:,0], + # j = faces[:,1], + # k = faces[:,2], + # vertexcolor=colors_binned + # )] + + # fig = go.Figure(data=meshes) + # fig.update_layout(scene_aspectmode='manual', scene_aspectratio=dict(x=1, y=1, z=1)) + # fig.update_layout( + # scene = dict( + # xaxis = dict(nticks=4, range=[-1,1],), + # yaxis = dict(nticks=4, range=[-1,1],), + # zaxis = dict(nticks=4, range=[-1,1],),) + # ) + + return fig, dice + +import wandb + +def init_wandb(): + run = wandb.init(project='plots', name='curvatures', dir='/home/workspace/local-data/temp/') + artifact = wandb.Artifact('code', type='code') + artifact.add_file('curvature_analysis.py') + run.log_artifact(artifact) + +if __name__ == '__main__': + init_wandb() + + is_skirt = False + skirt_modifier = '_skirt' if is_skirt else '' + F = np.load("custom_utils/smpl_faces" + skirt_modifier + ".npy").astype(np.int32) + # k_neighbours(k=25) + + folder = '/home/workspace/local-data/test_ready' + results_folder = '/home/workspace/data/results/iccv/iccv_pyr_32_64_128_256_512-120000' + + # frames_file = 'frames_temp.txt' + # with open(os.path.join(folder, frames_file), 'r') as f: + # lines = f.readlines() + # lines = [line.strip() for line in lines] + # frames = lines + frames = os.listdir(os.path.join(folder, 'pose_shape')) + frames = [frame.split('.')[0] for frame in frames] + + temp_file = '/home/workspace/local-data/temp/' + ''.join(random.choice(string.ascii_lowercase) for i in range(10)) + '.ply' + if not is_skirt: + dist = np.load('custom_data/dijkstra.npy') + else: + dist = np.load('custom_data/dijkstra_skirt.npy') + + i = 0 + while True: + frame = np.random.choice(frames) + # frame = '03265_193' + # frame = '03102_49' + fig, dice = visualize_curvature_labels(folder, results_folder, frame) + wandb.log({f'{frame}-dice:{dice}': fig}, step=i) + i += 1 + print(frame) \ No newline at end of file diff --git a/cvae_curv.py b/cvae_curv.py index 83838a3..e64c402 100644 --- a/cvae_curv.py +++ b/cvae_curv.py @@ -153,11 +153,11 @@ def dice_dense(prediction, ground_truth, weight_map=None): return tf.reduce_mean(dice_score) -# @tf.function +@tf.function def train_step(batch): with tf.GradientTape() as tape: - pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']]) + pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True) ce_uv_loss = scce(tf.math.argmax(batch['uv_curvature'], axis=-1), pred) dice = dice_dense(pred, batch['uv_curvature']) @@ -189,7 +189,7 @@ def train_step(batch): @tf.function def valid_step(batch): - pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']]) + pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) ce_uv_loss = scce(tf.math.argmax(batch['uv_curvature'], axis=-1), pred) loss = ce_uv_loss @@ -216,7 +216,7 @@ def valid_step(batch): def plot_everything(batch: dict, type: str): - pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']]) + pred, mean, logvar = model([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) pred = np.argmax(pred.numpy(), axis=-1) mesh_pred = uv2mesh(pred) @@ -252,27 +252,51 @@ def plot_everything(batch: dict, type: str): def sample(): - iteration = load_model(model, f'checkpoints/iccv/cvae_curv', return_iteration=True) + iteration = load_model(model, f'checkpoints/iccv/cvae_curv_3', return_iteration=True) for batch in dataset_train: batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic'])) nr_of_samples = 4 - # pred = model.sample(batch['uv_static'], batch['uv_body_posed'], batch['uv_mask']) - # pred = model.sample_single(batch['uv_static'][0], batch['uv_body_posed'][0], batch['uv_mask'][0], nr_of_samples=nr_of_samples) - pred = model.sample(tf.tile(batch['uv_static'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][0][None,:,:], [nr_of_samples,1,1])) - # pred,_,_ = model([tf.tile(batch['uv_static'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][0][None,:,:], [nr_of_samples,1,1])], training=False) - - orig_img = np.argmax(batch['uv_curvature'][0].numpy(), axis=-1) - pred_img = np.argmax(pred.numpy(), axis=-1) - img = np.concatenate((orig_img, np.concatenate(pred_img, axis=1)), axis=1) - - colors = np.array([cm.color('jet', 1 - i) for i in np.linspace(0, 1, 6+1)]) - color_img = np.zeros((img.shape[0], img.shape[1], 3), np.uint8) - for i in range(7): - color_img[img == i, :] = colors[i] + for i in range(4): + # pred = model.sample(batch['uv_static'], batch['uv_body_posed'], batch['uv_mask']) + # pred = model.sample_single(batch['uv_static'][0], batch['uv_body_posed'][0], batch['uv_mask'][0], nr_of_samples=nr_of_samples) + pred = model.sample(tf.tile(batch['uv_static'][i][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][i][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][i][None,:,:], [nr_of_samples,1,1])) + # pred,_,_ = model([tf.tile(batch['uv_static'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][0][None,:,:], [nr_of_samples,1,1])], training=False) + # pred = model.call_sample([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']]) + + orig_img = np.argmax(batch['uv_curvature'][i].numpy(), axis=-1) + pred_img = np.argmax(pred.numpy(), axis=-1) + img = np.concatenate((orig_img, np.concatenate(pred_img, axis=1)), axis=1) + colors = np.array([cm.color('jet', 1 - i) for i in np.linspace(0, 1, 6+1)]) + color_img = np.zeros((img.shape[0], img.shape[1], 3), np.uint8) + for i in range(7): + color_img[img == i, :] = colors[i] + ex.imshow(color_img).show() + print() print() +def sample_2(batch): + nr_of_samples = 4 + + # pred = model.sample(batch['uv_static'], batch['uv_body_posed'], batch['uv_mask']) + # pred = model.sample_single(batch['uv_static'][0], batch['uv_body_posed'][0], batch['uv_mask'][0], nr_of_samples=nr_of_samples) + # pred = model.sample(tf.tile(batch['uv_static'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][0][None,:,:], [nr_of_samples,1,1])) + pred,_,_ = model([tf.tile(batch['uv_static'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_body_posed'][0][None,:,:,:], [nr_of_samples,1,1,1]),tf.tile(batch['uv_curvature'][0][None,:,:,:], [nr_of_samples,1,1,1]), tf.tile(batch['uv_mask'][0][None,:,:], [nr_of_samples,1,1])], training=False) + # pred = model.call_sample([batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']]) + + orig_img = np.argmax(batch['uv_curvature'][0].numpy(), axis=-1) + pred_img = np.argmax(pred.numpy(), axis=-1) + img = np.concatenate((orig_img, np.concatenate(pred_img, axis=1)), axis=1) + + colors = np.array([cm.color('jet', 1 - i) for i in np.linspace(0, 1, 6+1)]) + color_img = np.zeros((img.shape[0], img.shape[1], 3), np.uint8) + for i in range(7): + color_img[img == i, :] = colors[i] + ex.imshow(color_img).show() + print() + + if __name__ == '__main__': # init_wandb() parser = argparse.ArgumentParser(description='') diff --git a/cvae_pyramid.py b/cvae_pyramid.py index 801f702..5958d19 100644 --- a/cvae_pyramid.py +++ b/cvae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_256_512_curv import config +from configs.iccv.config_abl_pyr_32_64_25p import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/evaluate_ae_pyramid.py b/evaluate_ae_pyramid.py index 2031708..0472d9d 100644 --- a/evaluate_ae_pyramid.py +++ b/evaluate_ae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_256_512_skirt import config +from configs.iccv.config_abl_pyr_32_64_128_256_512_curv import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/get_error.py b/get_error.py index 93ebaff..2c5b4e2 100644 --- a/get_error.py +++ b/get_error.py @@ -7,10 +7,10 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() folder = '/home/workspace/data/results/errors/iccv' - checkpoint_name = 'iccv_pyr_32_64_skin' - checkpoint_name_skirt = 'iccv_pyr_32_64_skin_skirt' - iteration = '130000' - iteration_skirt : str = '65000' + checkpoint_name = 'iccv_pyr_32_64_128_256_512' + checkpoint_name_skirt = 'iccv_pyr_32_64_128_256_512_skirt' + iteration = '120000' + iteration_skirt : str = '115000' # checkpoint_name = 'abl_skin' # checkpoint_name_skirt = checkpoint_name + '_skirt' @@ -28,23 +28,25 @@ if __name__ == '__main__': for row in csv_reader: per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) - with open(os.path.join(folder, checkpoint_name_skirt + '-' + iteration_skirt + '.csv'), 'r') as f: - csv_reader = csv.reader(f, delimiter=',') - for row in csv_reader: - per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) - print('\nNon-skirt:') for key in per_garment_errors.keys(): avg = np.mean(per_garment_errors[key]) errors.append(avg) print(key + ': ' + str(avg)) + + print('\nAvg non-skirt: ' + str(np.mean(errors))) + + with open(os.path.join(folder, checkpoint_name_skirt + '-' + iteration_skirt + '.csv'), 'r') as f: + csv_reader = csv.reader(f, delimiter=',') + for row in csv_reader: + per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) + print('\nSkirt:') for key in per_garment_errors_skirt.keys(): avg = np.mean(per_garment_errors_skirt[key]) errors_skirt.append(avg) print(key + ': ' + str(avg)) - print('\nAvg non-skirt: ' + str(np.mean(errors))) print('Avg skirt: ' + str(np.mean(errors_skirt))) cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt']) diff --git a/models/cvae_curv_model.py b/models/cvae_curv_model.py index 38940ce..08e58c4 100644 --- a/models/cvae_curv_model.py +++ b/models/cvae_curv_model.py @@ -50,7 +50,7 @@ class CVAE(tf.keras.Model): # @tf.function def sample_single(self, uv_template, uv_body_posed, uv_mask, eps=None, nr_of_samples=4): if eps is None: - eps = tf.random.normal(shape=(nr_of_samples, self.latent_dim)) + eps = tf.random.normal(shape=(nr_of_samples, 8, 4, self.latent_dim)) uv_template = tf.tile(tf.transpose(uv_template, [1,2,0])[tf.newaxis,:,:,:], [nr_of_samples,1,1,1]) uv_body_posed = tf.tile(tf.transpose(uv_body_posed, [1,2,0])[tf.newaxis,:,:,:], [nr_of_samples,1,1,1]) @@ -149,3 +149,15 @@ class CVAE(tf.keras.Model): # x = tf.math.softmax(x) return x, mean, logvar + + def call_sample(self, inputs): + uv_template, uv_body_posed, uv_curv, uv_mask = inputs + + z = tf.random.normal(shape=(uv_template.shape[0], 8, 4, self.latent_dim)) + + z_cond = self.encode_conds(uv_template, uv_body_posed, training=False) + + z = tf.concat([z, z_cond], axis=-1) + x = self.decoder([z, uv_mask], training=False) + + return x \ No newline at end of file diff --git a/plot_ablation_pyramid.py b/plot_ablation_pyramid.py index ba7ab48..b98f094 100644 --- a/plot_ablation_pyramid.py +++ b/plot_ablation_pyramid.py @@ -1,5 +1,5 @@ import os -os.environ["CUDA_VISIBLE_DEVICES"] = "5" +os.environ["CUDA_VISIBLE_DEVICES"] = "4" import tensorflow as tf @@ -30,15 +30,15 @@ from plotly.subplots import make_subplots from scipy.spatial.transform import Rotation from dataset import DatasetFactory, StaticDataset -from custom_utils.custom_utils import batch_to_dict, UV2Mesh, _maskFaces, load_model, group_inputs_for_levels +from custom_utils.custom_utils import batch_to_dict, UV2Mesh, _maskFaces, load_model, group_inputs_for_levels, save_stl from custom_utils.smpl import SMPL from convert_to_uv_api import convert_to_uv_api from models.upscaler import UpscalerBasicConv as Upscaler from postprocessing import fix_collision -from models.pyramid_model_optimized import Pyramid +from models.pyramid_model_optimized import PyramidNew as Pyramid -from configs.config_abl_2_pyr_32_64_128_256_512 import config as config_regular -from configs.config_abl_2_pyr_32_64_128_256_512_skirt import config as config_skirt +from configs.iccv.config_abl_pyr_32_64_128_256_512 import config as config_regular +from configs.iccv.config_abl_pyr_32_64_128_256_512_skirt import config as config_skirt def color_labels(img): @@ -182,7 +182,7 @@ def convert_to_mesh(uvs, is_skirt, mask): out = [] for i in range(len(uvs)): if uvs[i].shape[1] != 512: - uvs[i] = tf.image.resize(uvs[i], (512, 256), tf.image.ResizeMethod.LANCZOS5) + uvs[i] = tf.image.resize(uvs[i], (512, 256), tf.image.ResizeMethod.BILINEAR) if is_skirt: mesh = uv2mesh_skirt(uvs[i]) else: @@ -239,9 +239,9 @@ def get_closest_n_points(verts, idx, n): def get_experiment_plots(frames, skirt_mods, bodies, faces, masks, cameras): model: Pyramid = Pyramid(config_regular) - load_model(model, 'checkpoints/' + config_regular['name']) + load_model(model, 'checkpoints/iccv/' + config_regular['name']) model_skirt: Pyramid = Pyramid(config_skirt) - load_model(model_skirt, 'checkpoints/' + config_skirt['name']) + load_model(model_skirt, 'checkpoints/iccv/' + config_skirt['name']) result = [] for frame in frames: id = frame.split('_')[0] @@ -267,6 +267,10 @@ def get_experiment_plots(frames, skirt_mods, bodies, faces, masks, cameras): img = cv2.imread('/tmp/exp_temp.png') frame_out.append(img[:, 50:-50, :]) + for i, mesh in enumerate(meshes): + save_stl(os.path.join(output_folder, f'{frame}_{i+1}.stl'), mesh, faces[id].astype(np.int32)) + save_stl(os.path.join(output_folder, f'{frame}_final.stl'), final_mesh_fixed, faces[id].astype(np.int32)) + # fig = plot_fixed(final_mesh, faces[id], cameras[frame], bodies[frame]) # fig.write_image('/tmp/exp_temp.png') # img = cv2.imread('/tmp/exp_temp.png') @@ -297,6 +301,10 @@ def get_gt_image(frames, bodies, faces, cameras, skirt_mods): fig.write_image('/tmp/exp_temp.png') img_unposed = cv2.imread('/tmp/exp_temp.png') + save_stl(os.path.join(output_folder, f'{frame}_gt.stl'), gt_unposed[mask,:], faces[id].astype(np.int32)) + save_stl(os.path.join(output_folder, f'{frame}_gt_unposed.stl'), gt[mask,:], faces[id].astype(np.int32)) + save_stl(os.path.join(output_folder, f'{frame}_body.stl'), bodies[frame], F) + out.append(np.concatenate((img[:, 50:-50, :], img_unposed[:, 50:-50, :]), axis=1)) return out @@ -322,33 +330,36 @@ if __name__ == '__main__': # garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() garment_type_table = np.load('custom_data/garment_type_table_train.npy', allow_pickle=True).item() + camera_1 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=0.3, y=1.0, z=0.05)) + camera_2 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=-0.9, y=0.7, z=0.05)) - frames = ['05656_221', '00300_69', '06955_117'] + frames = ['01070_13'] + cameras = {'01070_13': camera_1} + # frames = ['05656_221', '00300_69', '06955_117'] # frames = ['02184_157', '02241_142', '06955_117', '06804_192'] # frames = ['01763_259', '007292_282', '00539_296', '00075_4'] # frames = [ '01763_259'] - camera_1 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=0.3, y=1.0, z=0.05)) - camera_2 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=-0.9, y=0.7, z=0.05)) # cameras = {'01763_259': camera_1, '007292_282': camera_1, '00539_296': camera_1, '00075_4': camera_2} - cameras = {'05656_221': camera_1, '00300_69': camera_1, '06955_117': camera_1} + # cameras = {'05656_221': camera_1, '00300_69': camera_1, '06955_117': camera_1} # cameras = {'02184_157': camera_1, '02241_142': camera_1, '06955_117': camera_1, '06804_192': camera_1} # abl_all replace abl_norm maybe - experiment = 'abl_2_pyr_32_64_128_256_512' + experiment = 'abl_pyr_32_64_128_256_512' skirt_mods = { frame.split('_')[0]: ('_skirt' if (garment_type_table[frame.split('_')[0]] == 'Skirt' or garment_type_table[frame.split('_')[0]] == 'Dress') else '') for frame in frames } data_folder = '/home/workspace/local-data/train' - data_folder_skirt = '/home/workspace/local-data/train_skirt' + data_folder_skirt = '/home/workspace/local-data_128/train_skirt' + results_folder_base = 'examples/pyramid' + + output_folder = os.path.join(results_folder_base) + os.makedirs(output_folder, exist_ok=True) + bodies = get_bodies(frames, skirt_mods) masks = get_masks(frames) faces = { frame.split('_')[0]: _maskFaces(np.zeros((14475,3)), F_skirt if skirt_mods[frame.split('_')[0]] == '_skirt' else F, masks[frame.split('_')[0]])[1] for frame in frames } gts = get_gt_image(frames, bodies, faces, cameras, skirt_mods) - results_folder_base = '/home/workspace/data/results' - - output_folder = os.path.join(results_folder_base, 'ablation_plots') - os.makedirs(output_folder, exist_ok=True) images = get_experiment_plots(frames, skirt_mods, bodies, faces, masks, cameras) diff --git a/sample_pro_2.py b/sample_pro_2.py index 1a28a2e..46d4c65 100644 --- a/sample_pro_2.py +++ b/sample_pro_2.py @@ -46,7 +46,7 @@ from models.pyramid_model_optimized import Pyramid # from configs.config_abl_overfit_dress_var_3 import config as config_main # from configs.config_abl_overfit_jumpsuit_2 import config as config_main # from configs.config_cvae_curv import config as config_curv -from configs.iccv.config_abl_skin_var import config as config_main +from configs.iccv.config_abl_all_skirt_var import config as config_main from configs.iccv.config_cvae_curv import config as config_curv # from models.cvae_model_overfit_jumpsuit import CVAE from models.cvae_model import CVAENew as CVAE @@ -289,7 +289,7 @@ def postprocess(mesh, body, faces, mask, do_smoothing=True): return mesh -def sampling_example(frame, camera, nr=10): +def sampling_example(frame, camera, nr=15): batch = get_temp_batch(frame, config_main) mask = batch['mesh_mask'][0].numpy().astype(bool) @@ -417,19 +417,25 @@ if __name__ == '__main__': args = vars(parser.parse_args()) smpl = SMPL() - F = np.load("custom_utils/smpl_faces.npy") + + if config_main['is_skirt']: + F = np.load("custom_utils/smpl_faces_skirt.npy") + folder = os.path.join('/home/workspace/local-data_128/train_skirt') + else: + F = np.load("custom_utils/smpl_faces.npy") + folder = os.path.join('/home/workspace/local-data/train') # F = np.load("custom_utils/smpl_faces_skirt.npy") F_body = np.load("custom_utils/smpl_faces.npy") # folder = os.path.join('/home/workspace/data/dataset', ('test_skirt_ready_2' if is_skirt else 'test_ready')) # folder = os.path.join('/home/workspace/local-data/train_skirt') - folder = os.path.join('/home/workspace/local-data/train') model = CVAE(config_main) - curv_model = CVAECurv(config_curv) + # curv_model = CVAECurv(config_curv) load_model(model, 'checkpoints/iccv/' + config_main['name']) - load_model(curv_model, 'checkpoints/iccv/' + config_curv['name']) + # load_model(curv_model, 'checkpoints/iccv/' + config_curv['name']) + camera_1 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=0.3, y=1.0, z=0.05)) # faces_folder = '/home/workspace/data/results/support/faces' @@ -464,10 +470,12 @@ if __name__ == '__main__': # frame = '04925_272' # frames = ['01070_13', '02276_232', '02978_242', '04008_295', '04173_193'] - # out_folder = 'examples/sampling' - # os.makedirs(out_folder, exist_ok=True) - # for frame in tqdm.tqdm(frames): - # sampling_example(frame, camera_1) + frames = ['00214_111', '01661_210', '02144_249', '02151_37', '02930_275', '03812_3'] + + out_folder = 'examples/sampling_skirt' + os.makedirs(out_folder, exist_ok=True) + for frame in tqdm.tqdm(frames): + sampling_example(frame, camera_1) # sampling_poses(frame, camera_01082_109) # sampling_garment(frame, camera_1)