diff --git a/aaa_helpers.txt b/aaa_helpers.txt index 8706dad..4994f34 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -23,6 +23,8 @@ nohup python cvae.py --name="lhunor_abl_all" > nohup/iccv/ablation/abl_all_3.out nohup python cvae.py --name="lhunor_abl_all_gnn" > nohup/iccv/ablation/abl_all_gnn.out 2>&1 & nohup python cvae.py --name="lhunor_abl_all_gnn_2" > nohup/iccv/ablation/abl_all_gnn.out 2>&1 & +nohup python cvae.py --name="lhunor_abl_all_cape_f" > nohup/iccv/ablation/abl_all_cape_f.out 2>&1 & + nohup python cvae.py --name="lhunor_abl_all_skirt" > nohup/iccv/ablation/abl_all_skirt.out 2>&1 & nohup python cvae.py --name="lhunor_abl_curv" > nohup/iccv/ablation/abl_curv.out 2>&1 & nohup python cvae.py --name="lhunor_abl_curv_skirt" > nohup/iccv/ablation/abl_curv_skirt.out 2>&1 & @@ -80,10 +82,19 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_reg_skirt" > nohup/ic nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f" > nohup/iccv/pyr/iccv_pyr_64_cape_f.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_m" > nohup/iccv/pyr/iccv_pyr_64_cape_m.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_m_00032_SL" > nohup/iccv/pyr/iccv_pyr_64_cape_m_00032_SL.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f_03223_SL" > nohup/iccv/pyr/iccv_pyr_64_cape_f_03223_SL.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f_03223_SS" > nohup/iccv/pyr/iccv_pyr_64_cape_f_03223_SS.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_128_cape_f" > nohup/iccv/pyr/iccv_pyr_64_128_cape_f.out 2>&1 & + +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_pre_cape_f" > nohup/iccv/pyr/iccv_pyr_64_pre_cape_f.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128" > nohup/iccv/pyr/iccv_pyr_32_64_128.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skirt.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_reg.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256" > nohup/iccv/pyr/iccv_pyr_32_64_128_256.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32 > nohup/iccv/evaluate/iccv_pyr_32.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_skirt > nohup/iccv/evaluate/iccv_pyr_32_skirt.out 2>&1 & @@ -95,8 +106,14 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_curv > nohup/ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_curv_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64 > nohup/iccv/evaluate/iccv_pyr_32_64.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_skin > nohup/iccv/evaluate/iccv_pyr_32_64_skin.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_reg > nohup/iccv/evaluate/iccv_pyr_32_64_reg.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv > nohup/iccv/evaluate/iccv_pyr_32_64_curv.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_curv_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index 4657faa..9f13451 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_skirt import config +from configs.iccv.config_abl_pyr_32_64_skin import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] # import tensorflow as tf diff --git a/configs/iccv/config_abl_all_cape_f.py b/configs/iccv/config_abl_all_cape_f.py new file mode 100644 index 0000000..6f76c16 --- /dev/null +++ b/configs/iccv/config_abl_all_cape_f.py @@ -0,0 +1,125 @@ +config = dict() + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK AVG GARMENT USED + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_curvature'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = True + +config['use_norm_loss'] = True + +ablation_count = [config['use_skinning'], config['remove_global_orientation'], config['use_curvature'], config['use_extra_losses'], config['use_offsets']].count(False) +assert ablation_count == 1 or ablation_count == 0 + +config['gpu'] = '6' +config['name'] = 'abl_all_cape_f' +config['notes'] = '' +config['tags'] = ['iccv', 'abl_all'] + +config['avg_garment_path'] = 'custom_data/train_cape_f_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv train_no_shape_one_hot_avg_uv +if config['use_skinning']: + assert (config['use_offsets'] and config['avg_garment_path'] != '') or (not config['use_offsets'] and config['avg_garment_path'] == '') + +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision + +config['batch_size'] = 4 +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = False +config['is_cape'] = True + +config['train_dir'] = '/home/workspace/local-data/cape_processed/train/female' +config['valid_dir'] = '/home/workspace/local-data/cape_processed/test/female' +config['test_dir'] = '/home/workspace/local-data/cape_processed/test/female' + +config['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_f' +config['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_f' +config['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_f' + + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['pretrained_model'] = '' + +############### +config['lr'] = 0.0001# 0.0001 +############### +config['starting_balancing_coeff'] = 0.0001 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +if config['use_skinning']: + config['signature_dynamic']['uv_unposed'] = (512, 256, 3) + config['signature_dynamic']['mesh_unposed'] = (config['smpl_res'], 3,) +else: + config['signature_dynamic']['uv_posed'] = (512, 256, 3) + + +config['upscale_enhancer'] = '' +config['freeze_encoders'] = True +config['use_one_hot'] = False +config['bottleneck_scale'] = 2 + +config['cvae'] = dict() +config['cvae']['is_label'] = False +config['cvae']['latent_dim'] = 64 * config['bottleneck_scale'] +config['cvae']['add_noise'] = False +config['cvae']['noise_amount'] = 0.0 + +config['cvae']['kernel_size'] = 7 +config['cvae']['encoder_depths'] = [3, 3, 3, 9, 3] +config['cvae']['encoder_dims'] = [32, 64, 128, 256, config['cvae']['latent_dim']] +config['cvae']['decoder_depths'] = config['cvae']['encoder_depths'] +config['cvae']['decoder_dims'] = config['cvae']['encoder_dims'] + + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128.py b/configs/iccv/config_abl_pyr_32_64_128.py index e81e57f..5a8ba83 100644 --- a/configs/iccv/config_abl_pyr_32_64_128.py +++ b/configs/iccv/config_abl_pyr_32_64_128.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '3' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_128_2' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/data/dataset/test_ready' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False diff --git a/configs/iccv/config_abl_pyr_32_64_128_256.py b/configs/iccv/config_abl_pyr_32_64_128_256.py new file mode 100644 index 0000000..4444cd1 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_256.py @@ -0,0 +1,188 @@ +config = dict() + +config['gpu'] = '3' +config['name'] = 'iccv_pyr_32_64_128_256' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128, 256] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_128_2' +config['pyramid']['pretrained_pyramid_iteration'] = '125000' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_256_skirt.py new file mode 100644 index 0000000..6213a79 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_256_skirt.py @@ -0,0 +1,188 @@ +config = dict() + +config['gpu'] = '0' +config['name'] = 'iccv_pyr_32_64_128_256_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128, 256] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_128_skirt' +config['pyramid']['pretrained_pyramid_iteration'] = '175000' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir'] = '/home/workspace/local-data/valid_skirt' +config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_reg.py b/configs/iccv/config_abl_pyr_32_64_128_reg.py new file mode 100644 index 0000000..0efedd4 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_reg.py @@ -0,0 +1,189 @@ +config = dict() + +config['gpu'] = '4' +config['name'] = 'iccv_pyr_32_64_128_reg' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'reg'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = False # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = False +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_reg' +config['pyramid']['pretrained_pyramid_iteration'] = '120000' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_skin.py b/configs/iccv/config_abl_pyr_32_64_128_skin.py new file mode 100644 index 0000000..32e2150 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_skin.py @@ -0,0 +1,196 @@ +config = dict() + +config['gpu'] = '5' +config['name'] = 'iccv_pyr_32_64_128_skin' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = False # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_skin' +config['pyramid']['pretrained_pyramid_iteration'] = '130000' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_127/train' +config['valid_dir'] = '/home/workspace/local-data_127/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +if config['use_skinning']: + config['signature_dynamic']['uv_unposed'] = (512, 256, 3) + config['signature_dynamic']['mesh_unposed'] = (config['smpl_res'], 3,) +else: + config['signature_dynamic']['uv_posed'] = (512, 256, 3) +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + if config['use_skinning']: + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + else: + config['signature_dynamic']['uv_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_skirt.py index 09d3c53..e22a499 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_skirt.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '0' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_128_skirt' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_64_curv.py b/configs/iccv/config_abl_pyr_32_64_curv.py index c33a985..412f89c 100644 --- a/configs/iccv/config_abl_pyr_32_64_curv.py +++ b/configs/iccv/config_abl_pyr_32_64_curv.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '4' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_curv' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'curv'] @@ -33,7 +33,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/data/dataset/test_ready' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False diff --git a/configs/iccv/config_abl_pyr_32_64_curv_skirt.py b/configs/iccv/config_abl_pyr_32_64_curv_skirt.py index c4ec776..775a481 100644 --- a/configs/iccv/config_abl_pyr_32_64_curv_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_curv_skirt.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '6' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_curv_skirt' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'curv', 'skirt'] @@ -33,7 +33,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train_skirt' config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' -config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' config['batch_size'] = 4 config['is_skirt'] = True config['is_cape'] = False diff --git a/configs/iccv/config_abl_pyr_32_64_reg.py b/configs/iccv/config_abl_pyr_32_64_reg.py index b463fe1..96617fa 100644 --- a/configs/iccv/config_abl_pyr_32_64_reg.py +++ b/configs/iccv/config_abl_pyr_32_64_reg.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '5' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_reg' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'reg'] diff --git a/configs/iccv/config_abl_pyr_32_64_skin.py b/configs/iccv/config_abl_pyr_32_64_skin.py index ec77673..ab95873 100644 --- a/configs/iccv/config_abl_pyr_32_64_skin.py +++ b/configs/iccv/config_abl_pyr_32_64_skin.py @@ -28,11 +28,11 @@ config['pretrained_model_iteration'] = '' config['pyramid'] = dict() config['pyramid']['resolutions'] = [32, 64] config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_skin' -config['pyramid']['pretrained_pyramid_iteration'] = '200000' +config['pyramid']['pretrained_pyramid_iteration'] = '235000' config['pyramid']['final_upscale_enhancer'] = '' -config['train_dir'] = '/home/workspace/local-data_127/train' -config['valid_dir'] = '/home/workspace/local-data_127/valid' +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' config['test_dir'] = '/home/workspace/local-data_127/test_ready' config['batch_size'] = 4 config['is_skirt'] = False diff --git a/configs/iccv/config_abl_pyr_64_128_cape_f.py b/configs/iccv/config_abl_pyr_64_128_cape_f.py new file mode 100644 index 0000000..c152bb5 --- /dev/null +++ b/configs/iccv/config_abl_pyr_64_128_cape_f.py @@ -0,0 +1,189 @@ +config = dict() + +config['gpu'] = '5' +config['name'] = 'iccv_pyr_64_128_cape_f' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'cape_f'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_64_cape_f' +config['pyramid']['pretrained_pyramid_iteration'] = '140000' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/cape_processed/train/female' +config['valid_dir'] = '/home/workspace/local-data/cape_processed/test/female' +config['test_dir'] = '/home/workspace/local-data/cape_processed/test/female' +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = True + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_f' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_f' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_f' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_cape_f'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_cape_f'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_cape_f'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_64_cape_f_03223_SL.py b/configs/iccv/config_abl_pyr_64_cape_f_03223_SL.py index e43eb28..3db9194 100644 --- a/configs/iccv/config_abl_pyr_64_cape_f_03223_SL.py +++ b/configs/iccv/config_abl_pyr_64_cape_f_03223_SL.py @@ -1,7 +1,7 @@ config = dict() -config['gpu'] = '2' -config['name'] = 'iccv_pyr_64_cape_f' +config['gpu'] = '3' +config['name'] = 'iccv_pyr_64_cape_f_03223_SL' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'cape_f'] diff --git a/custom_data/train_cape_f_no_shape_avg_uv.npy b/custom_data/train_cape_f_no_shape_avg_uv.npy new file mode 100644 index 0000000..27ac45f Binary files /dev/null and b/custom_data/train_cape_f_no_shape_avg_uv.npy differ diff --git a/custom_data/train_cape_m_no_shape_avg_uv.npy b/custom_data/train_cape_m_no_shape_avg_uv.npy new file mode 100644 index 0000000..59df8c1 Binary files /dev/null and b/custom_data/train_cape_m_no_shape_avg_uv.npy differ diff --git a/custom_utils/custom_utils.py b/custom_utils/custom_utils.py index 4aed3a4..1f180bf 100644 --- a/custom_utils/custom_utils.py +++ b/custom_utils/custom_utils.py @@ -292,7 +292,7 @@ def sparse_to_tensor(L): return tf.SparseTensor(idx, data, shape) -def load_model(model, checkpoint, disable_assert=False, assert_consumed_on=False, iteration=''): +def load_model(model, checkpoint, disable_assert=False, assert_consumed_on=False, iteration='', return_iteration=False): ckpt = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(ckpt, checkpoint, max_to_keep=1) if not disable_assert: @@ -306,7 +306,14 @@ def load_model(model, checkpoint, disable_assert=False, assert_consumed_on=False ckpt_ind = [i for i in range(len(manager.checkpoints)) if manager.checkpoints[i].endswith(name_base + '-' + iteration)][0] ckpt.restore(manager.checkpoints[ckpt_ind]).assert_existing_objects_matched()#.assert_consumed() else: - ckpt.restore(manager.latest_checkpoint).assert_existing_objects_matched()#.assert_consumed() + if not return_iteration: + ckpt.restore(manager.latest_checkpoint).assert_existing_objects_matched()#.assert_consumed() + else: + ckpt.restore(manager.latest_checkpoint).assert_existing_objects_matched() + if return_iteration: + iteration = manager.latest_checkpoint.split('-')[-1] + return iteration + def get_summary_str(model): @@ -425,3 +432,14 @@ class Accumulator(): def reset(self): self.dict = dict() + + +from stl import mesh as mesh_stl + +def save_stl(file, mesh, faces): + obj = mesh_stl.Mesh(np.zeros(faces.shape[0], dtype=mesh_stl.Mesh.dtype)) + for i, f in enumerate(faces): + for j in range(3): + obj.vectors[i][j] = mesh[f[j],:] + + obj.save(file) \ No newline at end of file diff --git a/cvae.py b/cvae.py index 062f532..7335287 100644 --- a/cvae.py +++ b/cvae.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_skin_var import config +from configs.iccv.config_abl_all_cape_f import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/cvae_pyramid.py b/cvae_pyramid.py index dc4d3a6..e1a7811 100644 --- a/cvae_pyramid.py +++ b/cvae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128 import config +from configs.iccv.config_abl_pyr_64_pre_cape_f import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/dataset.py b/dataset.py index d2ee42d..95f43eb 100644 --- a/dataset.py +++ b/dataset.py @@ -381,18 +381,18 @@ if __name__ == '__main__': # 'uv_shaped_body': (3, 512, 256,), } - dataset = factory.get_dataset(folder='/home/workspace/local-data/train_skirt', - batch_size=1, + dataset = factory.get_dataset(folder='/home/workspace/local-data/cape_processed/train/male', + batch_size=10, signature=signature_dynamic, resolutions=[512], do_repeat=False, use_one_hot=False, - is_cape=False, - frames_file='frames_dress_30k.txt') - static_dataset = StaticDataset(folder='/home/workspace/local-data/train_skirt', + is_cape=True) + # frames_file='frames_dress_30k.txt' + static_dataset = StaticDataset(folder='/home/workspace/local-data/cape_processed/train/male', signature=signature_static, resolutions=[512], - is_cape=False) + is_cape=True) # for i, batch in enumerate(dataset): @@ -404,7 +404,7 @@ if __name__ == '__main__': for i, batch in enumerate(dataset): print(i) batch = static_dataset.expand_batch(batch_to_dict(batch, signature_dynamic)) - ex.imshow(batch['uv_curvature'][0].numpy().argmax(-1) / 6.0).show() + # ex.imshow(batch['uv_curvature'][0].numpy().argmax(-1) / 6.0).show() sum_uv += batch['uv_unposed'].numpy().sum(axis=0) sum_mask += batch['uv_mask'].numpy().sum(axis=0) @@ -425,7 +425,7 @@ if __name__ == '__main__': k = F[:,2] )]).show() - np.save('custom_data/train_dress_30k_no_shape_avg_uv.npy', avg_garment) + np.save('custom_data/train_cape_m_no_shape_avg_uv.npy', avg_garment) print() diff --git a/downscale.py b/downscale.py index 66c07ea..a5d6736 100644 --- a/downscale.py +++ b/downscale.py @@ -8,7 +8,7 @@ import numpy as np import plotly.graph_objects as go import plotly.express as ex -os.environ["CUDA_VISIBLE_DEVICES"] = "6" +os.environ["CUDA_VISIBLE_DEVICES"] = "4" gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: @@ -197,7 +197,7 @@ if __name__ == '__main__': sizes = [32, 64, 128, 256] folder = args.folder - is_skirt = True + is_skirt = False uv_masks_body = dict() uv_mask_body = np.load('custom_utils/mask_256x512' + ('_skirt' if is_skirt else '') + '.npy') uv_mask_body = cv2.dilate(uv_mask_body.astype(np.float32), cv2.getStructuringElement(cv2.MORPH_RECT, (2 * 2 + 1, 2 * 2 + 1), (2, 2)), iterations=2) diff --git a/evaluate_ae_pyramid.py b/evaluate_ae_pyramid.py index 638c0ca..ab1a1bb 100644 --- a/evaluate_ae_pyramid.py +++ b/evaluate_ae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_var import config +from configs.iccv.config_abl_pyr_64_128_cape_f import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -93,13 +93,15 @@ if __name__ == '__main__': F = np.load("custom_utils/smpl_faces" + skirt_modifier + ".npy").astype(np.int32) - ids = os.listdir(os.path.join(folder, 'pose_shape')) - ids = [id.split('_')[0] for id in ids] - ids = set(ids) + # ids = os.listdir(os.path.join(folder, 'pose_shape')) + # ids = [id.split('_')[0] for id in ids] + # ids = set(ids) - iteration = '235000' - - load_model(model, 'checkpoints/iccv/' + checkpoint_name, iteration=iteration) + iteration = '' + if iteration == '': + iteration = load_model(model, 'checkpoints/iccv/' + checkpoint_name, return_iteration=True) + else: + load_model(model, 'checkpoints/iccv/' + checkpoint_name, iteration=iteration) # load_model(model, 'checkpoints/' + 'abl_all_120k') output_folder = '/home/workspace/local-data_128/results/iccv/' + checkpoint_name + '-' + iteration diff --git a/evaluate_cape.py b/evaluate_cape.py index 5b4b163..fa543e3 100644 --- a/evaluate_cape.py +++ b/evaluate_cape.py @@ -1,5 +1,5 @@ import os -from configs.config_abl_2_pyr_64_128_256_512_cape_m import config +from configs.iccv.config_abl_pyr_64_128_cape_f import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -41,7 +41,7 @@ from losses import collision_loss as collision_penalty from custom_utils.custom_utils import batch_to_dict, load_model from dataset import DatasetFactory, StaticDataset from custom_utils.smpl import SMPL -from models.pyramid_model_optimized import Pyramid +from models.pyramid_model_optimized import PyramidNew as Pyramid def init_wandb(): @@ -324,7 +324,7 @@ def plot_everything(batch: dict, type: str): def test(): - load_model(model, 'checkpoints/' + config['name']) # , iteration='60000' + load_model(model, 'checkpoints/iccv/' + config['name']) # , iteration='60000' clothing_verts_idx = np.load(os.path.join('CAPE', 'data', 'clothing_verts_idx.npy')) cape_mask = np.zeros((6890), bool) cape_mask[clothing_verts_idx] = True @@ -386,7 +386,7 @@ def test(): print('min: ' + str(errors.min())) print('max: ' + str(errors.max())) print('median: ' + str(np.median(errors))) - np.save('temp/error_'+ config['name'] +'.npy', errors) + np.save('temp/error_iccv_'+ config['name'] +'.npy', errors) # histograms = dict() # for key in test_loss.keys(): diff --git a/get_error.py b/get_error.py index 333aab4..2ed3d76 100644 --- a/get_error.py +++ b/get_error.py @@ -7,10 +7,10 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() folder = '/home/workspace/data/results/errors/iccv' - checkpoint_name = 'iccv_pyr_32_64' - checkpoint_name_skirt = 'iccv_pyr_32_64_skirt' - iteration = '200000' - iteration_skirt : str = '235000' + checkpoint_name = 'iccv_pyr_32_64_curv' + checkpoint_name_skirt = 'iccv_pyr_32_64_curv_skirt' + iteration = '100000' + iteration_skirt : str = '60000' # checkpoint_name = 'abl_skin' # checkpoint_name_skirt = checkpoint_name + '_skirt' @@ -53,7 +53,7 @@ if __name__ == '__main__': print('Global tshirt avg: ' + str(cum_avg_shirt)) print('Global top avg: ' + str(cum_avg_top)) - print('\nGlobl per garment: ') + print('\nGlobal per garment: ') global_errors = [] per_garment_errors['Tshirt'].extend(per_garment_errors_skirt['Tshirt']) diff --git a/plot_ablation.py b/plot_ablation.py index 032c859..3b261cd 100644 --- a/plot_ablation.py +++ b/plot_ablation.py @@ -17,7 +17,7 @@ from scipy.spatial.transform import Rotation import tps from dataset import DatasetFactory, StaticDataset -from custom_utils.custom_utils import batch_to_dict, UV2Mesh, _maskFaces, load_model +from custom_utils.custom_utils import batch_to_dict, UV2Mesh, _maskFaces, load_model, save_stl from custom_utils.smpl import SMPL from convert_to_uv_api import convert_to_uv_api from models.upscaler import UpscalerBasicConv as Upscaler @@ -92,7 +92,7 @@ def get_faces(): def get_bodies(frames, skirt_mods): - folder_base = '/home/workspace/data/dataset' + folder_base = '/home/workspace/local-data' out = dict() for frame in tqdm.tqdm(frames): id = frame.split('_')[0] @@ -128,7 +128,7 @@ def get_experiment_plots(folder, frames, experiments, skirt_mods, bodies, faces, frame_out = [] id = frame.split('_')[0] for exp in experiments: - garment = np.load(os.path.join(folder, exp + skirt_mods[id] + '-70000', frame + '.npy')) + garment = np.load(os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy')) if exp == 'abl_orient': garment = deorient(garment, frame, skirt_mods[id] == '_skirt') garment = postprocess(garment, bodies[frame], id, skirt_mods[id] == '_skirt') @@ -141,6 +141,20 @@ def get_experiment_plots(folder, frames, experiments, skirt_mods, bodies, faces, return out +def get_experiment_meshes(folder, frames, experiments, skirt_mods, bodies, faces, cameras): + out = [] + for frame in tqdm.tqdm(frames): + frame_out = [] + id = frame.split('_')[0] + for exp in experiments: + garment = np.load(os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy')) + if exp == 'abl_orient': + garment = deorient(garment, frame, skirt_mods[id] == '_skirt') + garment = postprocess(garment, bodies[frame], id, skirt_mods[id] == '_skirt') + frame_out.append([garment, bodies[frame], faces[id]]) + + out.append(frame_out) + return out def get_gt_image(frames, bodies, faces, cameras, skirt_mods): out = [] @@ -155,6 +169,17 @@ def get_gt_image(frames, bodies, faces, cameras, skirt_mods): return out + +def get_gt_meshes(frames, bodies, faces, cameras, skirt_mods): + out = [] + for frame in tqdm.tqdm(frames): + id = frame.split('_')[0] + mask = np.load(os.path.join('/home/workspace/data/dataset', ('test_skirt_ready_2' if skirt_mods[id] == '_skirt' else 'test_ready'), 'mesh_mask', id + '.npy')) + gt = np.load(os.path.join('/home/workspace/data/dataset', ('test_skirt_ready_2' if skirt_mods[id] == '_skirt' else 'test_ready'), 'mesh_posed_no_orient', frame + '.npz'))['mesh_posed_no_orient'].astype(np.float32) + out.append([gt[mask,:], bodies[frame], faces[id]]) + + return out + if __name__ == '__main__': # uv2mesh = UV2Mesh() @@ -165,16 +190,16 @@ if __name__ == '__main__': faces = get_faces() - frames = [] + frames = ['00079_135'] # frames = [ '00075_4', '00475_273', '00761_295', '00793_3', '00993_220', '01134_112', '01278_108', '00718_38', '00951_34', '00962_60','01065_38','01191_83','01240_168'] - rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')] - random.shuffle(rand_frames) - rand_frames = rand_frames[:25] - frames.extend(rand_frames) - rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape')] - random.shuffle(rand_frames) - rand_frames = rand_frames[:25] - frames.extend(rand_frames) + # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')] + # random.shuffle(rand_frames) + # rand_frames = rand_frames[:25] + # frames.extend(rand_frames) + # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape')] + # random.shuffle(rand_frames) + # rand_frames = rand_frames[:25] + # frames.extend(rand_frames) camera_1 = dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=-0.05), eye=dict(x=0.3, y=1.0, z=0.05)) @@ -182,23 +207,34 @@ if __name__ == '__main__': cameras = {frame: camera_1 for frame in frames} # abl_all replace abl_norm maybe - experiments = ['abl_norm', 'abl_skin', 'abl_curv', 'abl_extra_loss', 'abl_abs'] + experiments = [['iccv_pyr_32', '335000'], ['iccv_pyr_32_skin', '330000'], ['iccv_pyr_32_curv', '155000'], ['iccv_pyr_32_reg', '150000']] # , 'pyra_32_res' skirt_mods = { frame.split('_')[0]: ('_skirt' if os.path.exists('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape/' + frame + '.npy') else '') for frame in frames } bodies = get_bodies(frames, skirt_mods) - gts = get_gt_image(frames, bodies, faces, cameras, skirt_mods) + # gts = get_gt_image(frames, bodies, faces, cameras, skirt_mods) + gts = get_gt_meshes(frames, bodies, faces, cameras, skirt_mods) - results_folder_base = '/home/workspace/data/results' + results_folder_base = '/home/workspace/data/results/iccv/' output_folder = os.path.join(results_folder_base, 'ablation_plots') os.makedirs(output_folder, exist_ok=True) - images = get_experiment_plots(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras) + # images = get_experiment_plots(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras) + meshes = get_experiment_meshes(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras) + for mesh, frame in zip(meshes, frames): + for exp, (garment, body, faces) in zip(experiments, mesh): + save_stl(f'examples/{frame}_{exp[0]}_garment.stl', garment, faces) + save_stl(f'examples/{frame}_{exp[0]}_body.stl', body, F) - rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts, images)] + for mesh, frame in zip(gts, frames): + garment, body, faces = mesh + save_stl(f'examples/{frame}_gt_garment.stl', garment, faces) + save_stl(f'examples/{frame}_gt_body.stl', body, F) - for frame, row in zip(frames, rows): - custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png')) + # rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts, images)] + + # for frame, row in zip(frames, rows): + # custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png')) # final = np.concatenate(rows, axis=0) diff --git a/plot_examples.py b/plot_examples.py index e966d15..1ed6b89 100644 --- a/plot_examples.py +++ b/plot_examples.py @@ -171,17 +171,8 @@ def plot_upscaling(uv_low, uv_high, uv_mask_low, uv_mask_high, faces, color, mas print() -from stl import mesh as mesh_stl -def save_stl(file, mesh, faces): - obj = mesh_stl.Mesh(np.zeros(faces.shape[0], dtype=mesh_stl.Mesh.dtype)) - for i, f in enumerate(faces): - for j in range(3): - obj.vectors[i][j] = mesh[f[j],:] - - obj.save(file) - -if __name__ == '__main__': + if __name__ == '__main__': uv2mesh = UV2Mesh() smpl = SMPL() diff --git a/run_parallel.py b/run_parallel.py index 3fff046..bf8bbe3 100644 --- a/run_parallel.py +++ b/run_parallel.py @@ -6,13 +6,13 @@ if __name__ == '__main__': # folder = '/home/workspace/data/dataset/test_skirt_ready_2' # folder = '/home/workspace/local-data/cape_processed/train/male' - folder = '/home/workspace/local-data/train' - # folder = '/home/workspace/local-data/valid' + # folder = '/home/workspace/local-data/train' + folder = '/home/workspace/local-data/test_ready' # ids = os.listdir(folder) ids = os.listdir(folder + '/mesh_posed_no_orient')#[280000:] skirt = 0 - count = 200000 + count = 0 step = 10000 # !!!!!!! change skirt parameter in script @@ -24,6 +24,6 @@ if __name__ == '__main__': # os.system('nohup python convert_to_uv_local_single.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_posed_no_orient_test_skirt_ready_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_posed_no_orient_test_skirt_ready_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &') # os.system('nohup python convert_to_uv_local_single.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_train_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &') # os.system('nohup python convert_to_uv_local_single.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_valid_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_valid_skirt_local_unposed_onehot_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &') - os.system('nohup python downscale.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'downscale_train_posed_no_orient_' + str(count) + '_' + str(count+step) + ' > nohup/downscale_train_posed_no_orient_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &') + os.system('nohup python downscale.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'downscale_test_posed_no_orient_' + str(count) + '_' + str(count+step) + ' > nohup/downscale_test_posed_no_orient_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &') # os.system('python Data/convert_to_uv.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + '') count += step