From 5857aceecd95b9e474719438b112b50ae6aa13b9 Mon Sep 17 00:00:00 2001 From: Hunor Laczko Date: Tue, 14 Mar 2023 11:13:22 +0000 Subject: [PATCH] save --- aaa_helpers.txt | 9 +- calculate_error.py | 17 +- .../iccv/config_abl_pyr_32_64_128_jumpsuit.py | 2 +- .../config_abl_pyr_32_64_128_skirt_skirt.py | 2 +- configs/iccv/config_abl_pyr_32_64_128_top.py | 4 +- .../config_abl_pyr_32_64_128_top_skirt.py | 197 ++++++++++++++++++ .../iccv/config_abl_pyr_32_64_128_trousers.py | 2 +- .../iccv/config_abl_pyr_32_64_128_tshirt.py | 2 +- .../config_abl_pyr_32_64_128_tshirt_skirt.py | 196 +++++++++++++++++ evaluate_ae_pyramid.py | 2 +- get_error.py | 46 ++-- sample_pro_2.py | 79 +++---- 12 files changed, 487 insertions(+), 71 deletions(-) create mode 100644 configs/iccv/config_abl_pyr_32_64_128_top_skirt.py create mode 100644 configs/iccv/config_abl_pyr_32_64_128_tshirt_skirt.py diff --git a/aaa_helpers.txt b/aaa_helpers.txt index f7c107c..24ff44f 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -172,13 +172,20 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv_skirt nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_10p > nohup/iccv/evaluate/iccv_pyr_32_64_10p.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_25p > nohup/iccv/evaluate/iccv_pyr_32_64_25p.out 2>&1 & -nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohcup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skin > nohup/iccv/evaluate/iccv_pyr_32_64_128_skin.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skin_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skin_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_reg > nohup/iccv/evaluate/iccv_pyr_32_64_128_reg.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv > nohup/iccv/evaluate/iccv_pyr_32_64_128_curv.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_curv_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_trousers > nohup/iccv/evaluate/iccv_pyr_32_64_128_trousers.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_tshirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_tshirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_top > nohup/iccv/evaluate/iccv_pyr_32_64_128_top.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_jumpsuit > nohup/iccv/evaluate/iccv_pyr_32_64_128_jumpsuit.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt_dress> nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt_dress.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256 > nohup/iccv/evaluate/iccv_pyr_32_64_128_256.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_256_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_256_skirt.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index c08f03d..c86f13e 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config +from configs.iccv.config_abl_pyr_32_64_128_tshirt import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -58,7 +58,7 @@ if __name__ == '__main__': # smpl = SMPL(use_one_hot=config['use_one_hot']) skirt_modifier = '_skirt' if config['is_skirt'] else '' - error_type = 'collision' # 'euclidean' 'collision' edge normal + error_type = 'euclidean' # 'euclidean' 'collision' edge normal folder = config['test_dir'] @@ -92,9 +92,16 @@ if __name__ == '__main__': frames = [frame for frame in frames if not frame.endswith('_fixed_coll.npy')] # frames = frames[:100] - print('Reading bodies...') - bodies_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_posed_body_no_orient' - bodies = {frame : np.load(bodies_folder + '/' + frame[:-1] + 'z')['mesh_posed_body_no_orient'] for frame in tqdm(frames) } + # if 'frames_file_test' in config.keys(): + # with open(os.path.join(folder, config['frames_file_test']), 'r') as f: + # lines = f.readlines() + # lines = [line.strip() for line in lines] + # frames = lines + + if error_type == 'collision': + print('Reading bodies...') + bodies_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_posed_body_no_orient' + bodies = {frame : np.load(bodies_folder + '/' + frame[:-1] + 'z')['mesh_posed_body_no_orient'] for frame in tqdm(frames) } if config['remove_global_orientation']: gt_name = 'mesh_posed_no_orient' diff --git a/configs/iccv/config_abl_pyr_32_64_128_jumpsuit.py b/configs/iccv/config_abl_pyr_32_64_128_jumpsuit.py index 8eae665..db43e98 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_jumpsuit.py +++ b/configs/iccv/config_abl_pyr_32_64_128_jumpsuit.py @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['frames_file'] = 'frames_jumpsuit.txt' config['frames_file_valid'] = 'frames_jumpsuit.txt' config['frames_file_test'] = 'frames_jumpsuit.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_128_skirt_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_skirt_skirt.py index 37ec255..22d2156 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_skirt_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_skirt_skirt.py @@ -31,7 +31,7 @@ config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_skirt config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train_skirt' -config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['valid_dir'] = '/home/workspace/local-data/valid_skirt' config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' config['frames_file'] = 'frames_skirt.txt' config['frames_file_valid'] = 'frames_skirt.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_128_top.py b/configs/iccv/config_abl_pyr_32_64_128_top.py index 5c02bde..29188d0 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_top.py +++ b/configs/iccv/config_abl_pyr_32_64_128_top.py @@ -32,10 +32,10 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' -config['test_dir_skirt'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['test_dir_skirt'] = '/home/workspace/local-data/test_skirt_ready_2' config['frames_file'] = 'frames_top.txt' config['frames_file_valid'] = 'frames_top.txt' config['frames_file_test'] = 'frames_top.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_128_top_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_top_skirt.py new file mode 100644 index 0000000..72de796 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_top_skirt.py @@ -0,0 +1,197 @@ +config = dict() + +config['gpu'] = '3' +config['name'] = 'iccv_pyr_32_64_128_top_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_top' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' +config['test_dir_skirt'] = '/home/workspace/local-data/test_skirt_ready_2' +config['frames_file'] = 'frames_top.txt' +config['frames_file_valid'] = 'frames_top.txt' +config['frames_file_test'] = 'frames_top.txt' +config['frames_file_skirt'] = 'frames_top.txt' +config['frames_file_valid_skirt'] = 'frames_top.txt' +config['frames_file_test_skirt'] = 'frames_top.txt' + + +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_trousers.py b/configs/iccv/config_abl_pyr_32_64_128_trousers.py index fcb388f..0912a8d 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_trousers.py +++ b/configs/iccv/config_abl_pyr_32_64_128_trousers.py @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['frames_file'] = 'frames_trousers.txt' config['frames_file_valid'] = 'frames_trousers.txt' config['frames_file_test'] = 'frames_trousers.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_128_tshirt.py b/configs/iccv/config_abl_pyr_32_64_128_tshirt.py index 1fb943c..f931f83 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_tshirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_tshirt.py @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' config['test_dir_skirt'] = '/home/workspace/local-data_128/test_skirt_ready_2' diff --git a/configs/iccv/config_abl_pyr_32_64_128_tshirt_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_tshirt_skirt.py new file mode 100644 index 0000000..aad509b --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_tshirt_skirt.py @@ -0,0 +1,196 @@ +config = dict() + +config['gpu'] = '2' +config['name'] = 'iccv_pyr_32_64_128_tshirt_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_tshirt' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' +config['test_dir_skirt'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['frames_file'] = 'frames_tshirt.txt' +config['frames_file_valid'] = 'frames_tshirt.txt' +config['frames_file_test'] = 'frames_tshirt.txt' +config['frames_file_skirt'] = 'frames_tshirt.txt' +config['frames_file_valid_skirt'] = 'frames_tshirt.txt' +config['frames_file_test_skirt'] = 'frames_tshirt.txt' + +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/evaluate_ae_pyramid.py b/evaluate_ae_pyramid.py index a582d20..ef34b84 100644 --- a/evaluate_ae_pyramid.py +++ b/evaluate_ae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config +from configs.iccv.config_abl_pyr_32_64_128_tshirt_skirt import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/get_error.py b/get_error.py index 5379a97..fd488a9 100644 --- a/get_error.py +++ b/get_error.py @@ -8,11 +8,11 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() folder = '/home/workspace/data/results/errors/iccv' - checkpoint_name = 'iccv_pyr_32_64_128_skin' - checkpoint_name_skirt = 'iccv_pyr_32_64_128_skin_skirt' - iteration = '75' + '000' - iteration_skirt : str = '155' + '000' - error_type = 'collision' # 'euclidean' 'collision' edge normal + checkpoint_name = 'iccv_pyr_32_64_128_2' + checkpoint_name_skirt = 'iccv_pyr_32_64_128_skirt_dress' + iteration = '125' + '000' + iteration_skirt : str = '175' + '000' + error_type = '-euclidean' # '-euclidean' '-collision' -edge -normal errors = [] @@ -21,7 +21,7 @@ if __name__ == '__main__': per_garment_errors_skirt = { 'Tshirt': [], 'Dress': [], 'Skirt': [], 'Top': [] } - # with open(os.path.join(folder, f'{checkpoint_name}-{iteration}-{error_type}.csv'), 'r') as f: + # with open(os.path.join(folder, f'{checkpoint_name}-{iteration}{error_type}.csv'), 'r') as f: # csv_reader = csv.reader(f, delimiter=',') # for row in csv_reader: # per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) @@ -34,7 +34,7 @@ if __name__ == '__main__': # print('\nAvg non-skirt: ' + str(np.mean(errors))) - with open(os.path.join(folder, f'{checkpoint_name_skirt}-{iteration_skirt}-{error_type}.csv'), 'r') as f: + with open(os.path.join(folder, f'{checkpoint_name_skirt}-{iteration_skirt}{error_type}.csv'), 'r') as f: csv_reader = csv.reader(f, delimiter=',') for row in csv_reader: per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) @@ -47,24 +47,24 @@ if __name__ == '__main__': print('Avg skirt: ' + str(np.mean(errors_skirt))) - # cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt']) - # cum_avg_top = np.mean(per_garment_errors['Top'] + per_garment_errors_skirt['Top']) + cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt']) + cum_avg_top = np.mean(per_garment_errors['Top'] + per_garment_errors_skirt['Top']) - # print('Global tshirt avg: ' + str(cum_avg_shirt)) - # print('Global top avg: ' + str(cum_avg_top)) + print('Global tshirt avg: ' + str(cum_avg_shirt)) + print('Global top avg: ' + str(cum_avg_top)) - # print('\nGlobal per garment: ') + print('\nGlobal per garment: ') - # global_errors = [] - # per_garment_errors['Tshirt'].extend(per_garment_errors_skirt['Tshirt']) - # per_garment_errors['Top'].extend(per_garment_errors_skirt['Top']) - # per_garment_errors['Skirt'] = per_garment_errors_skirt['Skirt'] - # per_garment_errors['Dress'] = per_garment_errors_skirt['Dress'] - # for key in per_garment_errors.keys(): - # avg = np.mean(per_garment_errors[key]) - # global_errors.append(avg) - # print(key + ': ' + str(avg), 'samples: ' + str(len(per_garment_errors[key]))) + global_errors = [] + per_garment_errors['Tshirt'].extend(per_garment_errors_skirt['Tshirt']) + per_garment_errors['Top'].extend(per_garment_errors_skirt['Top']) + per_garment_errors['Skirt'] = per_garment_errors_skirt['Skirt'] + per_garment_errors['Dress'] = per_garment_errors_skirt['Dress'] + for key in per_garment_errors.keys(): + avg = np.mean(per_garment_errors[key]) + global_errors.append(avg) + print(key + ': ' + str(avg), 'samples: ' + str(len(per_garment_errors[key]))) - # print('Global avg: ' + str(np.mean(global_errors))) - # print(len(global_errors)) + print('Global avg: ' + str(np.mean(global_errors))) + print(len(global_errors)) \ No newline at end of file diff --git a/sample_pro_2.py b/sample_pro_2.py index 984191b..da516b7 100644 --- a/sample_pro_2.py +++ b/sample_pro_2.py @@ -41,13 +41,13 @@ from custom_utils.smpl import SMPL from convert_to_uv_api import convert_to_uv_api from models.upscaler import UpscalerBasicConv as Upscaler from postprocessing import fix_collision -from models.pyramid_model_optimized import Pyramid +# from models.pyramid_model_optimized import Pyramid # from configs.config_abl_overfit_dress_var_3 import config as config_main # from configs.config_abl_overfit_jumpsuit_2 import config as config_main # from configs.config_cvae_curv import config as config_curv # from configs.iccv.config_abl_all_skirt_var import config as config_main -from configs.iccv.config_abl_skin_var import config as config_main +from configs.iccv.config_abl_all_skirt import config as config_main from configs.iccv.config_cvae_curv import config as config_curv # from models.cvae_model_overfit_jumpsuit import CVAE from models.cvae_model import CVAENew as CVAE @@ -296,52 +296,54 @@ def sampling_example(frame, camera, nr=15): mask = batch['mesh_mask'][0].numpy().astype(bool) _, faces = _maskFaces(np.zeros((14475, 3)), F, mask) - noise = tf.random.normal(shape=(nr, 8, 4, 128)) - out = model.sample(tf.tile(batch['uv_static'], [nr,1,1,1]), tf.tile(batch['uv_body_posed'], [nr,1,1,1]), tf.tile(batch['uv_curvature'], [nr,1,1,1]), tf.tile(batch['uv_mask'], [nr,1,1]), noise) - # out,_,_ = model([tf.tile(batch['uv_unposed'], [nr,1,1,1]), tf.tile(batch['uv_static'], [nr,1,1,1]), tf.tile(batch['uv_body_posed'], [nr,1,1,1]), tf.tile(batch['uv_curvature'], [nr,1,1,1]), tf.tile(batch['uv_mask'], [nr,1,1])], training=False) + # noise = tf.random.normal(shape=(nr, 8, 4, 128)) + # out = model.sample(tf.tile(batch['uv_static'], [nr,1,1,1]), tf.tile(batch['uv_body_posed'], [nr,1,1,1]), tf.tile(batch['uv_curvature'], [nr,1,1,1]), tf.tile(batch['uv_mask'], [nr,1,1]), noise) + # # out,_,_ = model([tf.tile(batch['uv_unposed'], [nr,1,1,1]), tf.tile(batch['uv_static'], [nr,1,1,1]), tf.tile(batch['uv_body_posed'], [nr,1,1,1]), tf.tile(batch['uv_curvature'], [nr,1,1,1]), tf.tile(batch['uv_mask'], [nr,1,1])], training=False) - if config_main['use_skinning']: - posed_mesh = [get_posed_mesh(uv, batch['pose_shape'], batch['mesh_body_posed'], mask) for uv in tf.unstack(out)] - else: - posed_mesh = [uv2mesh(uv[tf.newaxis,:,:,:]) for uv in tf.unstack(out)] - gt_mesh = batch['mesh_posed'][0].numpy()[mask, :] + # if config_main['use_skinning']: + # posed_mesh = [get_posed_mesh(uv, batch['pose_shape'], batch['mesh_body_posed'], mask) for uv in tf.unstack(out)] + # else: + # posed_mesh = [uv2mesh(uv[tf.newaxis,:,:,:]) for uv in tf.unstack(out)] + # gt_mesh = batch['mesh_posed'][0].numpy()[mask, :] body = batch['mesh_body_posed'][0].numpy() - posed_mesh_fixed = [postprocess(mesh, body, faces, mask, do_smoothing=True) for mesh in posed_mesh] - # posed_mesh_fixed = [mesh[0].numpy()[mask, :] for mesh in posed_mesh] + # posed_mesh_fixed = [postprocess(mesh, body, faces, mask, do_smoothing=True) for mesh in posed_mesh] + # # posed_mesh_fixed = [mesh[0].numpy()[mask, :] for mesh in posed_mesh] - # error = [np.sqrt(((gt_mesh-m) ** 2).sum(-1)).mean() for m in posed_mesh_fixed] + # # error = [np.sqrt(((gt_mesh-m) ** 2).sum(-1)).mean() for m in posed_mesh_fixed] - # row_sampling = [] - # fig = plot_fixed(gt_mesh, body, faces, camera) - # fig.write_image('/tmp/exp_temp_7.png') - # img = cv2.imread('/tmp/exp_temp_7.png') - # row_sampling.append(img[:, 100:700, :]) - # for mesh in posed_mesh_fixed: - # fig = plot_fixed(mesh, body, faces, camera) - # fig.write_image('/tmp/exp_temp_7.png') - # img = cv2.imread('/tmp/exp_temp_7.png') - # row_sampling.append(img[:, 100:700, :]) + # # row_sampling = [] + # # fig = plot_fixed(gt_mesh, body, faces, camera) + # # fig.write_image('/tmp/exp_temp_7.png') + # # img = cv2.imread('/tmp/exp_temp_7.png') + # # row_sampling.append(img[:, 100:700, :]) + # # for mesh in posed_mesh_fixed: + # # fig = plot_fixed(mesh, body, faces, camera) + # # fig.write_image('/tmp/exp_temp_7.png') + # # img = cv2.imread('/tmp/exp_temp_7.png') + # # row_sampling.append(img[:, 100:700, :]) # custom_imshow(np.concatenate(row_sampling, axis=1)).show() - os.makedirs(os.path.join(out_folder, frame), exist_ok=True) + # os.makedirs(os.path.join(out_folder, frame), exist_ok=True) - plot_fixed(np.zeros_like(gt_mesh), body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_body.png')) - plot_fixed(gt_mesh, body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_gt.png')) - for i in range(nr): - plot_fixed(posed_mesh_fixed[i], body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_' + str(i) + '.png')) + # plot_fixed(np.zeros_like(gt_mesh), body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_body.png')) + # plot_fixed(gt_mesh, body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_gt.png')) + # for i in range(nr): + # plot_fixed(posed_mesh_fixed[i], body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_' + str(i) + '.png')) # save_stl(os.path.join(out_folder, frame, frame + '_body.stl'), body, F_body) # save_stl(os.path.join(out_folder, frame, frame + '_gt.stl'), gt_mesh, faces.astype(np.int32)) # for i in range(nr): # save_stl(os.path.join(out_folder, frame, frame + '_' + str(i) + '.stl'), posed_mesh_fixed[i], faces.astype(np.int32)) - np.save(os.path.join(out_folder, frame, frame + '_gt.npy'), gt_mesh) - np.save(os.path.join(out_folder, frame, frame + '_body.npy'), body) - np.save(os.path.join(out_folder, frame, frame + '_faces.npy'), faces.astype(np.int32)) - for i in range(nr): - np.save(os.path.join(out_folder, frame, frame + '_' + str(i) + '.npy'), posed_mesh_fixed[i]) + # np.save(os.path.join(out_folder, frame, frame + '_gt.npy'), gt_mesh) + # np.save(os.path.join(out_folder, frame, frame + '_body.npy'), body) + # np.save(os.path.join(out_folder, frame, frame + '_faces.npy'), faces.astype(np.int32)) + # for i in range(nr): + # np.save(os.path.join(out_folder, frame, frame + '_' + str(i) + '.npy'), posed_mesh_fixed[i]) + save_stl(os.path.join(out_folder, frame + '_gt_body.stl'), body, F_body) + save_stl(os.path.join(out_folder, frame + '_posed_body.stl'), body, F_body) # custom_imshow(np.concatenate(row_sampling, axis=1)).write_image(os.path.join(out_folder, frame + '_sampling.png')) print() @@ -417,6 +419,7 @@ def sampling_garment(frame, camera): print() + if __name__ == '__main__': parser = argparse.ArgumentParser(description='') parser.add_argument('--name', type=str) @@ -476,9 +479,15 @@ if __name__ == '__main__': # frame = '04925_272' # frames = ['01070_13', '02276_232', '02978_242', '04008_295', '04173_193'] - frames = ['02276_232', '02978_242', '04173_193'] + # frames = ['00073_160', '00073_217', '00096_86', \ + # '04347_100', '04360_2', '04404_41', '04442_85', '04466_59', '04620_68', '04620_129', \ + # '04655_28', '04668_255', '04682_96', '04709_37', '06117_123', '06141_162'] + + frames = ['00265_257', '04070_263', '00239_9', '04608_170', '04372_289', '00077_106', '00205_164', '00323_97', '00185_7', '00079_250', \ + '04143_56', '00240_136', '00185_33'] - out_folder = 'examples/sampling_analysis' + + out_folder = 'examples/qualitative/bodies' os.makedirs(out_folder, exist_ok=True) for frame in tqdm.tqdm(frames): sampling_example(frame, camera_1)