diff --git a/aaa_helpers.txt b/aaa_helpers.txt index d14a195..34598d6 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -104,6 +104,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_pre_cape_f" > nohup/iccv nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128" > nohup/iccv/pyr/iccv_pyr_32_64_128.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin_2.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skin_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skin_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_reg_2.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_curv.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_curv_skirt.out 2>&1 & @@ -117,6 +118,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256" > nohup/iccv nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skirt_2.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_skin" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_skin.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_curv_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_curv_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_reg" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_reg.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_256_512" > nohup/iccv/pyr/iccv_pyr_32_64_128_256_512.out 2>&1 & @@ -143,6 +145,8 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_reg > nohu nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv > nohup/iccv/evaluate/iccv_pyr_32_64_curv.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_curv_skirt.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_10p > nohup/iccv/evaluate/iccv_pyr_32_64_10p.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_25p > nohup/iccv/evaluate/iccv_pyr_32_64_25p.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index 62f7622..8a71f32 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_25p import config +from configs.iccv.config_abl_pyr_32_64_25p import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] # import tensorflow as tf @@ -65,7 +65,7 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() - output_folder = '/home/workspace/local-data/results/iccv/' + checkpoint_name + '-' + output_folder = '/home/workspace/local-data_128/results/iccv/' + checkpoint_name + '-' # check if there is a folder that starts with this name using glob folders = glob.glob(output_folder + '*') assert len(folders) == 1, f'There should be only one checkpoint that starts with this name, but found {len(folders)}' diff --git a/configs/iccv/config_abl_pyr_32_64_10p.py b/configs/iccv/config_abl_pyr_32_64_10p.py index dc2d547..9effab5 100644 --- a/configs/iccv/config_abl_pyr_32_64_10p.py +++ b/configs/iccv/config_abl_pyr_32_64_10p.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '4' +config['gpu'] = '0' config['name'] = 'iccv_pyr_32_64_10p' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data_127/train' config['valid_dir'] = '/home/workspace/local-data_127/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['frames_file'] = 'frames_10p.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512.py b/configs/iccv/config_abl_pyr_32_64_128_256_512.py index 158fcbd..27da067 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '0' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_128_256_512' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -22,7 +22,7 @@ config['use_offsets'] = False config['use_norm_loss'] = True config['use_curvature'] = True -config['pretrained_model'] = '' +config['pretrained_model'] = 'checkpoints/iccv/iccv_pyr_32_64_128_256_512' config['pretrained_model_iteration'] = '' config['pyramid'] = dict() @@ -32,13 +32,15 @@ config['pyramid']['pretrained_pyramid_iteration'] = '90000' config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' -config['valid_dir'] = '/home/workspace/local-data/valid' +config['valid_dir'] = '/home/workspace/local-data/train' config['test_dir'] = '/home/workspace/local-data/test_ready' +config['frames_file'] = 'frames_00761_0.txt' +config['frames_file_valid'] = 'frames_00761_0.txt' config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False -config['dataset_in_memory'] = False +config['dataset_in_memory'] = True config['resolution'] = None config['smpl_res'] = 14475 diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_curv_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_256_curv_skirt.py new file mode 100644 index 0000000..0c1cf78 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_128_256_curv_skirt.py @@ -0,0 +1,187 @@ +config = dict() + +config['gpu'] = '1' +config['name'] = 'iccv_pyr_32_64_128_256_curv_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'curv', 'skirt'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = False + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64, 128, 256] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_128_curv_skirt' +config['pyramid']['pretrained_pyramid_iteration'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir'] = '/home/workspace/local-data/valid_skirt' +config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_128_curv_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_curv_skirt.py index 83c2396..9bb2246 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_curv_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_curv_skirt.py @@ -94,11 +94,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -107,13 +106,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -122,7 +120,7 @@ for res in config['pyramid']['resolutions']: level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 level['encoder_depths'] = [9, 9, 9] - level['encoder_dims'] = [128, 128, level['latent_dim']] + level['encoder_dims'] = [64, 128, level['latent_dim']] level['decoder_depths'] = level['encoder_depths'] level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' diff --git a/configs/iccv/config_abl_pyr_32_64_128_reg_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_reg_skirt.py index 4da9afd..7b79002 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_reg_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_reg_skirt.py @@ -94,11 +94,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -107,13 +106,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -122,7 +120,7 @@ for res in config['pyramid']['resolutions']: level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 level['encoder_depths'] = [9, 9, 9] - level['encoder_dims'] = [128, 128, level['latent_dim']] + level['encoder_dims'] = [64, 128, level['latent_dim']] level['decoder_depths'] = level['encoder_depths'] level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' diff --git a/configs/iccv/config_abl_pyr_32_64_128_skin_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_skin_skirt.py index 8e48da3..dc6e45b 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_skin_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_skin_skirt.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '5' +config['gpu'] = '0' config['name'] = 'iccv_pyr_32_64_128_skin_skirt' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -31,7 +31,7 @@ config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_64_skin_ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data_128/train_skirt' -config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['valid_dir'] = '/home/workspace/local-data/valid_skirt' config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' config['batch_size'] = 4 config['is_skirt'] = True @@ -100,11 +100,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -113,13 +112,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -128,7 +126,7 @@ for res in config['pyramid']['resolutions']: level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 level['encoder_depths'] = [9, 9, 9] - level['encoder_dims'] = [128, 128, level['latent_dim']] + level['encoder_dims'] = [64, 128, level['latent_dim']] level['decoder_depths'] = level['encoder_depths'] level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' diff --git a/configs/iccv/config_abl_pyr_32_64_25p.py b/configs/iccv/config_abl_pyr_32_64_25p.py index 9966141..4c22edb 100644 --- a/configs/iccv/config_abl_pyr_32_64_25p.py +++ b/configs/iccv/config_abl_pyr_32_64_25p.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '5' +config['gpu'] = '1' config['name'] = 'iccv_pyr_32_64_25p' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] @@ -32,7 +32,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data_127/train' config['valid_dir'] = '/home/workspace/local-data_127/valid' -config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['test_dir'] = '/home/workspace/local-data/test_ready' config['frames_file'] = 'frames_25p.txt' diff --git a/configs/iccv/config_abl_pyr_32_64_curv_skirt.py b/configs/iccv/config_abl_pyr_32_64_curv_skirt.py index 775a481..1da7a05 100644 --- a/configs/iccv/config_abl_pyr_32_64_curv_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_curv_skirt.py @@ -86,7 +86,6 @@ config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' config['pyramid']['freeze_encoders'] = True - config['pyramid']['levels'] = [] for res in config['pyramid']['resolutions']: if res == 512: @@ -94,11 +93,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -107,13 +105,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -121,13 +118,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 128 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [16, 9, 9, 9] - level['encoder_dims'] = [64, 128, 128, 64] - level['decoder_input'] = (4, 2, 128) - level['decoder_depths'] = [16, 9, 9, 9] - level['decoder_dims'] = [256, 128, 128, level['latent_dim']] + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 64: diff --git a/configs/iccv/config_abl_pyr_32_64_reg_skirt.py b/configs/iccv/config_abl_pyr_32_64_reg_skirt.py index e44c7eb..8074658 100644 --- a/configs/iccv/config_abl_pyr_32_64_reg_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_reg_skirt.py @@ -86,7 +86,6 @@ config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' config['pyramid']['freeze_encoders'] = True - config['pyramid']['levels'] = [] for res in config['pyramid']['resolutions']: if res == 512: @@ -94,11 +93,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -107,13 +105,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -121,13 +118,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 128 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [16, 9, 9, 9] - level['encoder_dims'] = [64, 128, 128, 64] - level['decoder_input'] = (4, 2, 128) - level['decoder_depths'] = [16, 9, 9, 9] - level['decoder_dims'] = [256, 128, 128, level['latent_dim']] + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 64: diff --git a/configs/iccv/config_abl_pyr_32_64_skin_skirt copy.py b/configs/iccv/config_abl_pyr_32_64_skin_skirt copy.py new file mode 100644 index 0000000..590dcc7 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_64_skin_skirt copy.py @@ -0,0 +1,193 @@ +config = dict() + +config['gpu'] = '2' +config['name'] = 'iccv_pyr_32_64_skin_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = False # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32, 64] +config['pyramid']['pretrained_pyramid'] = 'checkpoints/iccv/iccv_pyr_32_skin_skirt' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +if config['use_skinning']: + config['signature_dynamic']['uv_unposed'] = (512, 256, 3) + config['signature_dynamic']['mesh_unposed'] = (config['smpl_res'], 3,) +else: + config['signature_dynamic']['uv_posed'] = (512, 256, 3) +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + if config['use_skinning']: + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + else: + config['signature_dynamic']['uv_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_64_skin_skirt.py b/configs/iccv/config_abl_pyr_32_64_skin_skirt.py index f6979e3..590dcc7 100644 --- a/configs/iccv/config_abl_pyr_32_64_skin_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_skin_skirt.py @@ -100,11 +100,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -113,13 +112,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -127,13 +125,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 128 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [16, 9, 9, 9] - level['encoder_dims'] = [64, 128, 128, 64] - level['decoder_input'] = (4, 2, 128) - level['decoder_depths'] = [16, 9, 9, 9] - level['decoder_dims'] = [256, 128, 128, level['latent_dim']] + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 64: diff --git a/curvature_analysis.py b/curvature_analysis.py index ec1e7d3..b373f26 100644 --- a/curvature_analysis.py +++ b/curvature_analysis.py @@ -338,7 +338,7 @@ def visualize_neighbourhoods(folder, frame): write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_25.obj', verts, faces.astype(np.int32), colors_binned_all[1].astype(np.int32)) write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_100.obj', verts, faces.astype(np.int32), colors_binned_all[2].astype(np.int32)) write_obj(f'examples/curvature_analysis/{id}_{frame_nr}_mesh.obj', verts, faces.astype(np.int32), np.ones((verts.shape[0], 3), dtype=np.int32) * 200) - + save_stl(f'examples/curvature_analysis/{id}_{frame_nr}_mesh.stl', verts, faces.astype(np.int32)) fig.show() return fig @@ -372,10 +372,10 @@ if __name__ == '__main__': i = 0 while True: frame = np.random.choice(frames) + print(frame) # frame = '03265_193' # frame = '03102_49' # fig, dice = visualize_curvature_labels(folder, results_folder, frame) # wandb.log({f'{frame}-dice:{dice}': fig}, step=i) visualize_neighbourhoods(folder, frame) - i += 1 - print(frame) \ No newline at end of file + i += 1 \ No newline at end of file diff --git a/cvae_pyramid.py b/cvae_pyramid.py index 5958d19..b2bb5d8 100644 --- a/cvae_pyramid.py +++ b/cvae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_25p import config +from configs.iccv.config_abl_pyr_32_64_128_256_512 import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -730,12 +730,12 @@ def generate_results(): @tf.function def speed_step(levels_input, conds): - return model.sample([levels_input, conds], training=False) + return model.sample([levels_input, conds]) if __name__ == '__main__': - init_wandb() + # init_wandb() parser = argparse.ArgumentParser(description='') parser.add_argument('--name', type=str, default='lhunor') args = vars(parser.parse_args()) @@ -846,52 +846,52 @@ if __name__ == '__main__': for batch in dataset_train: batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic'])) - # levels_input, conds = group_inputs_for_levels(batch, config) + levels_input, conds = group_inputs_for_levels(batch, config) - # start = time.perf_counter() - # out = speed_step(levels_input, conds) - # t = time.perf_counter() - start - # print(t) - # times.append(t) + start = time.perf_counter() + out = speed_step(levels_input, conds) + t = time.perf_counter() - start + print(t) + times.append(t) - # if step > 200: - # break + if step > 200: + break - train_loss = train_step(batch) - train_accumulator.update(train_loss) + # train_loss = train_step(batch) + # train_accumulator.update(train_loss) - if step % 25 == 0: - data_valid = valid_iterator.get_next() - data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic'])) - valid_loss = valid_step(data_valid) - valid_accumulator.update(valid_loss) + # if step % 25 == 0: + # data_valid = valid_iterator.get_next() + # data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic'])) + # valid_loss = valid_step(data_valid) + # valid_accumulator.update(valid_loss) - if train_loss['train/euclid_error'] < 0.015: - if balancing_activated_step == 0: - balancing_activated_step = step - balancing_coeff = balancing_coeff + 1 / 20000 + # if train_loss['train/euclid_error'] < 0.015: + # if balancing_activated_step == 0: + # balancing_activated_step = step + # balancing_coeff = balancing_coeff + 1 / 20000 - if step % 1000 == 0 or (step < 1000 and step % 100 == 0): - to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } }) - if step % 5000 == 0 or (step < 5000 and step % 1000 == 0): - train_figs = plot_everything(batch, 'train') - valid_figs = plot_everything(data_valid, 'valid') - to_log = { **to_log, **train_figs, **valid_figs } - if step == 0: - to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } } + # if step % 1000 == 0 or (step < 1000 and step % 100 == 0): + # to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } }) + # if step % 5000 == 0 or (step < 5000 and step % 1000 == 0): + # train_figs = plot_everything(batch, 'train') + # valid_figs = plot_everything(data_valid, 'valid') + # to_log = { **to_log, **train_figs, **valid_figs } + # if step == 0: + # to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } } - to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } } + # to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } } - wandb.log(to_log, step=step) - train_accumulator.reset() - valid_accumulator.reset() + # wandb.log(to_log, step=step) + # train_accumulator.reset() + # valid_accumulator.reset() - if step % 5000 == 0: - manager.save(checkpoint_number=step) + # if step % 5000 == 0: + # manager.save(checkpoint_number=step) - print(step) + # print(step) step += 1 - # print('mean:') - # print(np.mean(times[1:])) \ No newline at end of file + print('mean:') + print(np.mean(times[10:])) \ No newline at end of file diff --git a/evaluate_ae_pyramid.py b/evaluate_ae_pyramid.py index 0472d9d..334b6b3 100644 --- a/evaluate_ae_pyramid.py +++ b/evaluate_ae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_256_512_curv import config +from configs.iccv.config_abl_pyr_32_64_10p import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/get_error.py b/get_error.py index 2c5b4e2..03a21c4 100644 --- a/get_error.py +++ b/get_error.py @@ -7,9 +7,9 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() folder = '/home/workspace/data/results/errors/iccv' - checkpoint_name = 'iccv_pyr_32_64_128_256_512' + checkpoint_name = 'iccv_pyr_32_64_25p' checkpoint_name_skirt = 'iccv_pyr_32_64_128_256_512_skirt' - iteration = '120000' + iteration = '155000' iteration_skirt : str = '115000' # checkpoint_name = 'abl_skin' diff --git a/models/pyramid_level.py b/models/pyramid_level.py index 2d94698..d4508da 100644 --- a/models/pyramid_level.py +++ b/models/pyramid_level.py @@ -101,6 +101,7 @@ class PyramidLevelNew(tf.keras.Model): def __init__(self, config): super(PyramidLevelNew, self).__init__() self.size = config['resolution'] + self.latent_dim=config['latent_dim'] self.encoder = VariationalEncoderNew(config['encoder_depths'], config['encoder_dims'], kernel_size=config['kernel_size'], latent_dim=config['latent_dim']) self.decoder = DecoderNew(config['decoder_depths'], config['decoder_dims'], config['kernel_size']) @@ -126,13 +127,12 @@ class PyramidLevelNew(tf.keras.Model): def sample(self, z_cond, uv_mask, noise=None): if noise is None: - eps = tf.random.normal(shape=(uv_mask.shape[0], self.latent_dim)) + eps = tf.random.normal(shape=(uv_mask.shape[0], 8, 4, self.latent_dim)) else: eps = noise z = tf.concat([eps, z_cond], axis=-1) - - x = self.decoder(z) + x = self.decoder(z, training=False) if self.avg_garment is not None: x = x + self.avg_garment[tf.newaxis,:,:,:] diff --git a/models/pyramid_model_optimized.py b/models/pyramid_model_optimized.py index f3661e5..5019c7a 100644 --- a/models/pyramid_model_optimized.py +++ b/models/pyramid_model_optimized.py @@ -320,6 +320,87 @@ class PyramidNew(tf.keras.Model): return z_cond + def sample(self, inputs): + levels_input, conds = inputs + uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2] + z_cond = self.encode_conds(uv_static, uv_body, uv_curv, training=False) + + pred = [None, None, None, None, None] + for i in range(len(self.levels)): + pred[i] = self.levels[i].sample(z_cond, levels_input[i][-2]) + + pred_1 = pred[0] + pred_2 = pred[1] + pred_3 = pred[2] + pred_4 = pred[3] + pred_5 = pred[4] + + pred_1 = self.levels[0].enhance(pred_1, levels_input[0][-2], training=False) + pred_1 = tf.image.resize(pred_1, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR) + pred_1 = pred_1 * levels_input[1][-2][:,:,:,tf.newaxis] + + pred_2 = pred_1 + pred_2 + pred_2 = self.levels[1].enhance(pred_2, levels_input[1][-2], training=False) + pred_2 = tf.image.resize(pred_2, (self.levels[2].size, self.levels[2].size//2), tf.image.ResizeMethod.BILINEAR) + pred_2 = pred_2 * levels_input[2][-2][:,:,:,tf.newaxis] + + pred_3 = pred_2 + pred_3 + pred_3 = self.levels[2].enhance(pred_3, levels_input[2][-2], training=False) + pred_3 = tf.image.resize(pred_3, (self.levels[3].size, self.levels[3].size//2), tf.image.ResizeMethod.BILINEAR) + pred_3 = pred_3 * levels_input[3][-2][:,:,:,tf.newaxis] + + pred_4 = pred_3 + pred_4 + pred_4 = self.levels[3].enhance(pred_4, levels_input[3][-2], training=False) + pred_4 = tf.image.resize(pred_4, (self.levels[4].size, self.levels[4].size//2), tf.image.ResizeMethod.BILINEAR) + pred_4 = pred_4 * levels_input[4][-2][:,:,:,tf.newaxis] + + pred_5 = pred_4 + pred_5 + pred_5 = pred_5 * levels_input[4][-2][:,:,:,tf.newaxis] + + return pred_5 + + + def sample_seq(self, inputs): + levels_input, conds = inputs + uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2] + z_cond = self.encode_conds(uv_static, uv_body, uv_curv, training=False) + + # level 1 + pred_1 = self.levels[0].sample(z_cond, levels_input[0][-2]) + pred_1 = self.levels[0].enhance(pred_1, levels_input[0][-2], training=False) + pred_1 = tf.image.resize(pred_1, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR) + pred_1 = pred_1 * levels_input[1][-2][:,:,:,tf.newaxis] + + # level 2 + pred_2 = self.levels[1].sample(z_cond, levels_input[1][-2]) + pred_2 = pred_1 + pred_2 + pred_2 = self.levels[1].enhance(pred_2, levels_input[1][-2], training=False) + pred_2 = tf.image.resize(pred_2, (self.levels[2].size, self.levels[2].size//2), tf.image.ResizeMethod.BILINEAR) + pred_2 = pred_2 * levels_input[2][-2][:,:,:,tf.newaxis] + + # level 3 + pred_3 = self.levels[2].sample(z_cond, levels_input[2][-2]) + pred_3 = pred_2 + pred_3 + pred_3 = self.levels[2].enhance(pred_3, levels_input[2][-2], training=False) + pred_3 = tf.image.resize(pred_3, (self.levels[3].size, self.levels[3].size//2), tf.image.ResizeMethod.BILINEAR) + pred_3 = pred_3 * levels_input[3][-2][:,:,:,tf.newaxis] + + # level 4 + pred_4 = self.levels[3].sample(z_cond, levels_input[3][-2]) + pred_4 = pred_3 + pred_4 + pred_4 = self.levels[3].enhance(pred_4, levels_input[3][-2], training=False) + pred_4 = tf.image.resize(pred_4, (self.levels[4].size, self.levels[4].size//2), tf.image.ResizeMethod.BILINEAR) + pred_4 = pred_4 * levels_input[4][-2][:,:,:,tf.newaxis] + + # level 5 + pred_5 = self.levels[4].sample(z_cond, levels_input[4][-2]) + pred_5 = pred_5 * levels_input[4][-2][:,:,:,tf.newaxis] + + pred_5 = pred_4 + pred_5 + + return pred_5 + + def call(self, inputs, training): levels_input, conds = inputs uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2] diff --git a/plot_ablation.py b/plot_ablation.py index 3b261cd..a3e2fbc 100644 --- a/plot_ablation.py +++ b/plot_ablation.py @@ -1,5 +1,5 @@ import os -os.environ["CUDA_VISIBLE_DEVICES"] = "6" +os.environ["CUDA_VISIBLE_DEVICES"] = "1" import sys import time import cv2 @@ -128,7 +128,11 @@ def get_experiment_plots(folder, frames, experiments, skirt_mods, bodies, faces, frame_out = [] id = frame.split('_')[0] for exp in experiments: - garment = np.load(os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy')) + path = os.path.join(folder, exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy') + if not os.path.exists(path): + path = os.path.join('/home/workspace/local-data_128/results/iccv', exp[0] + skirt_mods[id] + '-' + exp[1], frame + '.npy') + assert os.path.exists(path), path + garment = np.load(path) if exp == 'abl_orient': garment = deorient(garment, frame, skirt_mods[id] == '_skirt') garment = postprocess(garment, bodies[frame], id, skirt_mods[id] == '_skirt') @@ -190,11 +194,11 @@ if __name__ == '__main__': faces = get_faces() - frames = ['00079_135'] + frames = ['00079_135', '00505_23', '00688_25', '01693_128', '00408_34'] # frames = [ '00075_4', '00475_273', '00761_295', '00793_3', '00993_220', '01134_112', '01278_108', '00718_38', '00951_34', '00962_60','01065_38','01191_83','01240_168'] - # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')] - # random.shuffle(rand_frames) - # rand_frames = rand_frames[:25] + rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_ready/pose_shape')] + random.shuffle(rand_frames) + rand_frames = rand_frames[:25] # frames.extend(rand_frames) # rand_frames = [frame.split('.')[0] for frame in os.listdir('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape')] # random.shuffle(rand_frames) @@ -207,34 +211,38 @@ if __name__ == '__main__': cameras = {frame: camera_1 for frame in frames} # abl_all replace abl_norm maybe - experiments = [['iccv_pyr_32', '335000'], ['iccv_pyr_32_skin', '330000'], ['iccv_pyr_32_curv', '155000'], ['iccv_pyr_32_reg', '150000']] # , 'pyra_32_res' + # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_256_512_curv', '80000'], ['iccv_pyr_32_64_128_256_512_skin', '115000'], ['iccv_pyr_32_64_128_256_512_reg', '75000']] # , 'pyra_32_res' + # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_256_curv', '80000'], ['iccv_pyr_32_64_128_256_skin', '145000'], ['iccv_pyr_32_64_128_256_reg', '35000']] # , 'pyra_32_res' + # experiments = [['iccv_pyr_32_64_128_256_512', '120000'], ['iccv_pyr_32_64_128_curv', '20000'], ['iccv_pyr_32_64_128_skin_2', '20000'], ['iccv_pyr_32_64_128_reg_2', '80000']] # , 'pyra_32_res' + experiments = [['abl_all_3', '130000']] # , 'pyra_32_res' skirt_mods = { frame.split('_')[0]: ('_skirt' if os.path.exists('/home/workspace/data/dataset/test_skirt_ready_2/pose_shape/' + frame + '.npy') else '') for frame in frames } bodies = get_bodies(frames, skirt_mods) - # gts = get_gt_image(frames, bodies, faces, cameras, skirt_mods) - gts = get_gt_meshes(frames, bodies, faces, cameras, skirt_mods) results_folder_base = '/home/workspace/data/results/iccv/' - output_folder = os.path.join(results_folder_base, 'ablation_plots') + output_folder = os.path.join('examples/ablation/img_3') + output_folder_stl = os.path.join('examples/ablation/stl_3') os.makedirs(output_folder, exist_ok=True) + os.makedirs(output_folder_stl, exist_ok=True) + # gts_img = get_gt_image(frames, bodies, faces, cameras, skirt_mods) # images = get_experiment_plots(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras) + # rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts_img, images)] + # for frame, row in zip(frames, rows): + # custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png')) + + gts = get_gt_meshes(frames, bodies, faces, cameras, skirt_mods) meshes = get_experiment_meshes(results_folder_base, frames, experiments, skirt_mods, bodies, faces, cameras) for mesh, frame in zip(meshes, frames): for exp, (garment, body, faces) in zip(experiments, mesh): - save_stl(f'examples/{frame}_{exp[0]}_garment.stl', garment, faces) - save_stl(f'examples/{frame}_{exp[0]}_body.stl', body, F) - + save_stl(f'{output_folder_stl}/{frame}_{exp[0]}_garment.stl', garment, faces) + save_stl(f'{output_folder_stl}/{frame}_{exp[0]}_body.stl', body, F) for mesh, frame in zip(gts, frames): garment, body, faces = mesh - save_stl(f'examples/{frame}_gt_garment.stl', garment, faces) - save_stl(f'examples/{frame}_gt_body.stl', body, F) + save_stl(f'{output_folder_stl}/{frame}_gt_garment.stl', garment, faces) + save_stl(f'{output_folder_stl}/{frame}_gt_body.stl', body, F) - # rows = [np.concatenate((gt, exp), axis=1) for gt, exp in zip(gts, images)] - - # for frame, row in zip(frames, rows): - # custom_imshow(row).write_image(os.path.join(output_folder, frame + '_ablation.png')) # final = np.concatenate(rows, axis=0)