diff --git a/aaa_helpers.txt b/aaa_helpers.txt index 34598d6..8d0a954 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -77,6 +77,13 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_reg" > nohup/iccv/pyr/ic nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_reg_skirt" > nohup/iccv/pyr/iccv_pyr_32_reg_skirt.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_10p" > nohup/iccv/pyr/iccv_pyr_32_10p.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_25p" > nohup/iccv/pyr/iccv_pyr_32_25p.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_jumpsuit" > nohup/iccv/pyr/iccv_pyr_32_jumpsuit.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_trousers" > nohup/iccv/pyr/iccv_pyr_32_trousers.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_top" > nohup/iccv/pyr/iccv_pyr_32_top.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_tshirt" > nohup/iccv/pyr/iccv_pyr_32_tshirt.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_skirt_skirt" > nohup/iccv/pyr/iccv_pyr_32_skirt_skirt.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_skirt_dress" > nohup/iccv/pyr/iccv_pyr_32_skirt_dress.out 2>&1 & +nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_skirt_batch" > nohup/iccv/pyr/iccv_pyr_32_skirt_batch.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64" > nohup/iccv/pyr/iccv_pyr_32_64.out 2>&1 & nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_skirt.out 2>&1 & @@ -151,6 +158,7 @@ nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_25p > nohu nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128 > nohup/iccv/evaluate/iccv_pyr_32_64_128.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skin > nohup/iccv/evaluate/iccv_pyr_32_64_128_skin.out 2>&1 & +nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_skin_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_skin_skirt.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_reg > nohup/iccv/evaluate/iccv_pyr_32_64_128_reg.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv > nohup/iccv/evaluate/iccv_pyr_32_64_128_curv.out 2>&1 & nohup python evaluate_ae_pyramid.py --name evaluate_ae_iccv_pyr_32_64_128_curv_skirt > nohup/iccv/evaluate/iccv_pyr_32_64_128_curv_skirt.out 2>&1 & diff --git a/calculate_error.py b/calculate_error.py index 8a71f32..c08f03d 100644 --- a/calculate_error.py +++ b/calculate_error.py @@ -1,37 +1,37 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_25p import config +from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] -# import tensorflow as tf +import tensorflow as tf -# gpus = tf.config.experimental.list_physical_devices('GPU') -# if gpus: -# try: -# # Currently, memory growth needs to be the same across GPUs -# for gpu in gpus: -# tf.config.experimental.set_memory_growth(gpu, True) -# logical_gpus = tf.config.experimental.list_logical_devices('GPU') -# print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") -# except RuntimeError as e: -# # Memory growth must be set before GPUs have been initialized -# print(e) +gpus = tf.config.experimental.list_physical_devices('GPU') +if gpus: + try: + # Currently, memory growth needs to be the same across GPUs + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + logical_gpus = tf.config.experimental.list_logical_devices('GPU') + print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") + except RuntimeError as e: + # Memory growth must be set before GPUs have been initialized + print(e) import csv import glob import time import argparse import wandb -import tqdm +from tqdm import tqdm import random import numpy as np import plotly.graph_objects as go import plotly.express as ex - -# if config['is_skirt']: -# from losses import edge_loss_skirt as edge_loss -# else: -# from losses import edge_loss +from losses import collision_loss, normal_loss +if config['is_skirt']: + from losses import edge_loss_skirt as edge_loss +else: + from losses import edge_loss @@ -53,9 +53,13 @@ if __name__ == '__main__': parser.add_argument('--name', type=str) args = vars(parser.parse_args()) + print(config['name']) + # smpl = SMPL(use_one_hot=config['use_one_hot']) skirt_modifier = '_skirt' if config['is_skirt'] else '' + error_type = 'collision' # 'euclidean' 'collision' edge normal + folder = config['test_dir'] checkpoint_name = config['name'] @@ -65,7 +69,7 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() - output_folder = '/home/workspace/local-data_128/results/iccv/' + checkpoint_name + '-' + output_folder = '/home/workspace/local-data/results/iccv/' + checkpoint_name + '-' # check if there is a folder that starts with this name using glob folders = glob.glob(output_folder + '*') assert len(folders) == 1, f'There should be only one checkpoint that starts with this name, but found {len(folders)}' @@ -78,6 +82,7 @@ if __name__ == '__main__': # ids = [id.split('.')[0] for id in ids] # faces = {id : np.load(faces_folder + '/' + id + '.npy') for id in ids } + print('Reading masks...') masks_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_mask' ids = os.listdir(masks_folder) ids = [id.split('.')[0] for id in ids] @@ -85,6 +90,11 @@ if __name__ == '__main__': frames = os.listdir(output_folder) frames = [frame for frame in frames if not frame.endswith('_fixed_coll.npy')] + # frames = frames[:100] + + print('Reading bodies...') + bodies_folder = '/home/workspace/local-data' + ('' if config['is_skirt'] else '_127') + '/test' + ('_skirt' if config['is_skirt'] else '') + '_ready' + ('_2' if config['is_skirt'] else '') + '/mesh_posed_body_no_orient' + bodies = {frame : np.load(bodies_folder + '/' + frame[:-1] + 'z')['mesh_posed_body_no_orient'] for frame in tqdm(frames) } if config['remove_global_orientation']: gt_name = 'mesh_posed_no_orient' @@ -99,12 +109,32 @@ if __name__ == '__main__': else: per_garment_errors = { 'Tshirt': [], 'Jumpsuit': [], 'Trousers': [], 'Top': [] } - for frame in tqdm.tqdm(frames): + for frame in tqdm(frames): id = frame.split('_')[0] frame_nr = frame.split('.')[0].split('_')[1] gt = np.load(os.path.join(orig_folder, id + '_' + frame_nr + '.npz'))[gt_name][masks[id], :].astype(np.float32) pred = np.load(os.path.join(output_folder, id + '_' + frame_nr + '.npy')) - metric = np.mean(np.sqrt(np.sum((gt - pred) ** 2, axis=-1))) + if error_type == 'edge': + pred_unmasked = np.zeros((1, 14475, 3), dtype=np.float32) + pred_unmasked[0, masks[id], :] = pred + gt_unmasked = np.zeros((1, 14475, 3), dtype=np.float32) + gt_unmasked[0, masks[id], :] = gt + metric = edge_loss(gt_unmasked, pred_unmasked, masks[id][None,:].astype(np.float32)).numpy()[0] + elif error_type == 'euclidean': + metric = np.mean(np.sqrt(np.sum((gt - pred) ** 2, axis=-1))) + elif error_type == 'collision': + pred_unmasked = np.zeros((1, 14475, 3), dtype=np.float32) + pred_unmasked[0, masks[id], :] = pred + metric = collision_loss(pred_unmasked, bodies[frame][None,:,:], masks[id][None,:].astype(np.float32))[0].numpy()[0] + elif error_type == 'normal': + pred_unmasked = np.zeros((1, 14475, 3), dtype=np.float32) + pred_unmasked[0, masks[id], :] = pred + gt_unmasked = np.zeros((1, 14475, 3), dtype=np.float32) + gt_unmasked[0, masks[id], :] = gt + metric = normal_loss(gt_unmasked, pred_unmasked, masks[id][None,:].astype(np.float32)).numpy() + else: + raise ValueError('Error type not supported') + errors[id + '_' + frame_nr] = metric per_garment_errors[garment_type_table[id]].append(metric) @@ -112,7 +142,7 @@ if __name__ == '__main__': for key in per_garment_errors.keys(): print(key + ': ' + str(np.mean(per_garment_errors[key]))) - with open('/home/workspace/data/results/errors/iccv/' + checkpoint_name + '-' + iteration + '.csv', mode='w') as f: + with open(f'/home/workspace/data/results/errors/iccv/{checkpoint_name}-{iteration}-{error_type}.csv', mode='w') as f: file_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) for key in sorted(errors.keys()): diff --git a/compare_sampling.py b/compare_sampling.py new file mode 100644 index 0000000..c126311 --- /dev/null +++ b/compare_sampling.py @@ -0,0 +1,65 @@ +import os +import sys +import numpy as np +import trimesh as tm +from stl import mesh + +# os.environ["CUDA_VISIBLE_DEVICES"] = '1' +# import tensorflow as tf + +# gpus = tf.config.experimental.list_physical_devices('GPU') +# if gpus: +# try: +# # Currently, memory growth needs to be the same across GPUs +# for gpu in gpus: +# tf.config.experimental.set_memory_growth(gpu, True) +# logical_gpus = tf.config.experimental.list_logical_devices('GPU') +# print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") +# except RuntimeError as e: +# # Memory growth must be set before GPUs have been initialized +# print(e) + +# from losses import collision_loss, normal_loss + + +# from losses import edge_loss_skirt as edge_loss +# from losses import edge_loss + + +if __name__ == '__main__': + frame = '04173_193' # '02276_232', '02978_242', '04173_193' + folder = 'examples/sampling_analysis' + # original_folder = '/home/workspace/data/dataset/local-cache/train_skirt' + original_folder = '/home/workspace/local-data/train' + # folder = 'examples/sampling' + + id = frame.split('_')[0] + mask = np.load(os.path.join(original_folder, 'mesh_mask', f'{id}.npy')) + gt = np.load(os.path.join(folder, frame, f'{frame}_gt.npy')) + faces = np.load(os.path.join(folder, frame, f'{frame}_faces.npy')) + edges = tm.Trimesh(vertices=gt, faces=faces, validate=False, process=False).edges + edges_gt = np.linalg.norm(gt[edges[:, 0]] - gt[edges[:, 1]], axis=1) + area_gt = np.sum(np.linalg.norm(np.cross(gt[faces[:, 0]] - gt[faces[:, 1]], gt[faces[:, 0]] - gt[faces[:, 2]]), axis=1)) + + errors = [] + vars = [] + area_diffs = [] + # for i in [1,2,4,10]: + for i in range(15): + sample = np.load(os.path.join(folder, frame, f'{frame}_{i}.npy')) + edges_sample = np.linalg.norm(sample[edges[:, 0]] - sample[edges[:, 1]], axis=1) + error = np.mean(np.abs(edges_sample - edges_gt)) + var = np.var(np.abs(edges_sample - edges_gt)) + errors.append(error) + vars.append(var) + + area_sample = np.sum(np.linalg.norm(np.cross(sample[faces[:, 0]] - sample[faces[:, 1]], sample[faces[:, 0]] - sample[faces[:, 2]]), axis=1)) + area_diff = np.abs(area_sample - area_gt) + area_diffs.append(area_diff) + + print(np.array(errors).mean()) + print(np.array(vars).mean()) + print(np.array(area_diffs).mean()) + print(area_gt) + + diff --git a/configs/iccv/config_abl_all_skirt_var.py b/configs/iccv/config_abl_all_skirt_var.py index b06d065..e2aecc1 100644 --- a/configs/iccv/config_abl_all_skirt_var.py +++ b/configs/iccv/config_abl_all_skirt_var.py @@ -18,8 +18,8 @@ config['use_norm_loss'] = True ablation_count = [config['use_skinning'], config['remove_global_orientation'], config['use_curvature'], config['use_extra_losses'], config['use_offsets']].count(False) assert ablation_count == 1 or ablation_count == 0 -config['gpu'] = '0' -config['name'] = 'abl_all_skirt_var' +config['gpu'] = '3' +config['name'] = 'abl_all_skirt_var_dasda' config['notes'] = '' config['tags'] = ['iccv', 'abl_all', 'skirt'] @@ -35,9 +35,9 @@ config['is_skirt'] = True config['dataset_in_memory'] = False config['is_cape'] = False -config['train_dir'] = '/home/workspace/local-data_128/train_skirt' +config['train_dir'] = '/home/workspace/local-data/train_skirt' config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' -config['test_dir'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' config['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' config['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' diff --git a/configs/iccv/config_abl_pyr_32.py b/configs/iccv/config_abl_pyr_32.py index 858bfa4..535dfaa 100644 --- a/configs/iccv/config_abl_pyr_32.py +++ b/configs/iccv/config_abl_pyr_32.py @@ -33,6 +33,7 @@ config['pyramid']['final_upscale_enhancer'] = '' config['train_dir'] = '/home/workspace/local-data/train' config['valid_dir'] = '/home/workspace/local-data/valid' config['test_dir'] = '/home/workspace/local-data_127/test_ready' + config['batch_size'] = 4 config['is_skirt'] = False config['is_cape'] = False @@ -93,11 +94,10 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 512 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 7 - level['encoder_depths'] = [3, 3, 9, 3] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (16, 8, 80) - level['decoder_depths'] = [3, 3, 9, 3] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' level['upscale_enhancer'] = '' config['pyramid']['levels'].append(level) @@ -106,13 +106,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 256 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [9, 9, 9, 9] - level['encoder_dims'] = [32, 64, 128, level['latent_dim']] - level['decoder_input'] = (8, 4, 128) - level['decoder_depths'] = [9, 9, 9, 9] - level['decoder_dims'] = [256, 128, 64, 16] + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 128: @@ -120,13 +119,12 @@ for res in config['pyramid']['resolutions']: level['resolution'] = 128 level['latent_dim'] = 64 * config['bottleneck_scale'] level['kernel_size'] = 5 - level['encoder_depths'] = [16, 9, 9, 9] - level['encoder_dims'] = [64, 128, 128, 64] - level['decoder_input'] = (4, 2, 128) - level['decoder_depths'] = [16, 9, 9, 9] - level['decoder_dims'] = [256, 128, 128, level['latent_dim']] + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 64: @@ -139,7 +137,7 @@ for res in config['pyramid']['resolutions']: level['decoder_depths'] = level['encoder_depths'] level['decoder_dims'] = level['encoder_dims'] level['avg_garment_path'] = '' - level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' level['upscale_enhancer_freeze'] = True config['pyramid']['levels'].append(level) elif res == 32: diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512.py b/configs/iccv/config_abl_pyr_32_64_128_256_512.py index 27da067..658c497 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '1' +config['gpu'] = '4' config['name'] = 'iccv_pyr_32_64_128_256_512' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py index eec6410..dd69ff3 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_curv.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '2' +config['gpu'] = '4' config['name'] = 'iccv_pyr_32_64_128_256_512_curv' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'curv'] diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py index 8ee9402..dcd287d 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_reg.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '0' +config['gpu'] = '4' config['name'] = 'iccv_pyr_32_64_128_256_512_reg' config['notes'] = '' config['tags'] = ['pyr', 'iccv', 'reg'] diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py index 59a3ccf..5741895 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_skin.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '1' +config['gpu'] = '4' config['name'] = 'iccv_pyr_32_64_128_256_512_skin' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_64_128_256_512_skirt.py b/configs/iccv/config_abl_pyr_32_64_128_256_512_skirt.py index bb50059..f2917fa 100644 --- a/configs/iccv/config_abl_pyr_32_64_128_256_512_skirt.py +++ b/configs/iccv/config_abl_pyr_32_64_128_256_512_skirt.py @@ -1,6 +1,6 @@ config = dict() -config['gpu'] = '5' +config['gpu'] = '4' config['name'] = 'iccv_pyr_32_64_128_256_512_skirt' config['notes'] = '' config['tags'] = ['pyr', 'iccv'] diff --git a/configs/iccv/config_abl_pyr_32_batch.py b/configs/iccv/config_abl_pyr_32_batch.py new file mode 100644 index 0000000..3b5a5d9 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_batch.py @@ -0,0 +1,187 @@ +config = dict() + +config['gpu'] = '7' +config['name'] = 'iccv_pyr_32_batch' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' + +config['batch_size'] = 64 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_jumpsuit.py b/configs/iccv/config_abl_pyr_32_jumpsuit.py new file mode 100644 index 0000000..c0f6754 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_jumpsuit.py @@ -0,0 +1,190 @@ +config = dict() + +config['gpu'] = '0' +config['name'] = 'iccv_pyr_32_jumpsuit' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['frames_file'] = 'frames_jumpsuit.txt' +config['frames_file_valid'] = 'frames_jumpsuit.txt' +config['frames_file_test'] = 'frames_jumpsuit.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_skirt_batch.py b/configs/iccv/config_abl_pyr_32_skirt_batch.py new file mode 100644 index 0000000..8eda958 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_skirt_batch.py @@ -0,0 +1,193 @@ +config = dict() + +config['gpu'] = '7' +config['name'] = 'iccv_pyr_32_skirt_batch' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train_skirt' +config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['frames_file'] = 'frames_skirt.txt' +config['frames_file_valid'] = 'frames_skirt.txt' +config['frames_file_test'] = 'frames_skirt.txt' + +config['batch_size'] = 128 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (16, 8, 80) + level['decoder_depths'] = [3, 3, 9, 3] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9, 9] + level['encoder_dims'] = [32, 64, 128, level['latent_dim']] + level['decoder_input'] = (8, 4, 128) + level['decoder_depths'] = [9, 9, 9, 9] + level['decoder_dims'] = [256, 128, 64, 16] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [16, 9, 9, 9] + level['encoder_dims'] = [64, 128, 128, 64] + level['decoder_input'] = (4, 2, 128) + level['decoder_depths'] = [16, 9, 9, 9] + level['decoder_dims'] = [256, 128, 128, level['latent_dim']] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_skirt_dress.py b/configs/iccv/config_abl_pyr_32_skirt_dress.py new file mode 100644 index 0000000..6335449 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_skirt_dress.py @@ -0,0 +1,190 @@ +config = dict() + +config['gpu'] = '3' +config['name'] = 'iccv_pyr_32_skirt_dress' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train_skirt' +config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['frames_file'] = 'frames_dress.txt' +config['frames_file_valid'] = 'frames_dress.txt' +config['frames_file_test'] = 'frames_dress.txt' + +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_skirt_skirt.py b/configs/iccv/config_abl_pyr_32_skirt_skirt.py new file mode 100644 index 0000000..a4f7dc1 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_skirt_skirt.py @@ -0,0 +1,190 @@ +config = dict() + +config['gpu'] = '4' +config['name'] = 'iccv_pyr_32_skirt_skirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train_skirt' +config['valid_dir'] = '/home/workspace/local-data_127/valid_skirt' +config['test_dir'] = '/home/workspace/local-data/test_skirt_ready_2' +config['frames_file'] = 'frames_skirt.txt' +config['frames_file_valid'] = 'frames_skirt.txt' +config['frames_file_test'] = 'frames_skirt.txt' + +config['batch_size'] = 4 +config['is_skirt'] = True +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_skirt' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_skirt' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_skirt' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2_skirt'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_skirt'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_skirt'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_skirt'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_top.py b/configs/iccv/config_abl_pyr_32_top.py new file mode 100644 index 0000000..4541d52 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_top.py @@ -0,0 +1,193 @@ +config = dict() + +config['gpu'] = '2' +config['name'] = 'iccv_pyr_32_top' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' +config['test_dir_skirt'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['frames_file'] = 'frames_top.txt' +config['frames_file_valid'] = 'frames_top.txt' +config['frames_file_test'] = 'frames_top.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_trousers.py b/configs/iccv/config_abl_pyr_32_trousers.py new file mode 100644 index 0000000..6cef383 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_trousers.py @@ -0,0 +1,190 @@ +config = dict() + +config['gpu'] = '1' +config['name'] = 'iccv_pyr_32_trousers' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['frames_file'] = 'frames_trousers.txt' +config['frames_file_valid'] = 'frames_trousers.txt' +config['frames_file_test'] = 'frames_trousers.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccv/config_abl_pyr_32_tshirt.py b/configs/iccv/config_abl_pyr_32_tshirt.py new file mode 100644 index 0000000..d318af1 --- /dev/null +++ b/configs/iccv/config_abl_pyr_32_tshirt.py @@ -0,0 +1,193 @@ +config = dict() + +config['gpu'] = '3' +config['name'] = 'iccv_pyr_32_tshirt' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv'] + +# DON'T FORGET TO RESET SSIM LOSS USAGE +# DON'T FORGET TO CHECK MODEL LOADING +# DON'T FORGET TO CHECK IF STORE IN MEMORY +# DON'T FORGET TO CHECK REG LOSS COEFF +# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512 +# DON'T FORGET TO CHECK IF SKIRT +# DON'T FORGET TO CHECK IF CAPE +# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2 +# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS + +config['use_skinning'] = True # and deshaping +config['remove_global_orientation'] = True +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_offsets'] = False +config['use_norm_loss'] = True +config['use_curvature'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/home/workspace/local-data/train' +config['valid_dir'] = '/home/workspace/local-data/valid' +config['test_dir'] = '/home/workspace/local-data_127/test_ready' +config['train_dir_skirt'] = '/home/workspace/local-data_128/train_skirt' +config['valid_dir_skirt'] = '/home/workspace/local-data/valid_skirt' +config['test_dir_skirt'] = '/home/workspace/local-data_128/test_skirt_ready_2' +config['frames_file'] = 'frames_tshirt.txt' +config['frames_file_valid'] = 'frames_tshirt.txt' +config['frames_file_test'] = 'frames_tshirt.txt' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), + 'pose_shape': (72 + 11,), + 'uv_body_posed': (512, 256, 3), + 'mesh_body_posed': (config['smpl_res'], 3,), + 'mesh_posed': (config['smpl_res'], 3,), + 'mesh_unposed': (config['smpl_res'], 3,), + 'uv_unposed': (512, 256, 3), + 'uv_curvature': (512, 256, 7), + } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } + +config['pyramid']['is_label'] = False +config['pyramid']['add_noise'] = False +config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3) + config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7) + config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3) + config['signature_static']['uv_mask' + res_modifier] = (res, res//2) + + +config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1' +config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1' +config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1' +config['pyramid']['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['encoder_depths'] = [3, 3, 3, 9, 3] + level['encoder_dims'] = [16, 32, 64, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = '' + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [3, 3, 9, 3] + level['encoder_dims'] = [64, 128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_256_2'#'checkpoints/up_3_d256_512_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 5 + level['encoder_depths'] = [9, 9, 9] + level['encoder_dims'] = [128, 128, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2'#'checkpoints/up_3_d128_256_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [9, 9] + level['encoder_dims'] = [64, level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2'#checkpoints/up_3_d64_128_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['latent_dim'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 3 + level['encoder_depths'] = [16] + level['encoder_dims'] = [level['latent_dim']] + level['decoder_depths'] = level['encoder_depths'] + level['decoder_dims'] = level['encoder_dims'] + level['avg_garment_path'] = '' + level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2'#'checkpoints/up_3_d32_64_2' + level['upscale_enhancer_freeze'] = True + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + +config['cond_ae_static'] = dict() +config['cond_ae_static']['is_label'] = False +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']] +config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths'] +config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims'] +config['cond_ae_static']['activation'] = 'tanh' + +config['cond_ae_curv'] = dict() +config['cond_ae_curv']['is_label'] = True +config['cond_ae_curv']['kernel_size'] = 7 +config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale'] +config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']] +config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths'] +config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims'] +config['cond_ae_curv']['activation'] = 'tanh' + +config['cond_ae_body'] = dict() +config['cond_ae_body']['is_label'] = False +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] +config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3] +config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']] +config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths'] +config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims'] +config['cond_ae_body']['activation'] = 'tanh' \ No newline at end of file diff --git a/cvae.py b/cvae.py index 27ee8cf..3fddfc8 100644 --- a/cvae.py +++ b/cvae.py @@ -2,6 +2,7 @@ import os import sys from configs.iccv.config_abl_all_skirt_var import config +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -479,7 +480,7 @@ def test(): if __name__ == '__main__': - init_wandb() + # init_wandb() parser = argparse.ArgumentParser(description='') parser.add_argument('--name', type=str) args = vars(parser.parse_args()) diff --git a/cvae_pyramid.py b/cvae_pyramid.py index b2bb5d8..6bea4f3 100644 --- a/cvae_pyramid.py +++ b/cvae_pyramid.py @@ -1,7 +1,8 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_128_256_512 import config +from configs.iccv.config_abl_pyr_32_tshirt import config +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf @@ -417,6 +418,7 @@ def test_step(batch): def plot_everything(batch: dict, type: str): + nr_to_plot = 4 if config['use_skinning']: mesh_orig, uv_orig = batch['mesh_unposed'].numpy(), batch['uv_unposed'].numpy() else: @@ -426,8 +428,8 @@ def plot_everything(batch: dict, type: str): output_levels = model(levels_input, training=False) mesh_pred_levels = [uv2mesh(output_levels[i][0]).numpy() for i in range(len(output_levels))] - mesh_orig_posed = batch['mesh_posed'].numpy() - body = batch['mesh_body_posed'].numpy() + mesh_orig_posed = batch['mesh_posed'].numpy()[:nr_to_plot] + body = batch['mesh_body_posed'].numpy()[:nr_to_plot] orig_img = (uv_orig + 1) / 2 uv_static_img = (batch['uv_static'].numpy() + 1) / 2 @@ -436,10 +438,10 @@ def plot_everything(batch: dict, type: str): pred_img = np.concatenate([tf.image.resize(output_levels[i][0], (512, 256), tf.image.ResizeMethod.LANCZOS5) for i in range(len(output_levels))], axis=2) pred_img = (pred_img - pred_img.min(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis]) / (pred_img.max(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis] - pred_img.min(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis]) img = tf.concat((uv_static_img, uv_body_posed_img, uv_curvature, orig_img, pred_img), axis=2) - imgs = [wandb.Image(img[i]) for i in range(4)] + imgs = [wandb.Image(img[i]) for i in range(nr_to_plot)] F = np.load("custom_utils/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy") - mask = batch['mesh_mask'].numpy().astype(bool) + mask = batch['mesh_mask'].numpy().astype(bool)[:nr_to_plot] _, faces = maskFacesBatch(np.ones_like(mesh_orig), F, mask) if config['use_skinning']: @@ -448,30 +450,30 @@ def plot_everything(batch: dict, type: str): shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]).numpy() - gt = [mesh_orig[i][mask[i], :] + shape_offsets[i, mask[i], :] for i in range(mask.shape[0])] + gt = [mesh_orig[i][mask[i], :] + shape_offsets[i, mask[i], :] for i in range(nr_to_plot)] - mesh_unposed_shaped_levels = [[] for _ in range(mask.shape[0])] + mesh_unposed_shaped_levels = [[] for _ in range(nr_to_plot)] for i in range(len(output_levels)): - mesh_unposed_shaped = [mesh_pred_levels[i][j, mask[j], :] + shape_offsets[j, mask[j]] for j in range(mask.shape[0])] - for k in range(mask.shape[0]): + mesh_unposed_shaped = [mesh_pred_levels[i][j, mask[j], :] + shape_offsets[j, mask[j]] for j in range(nr_to_plot)] + for k in range(nr_to_plot): mesh_unposed_shaped_levels[k].append(mesh_unposed_shaped[k]) # full res version - mesh_unposed_shaped_full = np.zeros((mask.shape[0], config['smpl_res'], 3), dtype=np.float32) - for i in range(mask.shape[0]): + mesh_unposed_shaped_full = np.zeros((nr_to_plot, config['smpl_res'], 3), dtype=np.float32) + for i in range(nr_to_plot): mesh_unposed_shaped_full[i, mask[i], :] = mesh_unposed_shaped[i] - mesh_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_unposed_shaped_full).numpy() + mesh_posed = smpl.run_garment(batch['pose_shape'][:nr_to_plot, :72], batch['pose_shape'][:nr_to_plot, 72:], mesh_unposed_shaped_full).numpy() else: mesh_posed = mesh_pred_levels[-1] - gt = [mesh_orig[i][mask[i], :] for i in range(mesh_posed.shape[0])] - mesh_unposed_shaped_levels = [[mesh_pred_levels[i][j, mask[j], :] for i in range(len(output_levels))] for j in range(mask.shape[0])] + gt = [mesh_orig[i][mask[i], :] for i in range(nr_to_plot)] + mesh_unposed_shaped_levels = [[mesh_pred_levels[i][j, mask[j], :] for i in range(len(output_levels))] for j in range(nr_to_plot)] - gt_posed = [mesh_orig_posed[i, mask[i], :] for i in range(mask.shape[0])] + gt_posed = [mesh_orig_posed[i, mask[i], :] for i in range(nr_to_plot)] mesh_posed_fixed = fix_collision(mesh_posed, body, mask).numpy() - mesh_posed = [mesh_posed[j, mask[j], :] for j in range(mask.shape[0])] + mesh_posed = [mesh_posed[j, mask[j], :] for j in range(nr_to_plot)] if config['is_cape']: collision = mesh_posed @@ -487,11 +489,11 @@ def plot_everything(batch: dict, type: str): if config['is_cape']: figs_coll = dict() else: - garment_posed_fixed = [mesh_posed_fixed[i, mask[i], :] for i in range(body.shape[0])] + garment_posed_fixed = [mesh_posed_fixed[i, mask[i], :] for i in range(nr_to_plot)] collision_fixed = [np.concatenate((body[i], garment_posed_fixed[i]), axis=0) for i in range(len(mesh_posed))] - gt_posed = [batch['mesh_posed'].numpy()[i, mask[i], :] for i in range(mask.shape[0])] + gt_posed = [batch['mesh_posed'].numpy()[i, mask[i], :] for i in range(nr_to_plot)] collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(gt_posed))] - figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i]) for i in range(4) } + figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i]) for i in range(nr_to_plot) } return { **figs, **figs_coll, **{ type + '/images': imgs, } } @@ -728,14 +730,23 @@ def generate_results(): pass @tf.function -def speed_step(levels_input, conds): +def speed_step(levels_input, conds, pose_shape, mask): + shape = pose_shape[:, 72:] + gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32) + shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]) - return model.sample([levels_input, conds]) + # pred = model.sample([levels_input, conds]) + out_levels = model([levels_input, conds], training=False) + pred = out_levels[-1][0] + mesh_pred = uv2mesh(pred) + mesh_pred_shaped = mesh_pred + shape_offsets * mask[:,:,tf.newaxis] + mesh_pred_posed = smpl.run_garment(pose_shape[:, :72], pose_shape[:, 72:], mesh_pred_shaped) + return mesh_pred_posed if __name__ == '__main__': - # init_wandb() + init_wandb() parser = argparse.ArgumentParser(description='') parser.add_argument('--name', type=str, default='lhunor') args = vars(parser.parse_args()) @@ -760,8 +771,12 @@ if __name__ == '__main__': resolutions=dataset_resolutions, in_memory=config['dataset_in_memory'], is_cape=config['is_cape'], - frames_file=config['frames_file'] if 'frames_file' in config.keys() else None) + frames_file=config['frames_file'] if 'frames_file' in config.keys() else None, + frames_file_skirt=config['frames_file'] if 'frames_file' in config.keys() else '', + folder_skirt=config['train_dir_skirt'] if 'train_dir_skirt' in config.keys() else '', + ) static_dataset_train = StaticDataset(folder=config['train_dir'], + folder_skirt=config['train_dir_skirt'] if 'train_dir_skirt' in config.keys() else '', signature=config['signature_static'], resolutions=dataset_resolutions, in_memory=True, @@ -772,8 +787,12 @@ if __name__ == '__main__': resolutions=dataset_resolutions, in_memory=config['dataset_in_memory'], is_cape=config['is_cape'], - frames_file=config['frames_file_valid'] if 'frames_file_valid' in config.keys() else None) + frames_file=config['frames_file_valid'] if 'frames_file_valid' in config.keys() else None, + frames_file_skirt=config['frames_file_valid'] if 'frames_file_valid' in config.keys() else '', + folder_skirt=config['valid_dir_skirt'] if 'valid_dir_skirt' in config.keys() else '', + ) static_dataset_valid = StaticDataset(folder=config['valid_dir'], + folder_skirt=config['valid_dir_skirt'] if 'valid_dir_skirt' in config.keys() else '', signature=config['signature_static'], resolutions=dataset_resolutions, in_memory=True, @@ -848,50 +867,50 @@ if __name__ == '__main__': levels_input, conds = group_inputs_for_levels(batch, config) - start = time.perf_counter() - out = speed_step(levels_input, conds) - t = time.perf_counter() - start - print(t) - times.append(t) + # start = time.perf_counter() + # out = speed_step(levels_input, conds, batch['pose_shape'] ,batch['mesh_mask']) + # t = time.perf_counter() - start + # print(t) + # times.append(t) - if step > 200: - break + # if step > 200: + # break - # train_loss = train_step(batch) - # train_accumulator.update(train_loss) + train_loss = train_step(batch) + train_accumulator.update(train_loss) - # if step % 25 == 0: - # data_valid = valid_iterator.get_next() - # data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic'])) - # valid_loss = valid_step(data_valid) - # valid_accumulator.update(valid_loss) + if step % 25 == 0: + data_valid = valid_iterator.get_next() + data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic'])) + valid_loss = valid_step(data_valid) + valid_accumulator.update(valid_loss) - # if train_loss['train/euclid_error'] < 0.015: - # if balancing_activated_step == 0: - # balancing_activated_step = step - # balancing_coeff = balancing_coeff + 1 / 20000 + if train_loss['train/euclid_error'] < 0.015: + if balancing_activated_step == 0: + balancing_activated_step = step + balancing_coeff = balancing_coeff + 1 / 20000 - # if step % 1000 == 0 or (step < 1000 and step % 100 == 0): - # to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } }) - # if step % 5000 == 0 or (step < 5000 and step % 1000 == 0): - # train_figs = plot_everything(batch, 'train') - # valid_figs = plot_everything(data_valid, 'valid') - # to_log = { **to_log, **train_figs, **valid_figs } - # if step == 0: - # to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } } + if step % 1000 == 0 or (step < 1000 and step % 100 == 0): + to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } }) + if step % 5000 == 0 or (step < 5000 and step % 1000 == 0): + train_figs = plot_everything(batch, 'train') + valid_figs = plot_everything(data_valid, 'valid') + to_log = { **to_log, **train_figs, **valid_figs } + if step == 0: + to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } } - # to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } } + to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } } - # wandb.log(to_log, step=step) - # train_accumulator.reset() - # valid_accumulator.reset() + wandb.log(to_log, step=step) + train_accumulator.reset() + valid_accumulator.reset() - # if step % 5000 == 0: - # manager.save(checkpoint_number=step) + if step % 5000 == 0: + manager.save(checkpoint_number=step) - # print(step) + print(step) step += 1 - print('mean:') - print(np.mean(times[10:])) \ No newline at end of file + # print('mean:') + # print(np.mean(times[10:])) \ No newline at end of file diff --git a/dataset.py b/dataset.py index 95f43eb..fd4edac 100644 --- a/dataset.py +++ b/dataset.py @@ -54,10 +54,11 @@ from custom_utils.custom_utils import nest_conditions, batch_to_dict class StaticDataset(): - def __init__(self, folder, signature, use_filled=False, resolutions=[512], in_memory=True, is_cape=False) -> None: + def __init__(self, folder, signature, folder_skirt='', use_filled=False, resolutions=[512], in_memory=True, is_cape=False) -> None: self.store = dict() self.signature = signature self.folder = folder + self.folder_skirt = folder_skirt self.in_memory = in_memory assert set(resolutions).issubset(set([32, 64, 128, 256, 512])) self.possible_keys = ['uv_static', 'uv_static_32', 'uv_static_64', 'uv_static_128', 'uv_static_256', @@ -85,6 +86,9 @@ class StaticDataset(): def read(self, id): data = dict() + temp_folder = self.folder + if not os.path.exists(os.path.join(temp_folder, 'uv_mask', id + '.npz')): + temp_folder = self.folder_skirt for res in self.resolutions: res_string = '' @@ -92,23 +96,23 @@ class StaticDataset(): res_string = '_' + str(res) if 'uv_static' + res_string in self.signature.keys(): - uv_static = np.load(os.path.join(self.folder, 'uv_static' + self.filled + res_string, id + '.npz')); uv_static = uv_static['uv_static' + self.filled]; data['uv_static' + res_string] = uv_static + uv_static = np.load(os.path.join(temp_folder, 'uv_static' + self.filled + res_string, id + '.npz')); uv_static = uv_static['uv_static' + self.filled]; data['uv_static' + res_string] = uv_static if 'uv_mask' + res_string in self.signature.keys(): if self.is_cape: data['uv_mask' + res_string] = self.uv_masks[res] else: - uv_mask = np.load(os.path.join(self.folder, 'uv_mask' + res_string, id + '.npz')); uv_mask=uv_mask['uv_mask']; uv_mask = uv_mask.astype(np.float32); data['uv_mask' + res_string] = uv_mask + uv_mask = np.load(os.path.join(temp_folder, 'uv_mask' + res_string, id + '.npz')); uv_mask=uv_mask['uv_mask']; uv_mask = uv_mask.astype(np.float32); data['uv_mask' + res_string] = uv_mask if 'mesh_static' in self.signature.keys(): - mesh_static = np.load(os.path.join(self.folder, 'mesh_static', id + '.npz')); mesh_static = mesh_static['mesh_static']; data['mesh_static'] = mesh_static.astype(np.float32) + mesh_static = np.load(os.path.join(temp_folder, 'mesh_static', id + '.npz')); mesh_static = mesh_static['mesh_static']; data['mesh_static'] = mesh_static.astype(np.float32) if 'mesh_mask' in self.signature.keys(): if self.is_cape: data['mesh_mask'] = np.ones((3638), np.float32) else: - mesh_mask = np.load(os.path.join(self.folder, 'mesh_mask', id + '.npy')); mesh_mask = mesh_mask.astype(np.float32); data['mesh_mask'] = mesh_mask + mesh_mask = np.load(os.path.join(temp_folder, 'mesh_mask', id + '.npy')); mesh_mask = mesh_mask.astype(np.float32); data['mesh_mask'] = mesh_mask if 'uv_shaped_body' in self.signature.keys(): - uv_shaped_body = np.load(os.path.join(self.folder, 'uv_shaped_body', id + '.npz')); uv_shaped_body = uv_shaped_body['uv_shaped_body']; data['uv_shaped_body'] = uv_shaped_body + uv_shaped_body = np.load(os.path.join(temp_folder, 'uv_shaped_body', id + '.npz')); uv_shaped_body = uv_shaped_body['uv_shaped_body']; data['uv_shaped_body'] = uv_shaped_body return data @@ -135,7 +139,7 @@ class StaticDataset(): class DatasetIdx(tf.data.Dataset): - def __new__(cls, folder, shuffle=True, is_cape=False, testing=False, frames_file=None): + def __new__(cls, folder, shuffle=True, is_cape=False, testing=False, frames_file=None, folder_skirt=None, frames_file_skirt=''): if frames_file is None: uv_static_folder = os.path.join(folder, 'pose_shape') frames = sorted(os.listdir(uv_static_folder)) @@ -156,6 +160,12 @@ class DatasetIdx(tf.data.Dataset): lines = [line.strip() for line in lines] frames = lines + if frames_file_skirt != '': + with open(os.path.join(folder_skirt, frames_file_skirt), 'r') as f: + lines = f.readlines() + lines = [line.strip() for line in lines] + frames.extend(lines) + if shuffle: random.shuffle(frames) @@ -194,10 +204,12 @@ class DatasetFactory(): use_one_hot=False, use_no_orient=True, testing=False, - frames_file=None): + frames_file=None, + frames_file_skirt='', + folder_skirt='',): assert set(resolutions).issubset(set([32, 64, 128, 256, 512])) - dataset_idx = DatasetIdx(folder, shuffle, is_cape=is_cape, testing=testing, frames_file=frames_file) + dataset_idx = DatasetIdx(folder, shuffle, is_cape=is_cape, testing=testing, folder_skirt=folder_skirt, frames_file=frames_file, frames_file_skirt=frames_file_skirt) separator = '_' if is_cape: @@ -222,6 +234,11 @@ class DatasetFactory(): id, frame_nr = frame_split[0].numpy().decode("utf-8"), frame_split[1].numpy().decode("utf-8") # id, frame_nr = frame_split[0].numpy(), frame_split[1].numpy() # static_info = static_data[id] + + temp_folder = folder + if not os.path.exists(os.path.join(temp_folder, 'pose_shape', id + separator + frame_nr + '.npy')): + temp_folder = folder_skirt + item = dict() if 'id' in signature.keys(): @@ -229,23 +246,23 @@ class DatasetFactory(): if 'frame_nr' in signature.keys(): item['frame_nr'] = frame_nr if 'pose_shape' in signature.keys(): - pose_shape = np.load(os.path.join(folder, 'pose_shape', id + separator + frame_nr + '.npy')) + pose_shape = np.load(os.path.join(temp_folder, 'pose_shape', id + separator + frame_nr + '.npy')) if use_no_orient: pose_shape[:3] = 0 item['pose_shape'] = pose_shape if 'mesh_body_posed' in signature.keys(): if use_no_orient: - mesh_posed_body_no_orient = np.load(os.path.join(folder, 'mesh_posed_body_no_orient', id + separator + frame_nr + '.npz')); mesh_posed_body_no_orient = mesh_posed_body_no_orient['mesh_posed_body_no_orient'] + mesh_posed_body_no_orient = np.load(os.path.join(temp_folder, 'mesh_posed_body_no_orient', id + separator + frame_nr + '.npz')); mesh_posed_body_no_orient = mesh_posed_body_no_orient['mesh_posed_body_no_orient'] item['mesh_body_posed'] = mesh_posed_body_no_orient else: - mesh_body_posed = np.load(os.path.join(folder, 'mesh_body_posed', id + '_' + frame_nr + '.npy')) + mesh_body_posed = np.load(os.path.join(temp_folder, 'mesh_body_posed', id + '_' + frame_nr + '.npy')) item['mesh_body_posed'] = mesh_body_posed if 'mesh_posed' in signature.keys(): if use_no_orient: - mesh_posed = np.load(os.path.join(folder, 'mesh_posed_no_orient', id + separator + frame_nr + '.npz')); mesh_posed = mesh_posed['mesh_posed_no_orient'].astype(np.float32) + mesh_posed = np.load(os.path.join(temp_folder, 'mesh_posed_no_orient', id + separator + frame_nr + '.npz')); mesh_posed = mesh_posed['mesh_posed_no_orient'].astype(np.float32) item['mesh_posed'] = mesh_posed else: - mesh_posed = np.load(os.path.join(folder, 'mesh_posed', id + separator + frame_nr + '.npz')); mesh_posed = mesh_posed['mesh_posed'].astype(np.float32) + mesh_posed = np.load(os.path.join(temp_folder, 'mesh_posed', id + separator + frame_nr + '.npz')); mesh_posed = mesh_posed['mesh_posed'].astype(np.float32) item['mesh_posed'] = mesh_posed # if 'mesh_unposed' in signature.keys(): # mesh_unposed = np.load(os.path.join(folder, 'mesh_unposed', id + '_' + frame_nr + '.npz')); mesh_unposed = mesh_unposed['mesh_unposed'] @@ -255,12 +272,12 @@ class DatasetFactory(): # item['uv_unposed'] = uv_unposed if 'mesh_unposed' in signature.keys(): # print('one_hot') - mesh_unposed_no_shape = np.load(os.path.join(folder, 'mesh_unposed_no_shape' + one_hot, id + separator + frame_nr + '.npz')); mesh_unposed_no_shape = mesh_unposed_no_shape['mesh_unposed_no_shape'] + mesh_unposed_no_shape = np.load(os.path.join(temp_folder, 'mesh_unposed_no_shape' + one_hot, id + separator + frame_nr + '.npz')); mesh_unposed_no_shape = mesh_unposed_no_shape['mesh_unposed_no_shape'] item['mesh_unposed'] = mesh_unposed_no_shape if 'mesh_curvature' in signature.keys(): - mesh_curvature = np.load(os.path.join(folder, 'mesh_curvature', id + separator + frame_nr + '.npz')); mesh_curvature = mesh_curvature['mesh_curvature'] + mesh_curvature = np.load(os.path.join(temp_folder, 'mesh_curvature', id + separator + frame_nr + '.npz')); mesh_curvature = mesh_curvature['mesh_curvature'] item['mesh_curvature'] = mesh_curvature.astype(np.int32) @@ -270,7 +287,7 @@ class DatasetFactory(): res_string = '_' + str(res) if 'uv_curvature' + res_string in signature.keys(): - uv_curvature = np.load(os.path.join(folder, 'uv_curvature' + res_string, id + separator + frame_nr + '.npz')); + uv_curvature = np.load(os.path.join(temp_folder, 'uv_curvature' + res_string, id + separator + frame_nr + '.npz')); if res_string != '': uv_curvature = uv_curvature['uv_curvature'] else: @@ -279,24 +296,24 @@ class DatasetFactory(): item['uv_curvature' + res_string] = uv_curvature if 'uv_unposed' + res_string in signature.keys(): - uv_unposed_no_shape = np.load(os.path.join(folder, 'uv_unposed_no_shape' + one_hot_2 + res_string, id + separator + frame_nr + '.npz')); uv_unposed_no_shape = uv_unposed_no_shape['uv_unposed_no_shape'] + uv_unposed_no_shape = np.load(os.path.join(temp_folder, 'uv_unposed_no_shape' + one_hot_2 + res_string, id + separator + frame_nr + '.npz')); uv_unposed_no_shape = uv_unposed_no_shape['uv_unposed_no_shape'] item['uv_unposed' + res_string] = uv_unposed_no_shape if 'uv_body_posed' + res_string in signature.keys(): if use_no_orient: - uv_posed_body_no_orient = np.load(os.path.join(folder, uv_posed_body_no_orient_folder_name + res_string, id + separator + frame_nr + '.npz')) + uv_posed_body_no_orient = np.load(os.path.join(temp_folder, uv_posed_body_no_orient_folder_name + res_string, id + separator + frame_nr + '.npz')) uv_posed_body_no_orient = uv_posed_body_no_orient['uv_posed_body_no_orient'] item['uv_body_posed' + res_string] = uv_posed_body_no_orient else: - uv_body_posed = np.load(os.path.join(folder, 'uv_body_posed', id + '_' + frame_nr + '.npz')); uv_body_posed = uv_body_posed['uv_body'] + uv_body_posed = np.load(os.path.join(temp_folder, 'uv_body_posed', id + '_' + frame_nr + '.npz')); uv_body_posed = uv_body_posed['uv_body'] item['uv_body_posed' + res_string] = uv_body_posed if 'uv_posed' + res_string in signature.keys(): if use_no_orient: - uv_posed = np.load(os.path.join(folder, 'uv_posed_no_orient' + res_string, id + separator + frame_nr + '.npz')); uv_posed = uv_posed['uv_posed_no_orient'] + uv_posed = np.load(os.path.join(temp_folder, 'uv_posed_no_orient' + res_string, id + separator + frame_nr + '.npz')); uv_posed = uv_posed['uv_posed_no_orient'] item['uv_posed' + res_string] = uv_posed else: - uv_posed = np.load(os.path.join(folder, 'uv_posed' + res_string, id + separator + frame_nr + '.npz')); uv_posed = uv_posed['uv_gt'] + uv_posed = np.load(os.path.join(temp_folder, 'uv_posed' + res_string, id + separator + frame_nr + '.npz')); uv_posed = uv_posed['uv_gt'] item['uv_posed' + res_string] = uv_posed # print('here') diff --git a/evaluate_ae_pyramid.py b/evaluate_ae_pyramid.py index 334b6b3..a582d20 100644 --- a/evaluate_ae_pyramid.py +++ b/evaluate_ae_pyramid.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_abl_pyr_32_64_10p import config +from configs.iccv.config_abl_pyr_32_64_128_skin_skirt import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/get_error.py b/get_error.py index 03a21c4..5379a97 100644 --- a/get_error.py +++ b/get_error.py @@ -1,5 +1,6 @@ import os import csv +import glob import numpy as np @@ -7,15 +8,12 @@ if __name__ == '__main__': garment_type_table = np.load('custom_data/garment_type_table_test.npy', allow_pickle=True).item() folder = '/home/workspace/data/results/errors/iccv' - checkpoint_name = 'iccv_pyr_32_64_25p' - checkpoint_name_skirt = 'iccv_pyr_32_64_128_256_512_skirt' - iteration = '155000' - iteration_skirt : str = '115000' + checkpoint_name = 'iccv_pyr_32_64_128_skin' + checkpoint_name_skirt = 'iccv_pyr_32_64_128_skin_skirt' + iteration = '75' + '000' + iteration_skirt : str = '155' + '000' + error_type = 'collision' # 'euclidean' 'collision' edge normal - # checkpoint_name = 'abl_skin' - # checkpoint_name_skirt = checkpoint_name + '_skirt' - # iteration = '70000' - # iteration_skirt : str = iteration errors = [] errors_skirt = [] @@ -23,20 +21,20 @@ if __name__ == '__main__': per_garment_errors_skirt = { 'Tshirt': [], 'Dress': [], 'Skirt': [], 'Top': [] } - with open(os.path.join(folder, checkpoint_name + '-' + iteration + '.csv'), 'r') as f: - csv_reader = csv.reader(f, delimiter=',') - for row in csv_reader: - per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) + # with open(os.path.join(folder, f'{checkpoint_name}-{iteration}-{error_type}.csv'), 'r') as f: + # csv_reader = csv.reader(f, delimiter=',') + # for row in csv_reader: + # per_garment_errors[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) - print('\nNon-skirt:') - for key in per_garment_errors.keys(): - avg = np.mean(per_garment_errors[key]) - errors.append(avg) - print(key + ': ' + str(avg)) + # print('\nNon-skirt:') + # for key in per_garment_errors.keys(): + # avg = np.mean(per_garment_errors[key]) + # errors.append(avg) + # print(key + ': ' + str(avg)) - print('\nAvg non-skirt: ' + str(np.mean(errors))) + # print('\nAvg non-skirt: ' + str(np.mean(errors))) - with open(os.path.join(folder, checkpoint_name_skirt + '-' + iteration_skirt + '.csv'), 'r') as f: + with open(os.path.join(folder, f'{checkpoint_name_skirt}-{iteration_skirt}-{error_type}.csv'), 'r') as f: csv_reader = csv.reader(f, delimiter=',') for row in csv_reader: per_garment_errors_skirt[garment_type_table[row[0].split('_')[0]]].append(float(row[1])) @@ -49,24 +47,24 @@ if __name__ == '__main__': print('Avg skirt: ' + str(np.mean(errors_skirt))) - cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt']) - cum_avg_top = np.mean(per_garment_errors['Top'] + per_garment_errors_skirt['Top']) + # cum_avg_shirt = np.mean(per_garment_errors['Tshirt'] + per_garment_errors_skirt['Tshirt']) + # cum_avg_top = np.mean(per_garment_errors['Top'] + per_garment_errors_skirt['Top']) - print('Global tshirt avg: ' + str(cum_avg_shirt)) - print('Global top avg: ' + str(cum_avg_top)) + # print('Global tshirt avg: ' + str(cum_avg_shirt)) + # print('Global top avg: ' + str(cum_avg_top)) - print('\nGlobal per garment: ') + # print('\nGlobal per garment: ') - global_errors = [] - per_garment_errors['Tshirt'].extend(per_garment_errors_skirt['Tshirt']) - per_garment_errors['Top'].extend(per_garment_errors_skirt['Top']) - per_garment_errors['Skirt'] = per_garment_errors_skirt['Skirt'] - per_garment_errors['Dress'] = per_garment_errors_skirt['Dress'] - for key in per_garment_errors.keys(): - avg = np.mean(per_garment_errors[key]) - global_errors.append(avg) - print(key + ': ' + str(avg), 'samples: ' + str(len(per_garment_errors[key]))) + # global_errors = [] + # per_garment_errors['Tshirt'].extend(per_garment_errors_skirt['Tshirt']) + # per_garment_errors['Top'].extend(per_garment_errors_skirt['Top']) + # per_garment_errors['Skirt'] = per_garment_errors_skirt['Skirt'] + # per_garment_errors['Dress'] = per_garment_errors_skirt['Dress'] + # for key in per_garment_errors.keys(): + # avg = np.mean(per_garment_errors[key]) + # global_errors.append(avg) + # print(key + ': ' + str(avg), 'samples: ' + str(len(per_garment_errors[key]))) - print('Global avg: ' + str(np.mean(global_errors))) - print(len(global_errors)) + # print('Global avg: ' + str(np.mean(global_errors))) + # print(len(global_errors)) \ No newline at end of file diff --git a/losses.py b/losses.py index ab802a6..01df2f3 100644 --- a/losses.py +++ b/losses.py @@ -44,7 +44,7 @@ def _nearest_neighbour(V, B): def _tf_nn_parallel(V, B): return tf.map_fn(fn=lambda elem: tf.py_function(func=_nearest_neighbour, inp=elem, Tout=tf.int32), elems=(V,B), fn_output_signature=tf.int32, parallel_iterations=V.shape[0]) -# @tf.function +@tf.function def collision_loss(V, B, mask, thr=.004): B_vn = vertex_normals(B, tf.tile(F_coll[None], [B.shape[0], 1, 1])) @@ -96,6 +96,7 @@ edges_skirt = np.stack(np.nonzero(laplacian_loss.todense())).transpose() def get_edge_mask(masks): return tf.reduce_min(tf.gather(masks, edges, axis=1), axis=-1) +@tf.function def edge_loss(x, y, mask): e_mask = get_edge_mask(mask) x_e = tf.gather(x, edges[:,0], axis=1) - tf.gather(x, edges[:,1], axis=1) @@ -237,13 +238,14 @@ def kl_loss(mean, logvar): kl_loss *= -0.5 return kl_loss - +@tf.function def normal_loss(pred, gt, mask, is_skirt=False, is_smpl_orig=False): f_norm = None if not is_skirt and not is_smpl_orig: f_norm = F_norm elif is_skirt: - f_norm = F_norm_skirt + # f_norm = F_norm_skirt + f_norm = tf.tile(F_coll_skirt[None,:,:], [pred.shape[0],1,1]) elif is_smpl_orig: f_norm = F_norm_orig else: diff --git a/plot_examples.py b/plot_examples.py index ab6c573..6928416 100644 --- a/plot_examples.py +++ b/plot_examples.py @@ -329,30 +329,33 @@ if __name__ == '__main__': x = np.concatenate((body, posed), axis=0) + # calcualte the sum of face ares of posed mesh that has faces_garment faces + # face_areas = np.linalg.norm(np.cross((x[faces[:,0],:] - x[faces[:,1],:]), (x[faces[:,0],:] - x[faces[:,2],:])), axis=1) + # m = tm.Trimesh(x, faces, process=False, validate=False, vertex_colors=colors) - # x = posed - # x_1 = unposed.copy() - # x_1[:, 0] = x_1[:, 0] + 0.75 - # x_2 = unposed_shaped.copy() - # x_2[:, 0] = x_2[:, 0] + 1.75 - # garments = np.concatenate((x, x_1, x_2), axis=0) - # all_faces = np.concatenate((faces_garment, faces_garment + x.shape[0], faces_garment + 2*x.shape[0]), axis=0).astype(np.int32) - # mesh = garments - # faces = all_faces - # fig = go.Figure() - # fig.add_trace(go.Mesh3d( - # x=mesh[:,0], - # y=mesh[:,2], - # z=mesh[:,1], - # i = faces[:,0], - # j = faces[:,1], - # k = faces[:,2], - # text="", - # lighting={ 'ambient':0.8, 'diffuse':0.5, 'roughness':0.5 }, + x = posed + x_1 = unposed.copy() + x_1[:, 0] = x_1[:, 0] + 0.75 + x_2 = unposed_shaped.copy() + x_2[:, 0] = x_2[:, 0] + 1.75 + garments = np.concatenate((x, x_1, x_2), axis=0) + all_faces = np.concatenate((faces_garment, faces_garment + x.shape[0], faces_garment + 2*x.shape[0]), axis=0).astype(np.int32) + mesh = garments + faces = all_faces + fig = go.Figure() + fig.add_trace(go.Mesh3d( + x=mesh[:,0], + y=mesh[:,2], + z=mesh[:,1], + i = faces[:,0], + j = faces[:,1], + k = faces[:,2], + text="", + lighting={ 'ambient':0.8, 'diffuse':0.5, 'roughness':0.5 }, - # ),) - # fig.show() + ),) + fig.show() # fig = plot_fixed(x, faces, colors) @@ -367,7 +370,7 @@ if __name__ == '__main__': # fig.write_image('examples/' + type + '_' + id_str + '_' + frame_nr + '_unposed_shaped.svg') # uv_mapping_example(body, F, colors[:14475, :], np.ones((14475), bool)) - uv_mapping_example_skirt(body, F, colors[:14475, :], np.ones((14475), bool)) + # uv_mapping_example_skirt(body, F, colors[:14475, :], np.ones((14475), bool)) # curvature_example(batch['mesh_curvature'][0].numpy(), uv_curvature, posed, faces_garment, mask) # plot_upscaling(batch['uv_unposed_32'].numpy()[0], batch['uv_unposed'].numpy()[0], batch['uv_mask_32'].numpy()[0], batch['uv_mask'].numpy()[0], faces_garment, colors_garment, mask) diff --git a/preprocessing.py b/preprocessing.py index 4b81f54..5d782ba 100644 --- a/preprocessing.py +++ b/preprocessing.py @@ -216,7 +216,7 @@ def generate_faces(): np.save(os.path.join(output_folder, id + '.npy'), faces.astype(np.int32)) -def generate_frames_file(folder, ratio=0.1): +def generate_frames_file(folder, ratio=0.1, garment='', file_name='frames.txt'): all_frames = os.listdir(os.path.join(folder, 'pose_shape')) ids = [frame.split('_')[0] for frame in all_frames] all_frames = [frame.split('.')[0] for frame in all_frames] @@ -232,17 +232,15 @@ def generate_frames_file(folder, ratio=0.1): selected = [] for id in tqdm.tqdm(sub_ids): - if garment_type_table[id] != 'Dress': + if garment_type_table[id] != garment: continue - if len(selected) > 30000: - break # frames = glob.glob(os.path.join(folder, 'pose_shape', id + '_*')) frames = [frame for frame in all_frames if frame.startswith(id + '_')] frames = [frame.split('/')[-1].split('.')[0] for frame in frames] selected.extend(frames) - with open(os.path.join(folder, 'frames_dress_30k.txt'), 'w') as f: + with open(os.path.join(folder, file_name), 'w') as f: for i, frame in enumerate(selected): f.write(frame + '\n') @@ -293,7 +291,7 @@ def generate_frames_file_filtered_cape(): if __name__ == '__main__': - src_folder = '/home/workspace/local-data/train_skirt' #cloth3d_skirt + src_folder = '/home/workspace/local-data/train' #cloth3d_skirt # src_folder = r'D:\Work\Dataset\validation_2' dest_folder = '/home/workspace/data/dataset/validation' @@ -306,5 +304,17 @@ if __name__ == '__main__': # verify_poses(src_folder) # create_garment_type_table() # generate_faces() - generate_frames_file(src_folder, ratio=1) + # generate_frames_file(src_folder, ratio=1) # generate_frames_file_filtered_cape() + + + # # folders = ['/home/workspace/local-data/train', '/home/workspace/local-data/test_ready', '/home/workspace/local-data/valid'] + # folders = ['/home/workspace/local-data/valid_skirt'] + # # garments = ['Jumpsuit', 'Trousers', 'Tshirt', 'Top'] + # # filename = ['frames_jumpsuit.txt', 'frames_trousers.txt', 'frames_tshirt.txt', 'frames_top.txt'] + # garments = ['Skirt', 'Dress', 'Tshirt', 'Top'] + # filename = ['frames_skirt.txt', 'frames_dress.txt', 'frames_tshirt.txt', 'frames_top.txt'] + + # for folder in folders: + # for garment, file in zip(garments, filename): + # generate_frames_file(folder, garment=garment, file_name=file, ratio=1) diff --git a/sample_pro_2.py b/sample_pro_2.py index 46d4c65..984191b 100644 --- a/sample_pro_2.py +++ b/sample_pro_2.py @@ -46,7 +46,8 @@ from models.pyramid_model_optimized import Pyramid # from configs.config_abl_overfit_dress_var_3 import config as config_main # from configs.config_abl_overfit_jumpsuit_2 import config as config_main # from configs.config_cvae_curv import config as config_curv -from configs.iccv.config_abl_all_skirt_var import config as config_main +# from configs.iccv.config_abl_all_skirt_var import config as config_main +from configs.iccv.config_abl_skin_var import config as config_main from configs.iccv.config_cvae_curv import config as config_curv # from models.cvae_model_overfit_jumpsuit import CVAE from models.cvae_model import CVAENew as CVAE @@ -331,11 +332,16 @@ def sampling_example(frame, camera, nr=15): for i in range(nr): plot_fixed(posed_mesh_fixed[i], body, faces, camera).write_image(os.path.join(out_folder, frame, frame + '_' + str(i) + '.png')) - save_stl(os.path.join(out_folder, frame, frame + '_body.stl'), body, F_body) - save_stl(os.path.join(out_folder, frame, frame + '_gt.stl'), gt_mesh, faces.astype(np.int32)) + # save_stl(os.path.join(out_folder, frame, frame + '_body.stl'), body, F_body) + # save_stl(os.path.join(out_folder, frame, frame + '_gt.stl'), gt_mesh, faces.astype(np.int32)) + # for i in range(nr): + # save_stl(os.path.join(out_folder, frame, frame + '_' + str(i) + '.stl'), posed_mesh_fixed[i], faces.astype(np.int32)) + np.save(os.path.join(out_folder, frame, frame + '_gt.npy'), gt_mesh) + np.save(os.path.join(out_folder, frame, frame + '_body.npy'), body) + np.save(os.path.join(out_folder, frame, frame + '_faces.npy'), faces.astype(np.int32)) for i in range(nr): - save_stl(os.path.join(out_folder, frame, frame + '_' + str(i) + '.stl'), posed_mesh_fixed[i], faces.astype(np.int32)) - + np.save(os.path.join(out_folder, frame, frame + '_' + str(i) + '.npy'), posed_mesh_fixed[i]) + # custom_imshow(np.concatenate(row_sampling, axis=1)).write_image(os.path.join(out_folder, frame + '_sampling.png')) print() @@ -470,9 +476,9 @@ if __name__ == '__main__': # frame = '04925_272' # frames = ['01070_13', '02276_232', '02978_242', '04008_295', '04173_193'] - frames = ['00214_111', '01661_210', '02144_249', '02151_37', '02930_275', '03812_3'] + frames = ['02276_232', '02978_242', '04173_193'] - out_folder = 'examples/sampling_skirt' + out_folder = 'examples/sampling_analysis' os.makedirs(out_folder, exist_ok=True) for frame in tqdm.tqdm(frames): sampling_example(frame, camera_1)