diff --git a/configs/iccvw/pyr_32.py b/configs/iccvw/pyr_32.py new file mode 100644 index 0000000..6b9335e --- /dev/null +++ b/configs/iccvw/pyr_32.py @@ -0,0 +1,300 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '1' +config['name'] = 'pyr_iccvw_32' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'torch'] +config['script'] = 'scripts/cvae_pyramid.py' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False +config['device'] = 'cuda' + + +config['use_skinning'] = True # and deshaping +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_norm_loss'] = True +config['use_normals'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [32] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['pretrained_pyramid_iteration'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['balancing_activation_limit'] = 0.012 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature']: set = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_body_posed', + 'mesh_posed', + 'mesh_unposed', + 'uv_unposed', + 'uv_normals', + 'uv_static', + 'mesh_mask', + 'uv_mask', + } + +config['pyramid']['is_label'] = False +# config['pyramid']['add_noise'] = False +# config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature'].add('uv_body_posed' + res_modifier) + config['signature'].add('uv_unposed' + res_modifier) + config['signature'].add('uv_normals' + res_modifier) + config['signature'].add('uv_static' + res_modifier) + config['signature'].add('uv_mask' + res_modifier) + + +config['static_model'] = 'checkpoints/iccvw/static_iccvw_4' +config['body_model'] = 'checkpoints/iccvw/body_iccvw_1' +config['normals_model'] = 'checkpoints/iccvw/norm_iccvw_1' +config['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['upscale_enhancer'] = '' + level['input_features'] = 3+3+3+7 if config['use_curvature'] else 3+3+3 + + block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 3), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 3), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 3), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['kernel_size'] = 5 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_256' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+7 if config['use_curvature'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, 128, 3), + CNBlockConfig(128, 128, 3), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 3), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 3), + CNBlockConfig(128, 128, 9), + CNBlockConfig(128, 65, 3), + CNBlockConfig(64, 3, 3), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['kernel_size'] = 5 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_128' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+7 if config['use_curvature'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 9), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 9), + CNBlockConfig(128, 128, 9), + CNBlockConfig(64, 3, 9), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['kernel_size'] = 3 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_64' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+7 if config['use_curvature'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 9), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 64, 9), + CNBlockConfig(64, 3, 9), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['kernel_size'] = 3 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_32' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(level['bottleneck_size'], None, 16), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 3, 16), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + + +####################### cond_ae_norm ############################ +config['cond_ae_norm'] = dict() +config['cond_ae_norm']['device'] = config['device'] +config['cond_ae_norm']['is_label'] = False +# config['cond_ae_norm']['latent_dim'] = config['latent_dim'] +config['cond_ae_norm']['kernel_size'] = 7 +config['cond_ae_norm']['bottleneck_size'] = 40 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae_norm']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_norm']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_norm']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), +] + +config['cond_ae_norm']['encoder_block_settings'] = block_setting +config['cond_ae_norm']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_norm']['activation'] = 'tanh' + + +####################### cond_ae_body ############################ +config['cond_ae_body'] = dict() +config['cond_ae_body']['device'] = config['device'] +config['cond_ae_body']['is_label'] = False +# config['cond_ae_body']['latent_dim'] = config['latent_dim'] +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae_body']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_body']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_body']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), +] + +config['cond_ae_body']['encoder_block_settings'] = block_setting +config['cond_ae_body']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_body']['activation'] = 'tanh' + +####################### cond_ae_static ############################ +config['cond_ae_static'] = dict() +config['cond_ae_static']['device'] = config['device'] +config['cond_ae_static']['is_label'] = False +# config['cond_ae_static']['latent_dim'] = config['latent_dim'] +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(8, 16, 3), + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, config['cond_ae_static']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_static']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_static']['bottleneck_size'], 64, 3), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 8, 3), + CNBlockConfig(8, 3, 3), +] + +config['cond_ae_static']['encoder_block_settings'] = block_setting +config['cond_ae_static']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_static']['activation'] = 'tanh' \ No newline at end of file diff --git a/configs/iccvw/pyr_64.py b/configs/iccvw/pyr_64.py new file mode 100644 index 0000000..fffda56 --- /dev/null +++ b/configs/iccvw/pyr_64.py @@ -0,0 +1,300 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '3' +config['name'] = 'pyr_iccvw_64' +config['notes'] = '' +config['tags'] = ['pyr', 'iccv', 'torch'] +config['script'] = 'scripts/cvae_pyramid.py' + +config['batch_size'] = 4 +config['is_skirt'] = False +config['is_cape'] = False +config['device'] = 'cuda' + + +config['use_skinning'] = True # and deshaping +config['use_extra_losses'] = True # edge, collision, 3d mesh +config['use_norm_loss'] = True +config['use_normals'] = True + +config['pretrained_model'] = '' +config['pretrained_model_iteration'] = '' + +config['pyramid'] = dict() +config['pyramid']['resolutions'] = [64] +config['pyramid']['pretrained_pyramid'] = '' +config['pyramid']['pretrained_pyramid_iteration'] = '' +config['pyramid']['final_upscale_enhancer'] = '' + +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['dataset_in_memory'] = False +config['resolution'] = None + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['lr'] = 0.0001 +config['starting_balancing_coeff'] = 0.001 +config['balancing_activation_limit'] = 0.012 +config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision +config['avg_garment_path'] = '' +config['bottleneck_scale'] = 2 + +config['signature']: set = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_body_posed', + 'mesh_posed', + 'mesh_unposed', + 'uv_unposed', + 'uv_normals', + 'uv_static', + 'mesh_mask', + 'uv_mask', + } + +config['pyramid']['is_label'] = False +# config['pyramid']['add_noise'] = False +# config['pyramid']['noise_amount'] = 0.0 + +config['pyramid']['loss_weights'] = [1,1,1,1,1] + +for res in config['pyramid']['resolutions']: + if res != 512: + res_modifier = '_' + str(res) + config['signature'].add('uv_body_posed' + res_modifier) + config['signature'].add('uv_unposed' + res_modifier) + config['signature'].add('uv_normals' + res_modifier) + config['signature'].add('uv_static' + res_modifier) + config['signature'].add('uv_mask' + res_modifier) + + +config['static_model'] = 'checkpoints/iccvw/static_iccvw_4' +config['body_model'] = 'checkpoints/iccvw/body_iccvw_1' +config['normals_model'] = 'checkpoints/iccvw/norm_iccvw_1' +config['freeze_encoders'] = True + + +config['pyramid']['levels'] = [] +for res in config['pyramid']['resolutions']: + if res == 512: + level = dict() + level['resolution'] = 512 + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['kernel_size'] = 7 + level['upscale_enhancer'] = '' + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 3), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 3), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 3), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 256: + level = dict() + level['resolution'] = 256 + level['kernel_size'] = 5 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_256' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, 128, 3), + CNBlockConfig(128, 128, 3), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 3), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 3), + CNBlockConfig(128, 128, 9), + CNBlockConfig(128, 65, 3), + CNBlockConfig(64, 3, 3), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 128: + level = dict() + level['resolution'] = 128 + level['kernel_size'] = 5 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_128' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 9), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 128, 9), + CNBlockConfig(128, 128, 9), + CNBlockConfig(64, 3, 9), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 64: + level = dict() + level['resolution'] = 64 + level['kernel_size'] = 3 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_64' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(64, level['bottleneck_size'], 9), + CNBlockConfig(level['bottleneck_size'], None, 9), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 64, 9), + CNBlockConfig(64, 3, 9), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + elif res == 32: + level = dict() + level['resolution'] = 32 + level['kernel_size'] = 3 + level['upscale_enhancer'] = 'checkpoints/iccvw/iccv_32' + level['upscale_enhancer_freeze'] = True + level['bottleneck_size'] = 64 * config['bottleneck_scale'] + level['input_features'] = 3+3+3+3 if config['use_normals'] else 3+3+3 + + block_setting = [ + CNBlockConfig(level['bottleneck_size'], None, 16), + ] + + block_setting_reverse = [ + CNBlockConfig(level['bottleneck_size'] * 2, 3, 16), + ] + + level['encoder_block_settings'] = block_setting + level['decoder_block_settings'] = block_setting_reverse + config['pyramid']['levels'].append(level) + else: + raise NotImplementedError + + +####################### cond_ae_norm ############################ +config['cond_ae_norm'] = dict() +config['cond_ae_norm']['device'] = config['device'] +config['cond_ae_norm']['is_label'] = False +# config['cond_ae_norm']['latent_dim'] = config['latent_dim'] +config['cond_ae_norm']['kernel_size'] = 7 +config['cond_ae_norm']['bottleneck_size'] = 40 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae_norm']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_norm']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_norm']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), +] + +config['cond_ae_norm']['encoder_block_settings'] = block_setting +config['cond_ae_norm']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_norm']['activation'] = 'tanh' + + +####################### cond_ae_body ############################ +config['cond_ae_body'] = dict() +config['cond_ae_body']['device'] = config['device'] +config['cond_ae_body']['is_label'] = False +# config['cond_ae_body']['latent_dim'] = config['latent_dim'] +config['cond_ae_body']['kernel_size'] = 7 +config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae_body']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_body']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_body']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), +] + +config['cond_ae_body']['encoder_block_settings'] = block_setting +config['cond_ae_body']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_body']['activation'] = 'tanh' + +####################### cond_ae_static ############################ +config['cond_ae_static'] = dict() +config['cond_ae_static']['device'] = config['device'] +config['cond_ae_static']['is_label'] = False +# config['cond_ae_static']['latent_dim'] = config['latent_dim'] +config['cond_ae_static']['kernel_size'] = 7 +config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(8, 16, 3), + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, config['cond_ae_static']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae_static']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae_static']['bottleneck_size'], 64, 3), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 8, 3), + CNBlockConfig(8, 3, 3), +] + +config['cond_ae_static']['encoder_block_settings'] = block_setting +config['cond_ae_static']['decoder_block_settings'] = block_setting_reverse +config['cond_ae_static']['activation'] = 'tanh' \ No newline at end of file diff --git a/models/pyramid_model.py b/models/pyramid_model.py index 0a041e8..da13b28 100644 --- a/models/pyramid_model.py +++ b/models/pyramid_model.py @@ -12,7 +12,8 @@ import torch.nn.functional as F from utils.checkpoints import load_model from models.base import AEEncoder from models.pyramid_level import PyramidLevel - +from torchvision.transforms import Resize +from torchvision.transforms.functional import InterpolationMode # class Pyramid(nn.Module): @@ -145,17 +146,20 @@ class Pyramid(nn.Module): if config['body_model'] != '': load_model(self.body_enc, config['body_model'] + '_enc') - self.curv_enc : AEEncoder = AEEncoder(config['cond_ae_curv']['encoder_block_settings'], kernel_size=config['cond_ae_curv']['kernel_size'], activation=config['cond_ae_curv']['activation'], is_label=True, nr_of_labels=7) - if config['curvature_model'] != '': - load_model(self.curv_enc, config['curvature_model'] + '_enc') + self.norm_enc : AEEncoder = AEEncoder(config['cond_ae_norm']['encoder_block_settings'], config['cond_ae_norm']['kernel_size'], config['cond_ae_norm']['activation'], device=config['device']) + if config['normals_model'] != '': + load_model(self.norm_enc, config['normals_model'] + '_enc') if config['freeze_encoders']: for param in self.static_enc.parameters(): param.requires_grad = False for param in self.body_enc.parameters(): param.requires_grad = False - for param in self.curv_enc.parameters(): param.requires_grad = False + for param in self.norm_enc.parameters(): param.requires_grad = False self.levels = nn.ModuleList([PyramidLevel({ **config['pyramid']['levels'][i] }) for i in range(len(config['pyramid']['levels']))]) + self.resizers = { res: Resize(size=(res, int(res/2)), interpolation=InterpolationMode.BILINEAR).to(config['device']) for res in config['pyramid']['resolutions'] } + self.resizers[512] = Resize(size=(512, 256), interpolation=InterpolationMode.BILINEAR).to(config['device']) + # if len(config['pyramid']['resolutions']) == 2: # self.levels[-1].decoder.decoder.last = tf.keras.Sequential([tf.keras.layers.Dense(3), tf.keras.layers.Lambda(lambda x: x / 100)]) @@ -187,20 +191,20 @@ class Pyramid(nn.Module): # x = self.upscale_enhancer([low, mask], training=training) # return x - def encode_conds(self, uv_template, uv_body_posed, uv_curv): + def encode_conds(self, uv_template, uv_body_posed, uv_norm): z_static = self.static_enc(uv_template) z_body = self.body_enc(uv_body_posed) - z_curv = self.curv_enc(uv_curv) + z_norm = self.norm_enc(uv_norm) - z_cond = torch.concat([z_static, z_body, z_curv], dim=1) + z_cond = torch.concat([z_static, z_body, z_norm], dim=1) return z_cond def forward(self, inputs): levels_input, conds = inputs - uv_static, uv_body, uv_curv = conds[0], conds[1], conds[2] + uv_static, uv_body, uv_norm = conds[0], conds[1], conds[2] - z_cond = self.encode_conds(uv_static, uv_body, uv_curv) + z_cond = self.encode_conds(uv_static, uv_body, uv_norm) # if self.add_noise == True and training == True: # z_cond = z_cond + (tf.random.uniform(z_cond.shape, maxval=self.noise_amount * 2) - self.noise_amount) @@ -211,13 +215,13 @@ class Pyramid(nn.Module): # pred = pred + levels_input[0][2] # uv_body_posed_32 if 1 != len(self.levels): pred = self.levels[0].enhance(pred, levels_input[0][-2]) # levels_input[i][-2]=uv_mask_res - pred = F.interpolate(pred, size=(self.levels[1].size, self.levels[1].size//2), mode='bilinear') + pred = self.resizers[self.levels[1].size](pred) out_levels.append([pred, mean, logvar]) pred = pred * levels_input[1][-2][:,None,:,:] else: # pred = self.enhance(pred, levels_input[0][-2], training=training) pred = self.levels[0].enhance(pred, levels_input[0][-2]) # levels_input[i][-2]=uv_mask_res - pred = F.interpolate(pred, size=(512, 256), mode='bilinear') + pred = self.resizers[512](pred) pred = pred * levels_input[-1][-1][:,None,:,:] # full res uv mask out_levels.append([pred, mean, logvar]) @@ -233,13 +237,13 @@ class Pyramid(nn.Module): if i != len(self.levels) - 1: pred = self.levels[i].enhance(pred, levels_input[i][-2]) out_levels.append([pred, mean, logvar]) - pred = F.interpolate(pred, size=(self.levels[i+1].size, self.levels[i+1].size//2), mode='bilinear') + pred = self.resizers[self.levels[i+1].size](pred) pred = pred * levels_input[i+1][-2][:,None,:,:] else: # if self.upscale_enhancer is not None: # pred = self.enhance(pred, levels_input[-1][-2], training=training) pred = self.levels[-1].enhance(pred, levels_input[-1][-2]) - pred = F.interpolate(pred, size=(512, 256), mode='bilinear') + pred = self.resizers[512](pred) pred = pred * levels_input[-1][-1][:,None,:,:] # full res uv mask out_levels.append([pred, mean, logvar]) diff --git a/scripts/cvae_pyramid.py b/scripts/cvae_pyramid.py index 32c3fcd..f4ca94b 100644 --- a/scripts/cvae_pyramid.py +++ b/scripts/cvae_pyramid.py @@ -10,11 +10,12 @@ parser.add_argument('--config', type=str) parser.add_argument('--debug', action='store_true') args = vars(parser.parse_args()) -config_module = importlib.import_module(args['config']) -config = config_module.config if args['debug']: - from configs.pyr_32 import config - # from configs.pyr_64_cape_m import config + from configs.iccvw.pyr_32 import config +else: + config_module = importlib.import_module(args['config']) + config = config_module.config + os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] if args['debug']: @@ -42,6 +43,7 @@ import torch.nn.functional as F from data.dataset import Dataset from utils.checkpoints import CheckpointSaver, load_model from utils.utils import InfiniteDataLoader as DataLoader +from utils.utils import text_to_html from utils.postprocess import fix_collision from losses.losses import collision_loss, normal_loss, kl_loss from pytorch_msssim import MS_SSIM @@ -52,7 +54,8 @@ from models.pyramid_model import Pyramid def init_wandb(): - run = wandb.init(project='iccv', name=config['name'], notes=config['notes'], dir='/data-local/127/temp', tags=config['tags']) + wandb.login(host='https://wandb2.lnrweb.com', key='local-2a78b434e9bc51e9f12625f8321a9e3ee34af16d', relogin=True, force=True) + run = wandb.init(project='iccvw', name=config['name'], notes=config['notes'], dir='/data-local/127/temp', tags=config['tags']) artifact = wandb.Artifact('code', type='code') artifact.add_file('scripts/cvae_pyramid.py') artifact.add_file('losses/losses.py') @@ -99,8 +102,8 @@ def plot_3d(mesh_gt, mesh_pred_unposed_levels, faces, mesh_posed, collision, col fig.add_trace(go.Mesh3d( x=mesh_posed[:,0], - y=mesh_posed[:,1], - z=mesh_posed[:,2], + y=mesh_posed[:,2], + z=mesh_posed[:,1], i = faces[:,0], j = faces[:,1], k = faces[:,2], @@ -270,13 +273,37 @@ def train_step(batch): reconstr_loss_total = reconstr_loss_total + n_loss * config['reg_loss_coeff'] kl_total += kl - loss = reconstr_loss_total + g_balancing_coeff * kl_total + loss: torch.Tensor = reconstr_loss_total + g_balancing_coeff * kl_total # scaler.scale(loss).backward() # scaler.step(optimizer) # scaler.update() # optimizer.zero_grad() + # b = 0 + # from custom_utils.custom_utils import _maskFaces + # mesh_mask = batch['mesh_mask'][b].cpu().numpy() + # faces = np.load('smpl/smpl_faces_skirt.npy') if batch['garment'][b] in ['Skirt', 'Dress'] else np.load('smpl/smpl_faces.npy') + # faces_body = np.load('smpl/smpl_faces.npy') + # _, faces = _maskFaces(np.zeros((14475, 3)), faces, mesh_mask) + + # # fig = go.Figure(data=[go.Mesh3d(x=batch['mesh_posed'].cpu().numpy()[0,mesh_mask,0], y=batch['mesh_posed'].cpu().numpy()[0,mesh_mask,2], z=batch['mesh_posed'].cpu().numpy()[0,mesh_mask,1], i=faces[:,0], j=faces[:,1], k=faces[:,2], color='lightpink', opacity=1), + # # go.Mesh3d(x=batch['mesh_body_posed'].cpu().numpy()[0,:,0], y=batch['mesh_body_posed'].cpu().numpy()[0,:,2], z=batch['mesh_body_posed'].cpu().numpy()[0,:,1], i=faces_body[:,0], j=faces_body[:,1], k=faces_body[:,2], color='lightblue', opacity=1), + # # ]) + + # fig = go.Figure(data=[go.Mesh3d(x=mesh_orig.cpu().numpy()[0,mesh_mask,0], y=mesh_orig.cpu().numpy()[0,mesh_mask,2], z=mesh_orig.cpu().numpy()[0,mesh_mask,1], i=faces[:,0], j=faces[:,1], k=faces[:,2], color='lightpink', opacity=1), + # ]) + # fig.update_layout(scene_aspectmode='cube') + # fig.update_layout(scene=dict(xaxis=dict(range=[-1,1]), yaxis=dict(range=[-1,1]), zaxis=dict(range=[-1,1]))) + # fig.show() + + # ex.imshow((uv_orig[b].cpu().numpy().transpose(1,2,0) + 1) / 2).show() + # upscaled = model.levels[0].enhance(levels_input[0][0], levels_input[0][-2]) # levels_input[i][-2]=uv_mask_res + # upscaled = model.resizers[512](upscaled) + # ex.imshow((levels_input[0][0][b].cpu().numpy().transpose(1,2,0) + 1) / 2).show() + # ex.imshow((upscaled[b].cpu().numpy().transpose(1,2,0) + 1) / 2).show() + + optimizer.zero_grad() loss.backward() optimizer.step() @@ -452,12 +479,12 @@ def plot_everything(batch: dict, type: str): orig_img = (torch.permute(uv_orig, [0,2,3,1]) + 1) / 2 uv_static_img = (torch.permute(batch['uv_static'], [0,2,3,1]) + 1) / 2 uv_body_posed_img = (torch.permute(batch['uv_body_posed'], [0,2,3,1]) + 1) / 2 - uv_curvature = (torch.tile(torch.argmax(torch.permute(batch['uv_curvature'], [0,2,3,1]), dim=-1)[:,:,:,None], [1,1,1,3]) / 6) - pred_img = torch.concatenate([torch.permute(F.interpolate(output_levels[i][0], size=(512, 256), mode='bilinear'), [0,2,3,1]) for i in range(len(output_levels))], dim=3) + uv_normals = (torch.permute(batch['uv_normals'], [0,2,3,1]) + 1.4) / 2.8 + pred_img = torch.concatenate([torch.permute(model.resizers[512](output_levels[i][0]), [0,2,3,1]) for i in range(len(output_levels))], dim=3) # pred_img = (pred_img - pred_img.min(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis]) / (pred_img.max(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis] - pred_img.min(axis=(1,2,3))[:,np.newaxis,np.newaxis,np.newaxis]) pred_img = (pred_img - pred_img.flatten().reshape(pred_img.shape[0], -1).min(1)[0][:,None,None,None]) / (pred_img.flatten().reshape(pred_img.shape[0], -1).max(1)[0][:,None,None,None] - pred_img.flatten().reshape(pred_img.shape[0], -1).min(1)[0][:,None,None,None]) - img = torch.concat((uv_static_img, uv_body_posed_img, uv_curvature, orig_img, pred_img), axis=2) + img = torch.concat((uv_static_img, uv_body_posed_img, uv_normals, orig_img, pred_img), axis=2) imgs = [wandb.Image(img[i].cpu().numpy()) for i in range(4)] _F = np.load("smpl/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy") @@ -521,7 +548,14 @@ def plot_everything(batch: dict, type: str): collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(gt_posed))] figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i]) for i in range(4) } - return { **figs, **figs_coll, **{ type + '/images': imgs, } } + batch_id = batch['id'] + batch_frame_nr = batch['frame_nr'].cpu().numpy() + batch_garment = batch['garment'] + batch_info = [f'batch_info: id: {batch_id[i]}, frame_nr: {batch_frame_nr[i]}, garment: {batch_garment[i]}' for i in range(len(batch_id))] + batch_info = text_to_html(batch_info) + batch_info = wandb.Html(batch_info) + + return { **figs, **figs_coll, **{ type + '/images': imgs, }, **{ f'{type}/batch_info': batch_info }, } def sample(folder): @@ -793,7 +827,7 @@ def train(): if train_loss['train/euclid_error'] < config['balancing_activation_limit']: g_balancing_coeff += 1 / 10000 - if step % 10000 == 0: + if step % 25000 == 0: ckpt_saver(model, step, optimizer, None) step += 1 @@ -840,12 +874,12 @@ if __name__ == '__main__': if len(config['pyramid']['resolutions']) == 1: for param in model.static_enc.parameters(): param.requires_grad = False for param in model.body_enc.parameters(): param.requires_grad = False - for param in model.curv_enc.parameters(): param.requires_grad = False + for param in model.norm_enc.parameters(): param.requires_grad = False else: load_model(model, config['pyramid']['pretrained_pyramid'], iteration=(config['pyramid']['pretrained_pyramid_iteration'] if 'pretrained_pyramid_iteration' in config.keys() else '')) for param in model.static_enc.parameters(): param.requires_grad = False for param in model.body_enc.parameters(): param.requires_grad = False - for param in model.curv_enc.parameters(): param.requires_grad = False + for param in model.norm_enc.parameters(): param.requires_grad = False # load_model(model.levels[0].upscale_enhancer, config['pyramid']['levels'][0]['upscale_enhancer']) # model.levels[0].upscale_enhancer.trainable = False @@ -866,6 +900,6 @@ if __name__ == '__main__': if not args['debug']: # model = torch.compile(model) - wandb.watch(model, log='all', log_freq=1000, log_graph=True) + wandb.watch(model, log='all', log_freq=10000, log_graph=True) train() \ No newline at end of file