diff --git a/aaa_helpers.txt b/aaa_helpers.txt index f86226b..38358dd 100644 --- a/aaa_helpers.txt +++ b/aaa_helpers.txt @@ -3,12 +3,14 @@ nohup python calculate_error.py --name error_al_all_150k > nohup/error/post_al_a nohup python auto_encoder_static.py --name="lhunor_static_iccv_1" > nohup/iccv/encoders/static_iccv_1.out 2>&1 & nohup python auto_encoder_static.py --name="lhunor_static_iccv_1_skirt" > nohup/iccv/encoders/static_iccv_1_skirt.out 2>&1 & nohup python auto_encoder_static.py --name="lhunor_static_iccv_1_cape_f" > nohup/iccv/encoders/static_iccv_1_cape_f.out 2>&1 & +nohup python auto_encoder_static.py --name="lhunor_static_iccv_1_cape_m" > nohup/iccv/encoders/static_iccv_1_cape_m.out 2>&1 & nohup python auto_encoder_body.py --name="lhunor_body_iccv_1" > nohup/iccv/encoders/body_iccv_1.out 2>&1 & nohup python auto_encoder_body.py --name="lhunor_body_iccv_1_skirt" > nohup/iccv/encoders/body_iccv_1_skirt.out 2>&1 & +nohup python auto_encoder_body.py --name="lhunor_body_iccv_1_skirt" > nohup/iccv/encoders/body_iccv_1_skirt.out 2>&1 & nohup python auto_encoder_curv.py --name="lhunor_curv_iccv_1 " > nohup/iccv/encoders/curv_iccv_1.out 2>&1 & nohup python auto_encoder_curv.py --name="lhunor_curv_iccv_1_skirt" > nohup/iccv/encoders/curv_iccv_1_skirt.out 2>&1 & nohup python auto_encoder_curv.py --name="lhunor_curv_iccv_1_cape_f" > nohup/iccv/encoders/curv_iccv_1_cape_f.out 2>&1 & -nohup python cvae.py --name="lhunor_cvae_iccv_gnn_1" > nohup/iccv/cvae_iccv_gnn_1.out 2>&1 & +nohup python cvae.py --name="lhunor_cvae_iccv_gnn_decoder_1" > nohup/iccv/cvae_iccv_gnn_decoder_1.out 2>&1 & skirts!!!! pyramid next week diff --git a/auto_encoder_static.py b/auto_encoder_static.py index cc7af10..ea709c4 100644 --- a/auto_encoder_static.py +++ b/auto_encoder_static.py @@ -1,6 +1,6 @@ import os import sys -from configs.iccv.config_ae_static_cape_f import config +from configs.iccv.config_ae_static_cape_m import config os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] import tensorflow as tf diff --git a/configs/iccv/config_abl_all.py b/configs/iccv/config_abl_all.py index 6062ffa..0f68d8a 100644 --- a/configs/iccv/config_abl_all.py +++ b/configs/iccv/config_abl_all.py @@ -19,14 +19,14 @@ config['only_curvatures'] = False ablation_count = [config['use_skinning'], config['remove_global_orientation'], config['use_curvature'], config['use_extra_losses'], config['use_offsets']].count(False) assert ablation_count == 1 or ablation_count == 0 -config['gpu'] = '3' -config['name'] = 'iccv_gnn_3' +config['gpu'] = '4' +config['name'] = 'iccv_gnn_decoder_1' config['notes'] = '' -config['tags'] = ['iccv', 'gnn'] +config['tags'] = ['iccv', 'gnn', 'ggn_decoder'] config['avg_garment_path'] = ''#'custom_data/train_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv train_no_shape_one_hot_avg_uv -if config['use_skinning']: - assert (config['use_offsets'] and config['avg_garment_path'] != '') or (not config['use_offsets'] and config['avg_garment_path'] == '') +# if config['use_skinning']: + # assert (config['use_offsets'] and config['avg_garment_path'] != '') or (not config['use_offsets'] and config['avg_garment_path'] == '') config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision diff --git a/configs/iccv/config_ae_static_cape_m.py b/configs/iccv/config_ae_static_cape_m.py new file mode 100644 index 0000000..c798b22 --- /dev/null +++ b/configs/iccv/config_ae_static_cape_m.py @@ -0,0 +1,51 @@ +config = dict() + + +config['batch_size'] = 4 +config['train_dir'] = '/home/workspace/local-data/cape_processed/train/male' +config['valid_dir'] = '/home/workspace/local-data/cape_processed/test/male' +config['test_dir'] = '/home/workspace/local-data/cape_processed/test/male' +config['use_filled'] = False +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = True +config['is_cape'] = True +config['remove_global_orientation'] = True + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['is_label'] = False +config['cond_ae']['upscale_output'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 8 * config['bottleneck_scale'] +config['cond_ae']['encoder_depths'] = [3, 3, 3, 27, 3] +config['cond_ae']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae']['bottleneck_size']] +config['cond_ae']['decoder_depths'] = config['cond_ae']['encoder_depths'] +config['cond_ae']['decoder_dims'] = config['cond_ae']['encoder_dims'] +config['cond_ae']['activation'] = 'tanh' + + +config['gpu'] = '5' +config['lr'] = 0.0005 +config['name'] = 'static_iccv_1_cape_m' +config['notes'] = ' ' +config['tags'] = ['iccv', 'static', 'cape_m'] + +config['cond_ae']['upscale_output'] = False +config['pretrained_model'] = '' + + +config['signature_dynamic'] = { 'id': (), 'frame_nr': (), } +config['signature_static'] = { 'uv_static': (512, 256, 3), + 'mesh_static': (config['smpl_res'], 3,), + 'mesh_mask': (config['smpl_res'],), + 'uv_mask': (512, 256,), + } \ No newline at end of file diff --git a/cvae.py b/cvae.py index b02cf9f..8a39122 100644 --- a/cvae.py +++ b/cvae.py @@ -88,7 +88,7 @@ def train_step(batch): pred, mean, logvar = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True) - mesh_pred = uv2mesh(pred) + mesh_pred = pred if config['use_skinning']: shape = batch['pose_shape'][:, 72:] @@ -105,18 +105,18 @@ def train_step(batch): e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask'])) c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask']) c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio) - l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2])) + # l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2])) l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1)) n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) # conc_loss = concept_loss(pred, batch['uv_unposed']) kl = kl_loss(mean, logvar) - reconstr_loss = l2_loss + reconstr_loss = l2_mesh_loss if config['use_extra_losses']: if 'block_collision_loss' in config.keys() and config['block_collision_loss'] == True: - reconstr_loss = reconstr_loss + l2_mesh_loss + (e_loss) * config['reg_loss_coeff'] + reconstr_loss = reconstr_loss + (e_loss) * config['reg_loss_coeff'] else: - reconstr_loss = reconstr_loss + l2_mesh_loss + (e_loss + c_loss) * config['reg_loss_coeff'] + reconstr_loss = reconstr_loss + (e_loss + c_loss) * config['reg_loss_coeff'] if config['use_norm_loss'] and step > 2000: reconstr_loss = reconstr_loss + n_loss * config['reg_loss_coeff'] @@ -132,7 +132,7 @@ def train_step(batch): return { 'train/loss': loss, 'train/edge_loss': e_loss, 'train/collision_loss': c_loss, 'train/euclid_error': metric, 'train/reconstr_loss': reconstr_loss, 'train/collision_ratio': c_ratio, 'train/mean': mean, 'train/logvar': logvar, 'train/kl_loss': kl, 'train/var': np.e ** logvar, - 'train/l2_loss': l2_loss, 'train/l2_mesh_loss': l2_mesh_loss, 'train/normal_loss': n_loss, + 'train/l2_mesh_loss': l2_mesh_loss, 'train/normal_loss': n_loss, # 'train/ms_ssim': ms_ssim, } @@ -146,7 +146,7 @@ def valid_step(batch): pred, mean, logvar = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) - mesh_pred = uv2mesh(pred) + mesh_pred = pred if config['use_skinning']: shape = batch['pose_shape'][:, 72:] @@ -163,15 +163,15 @@ def valid_step(batch): e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask'])) c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask']) c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio) - l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2])) + # l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2])) l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1)) n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) # conc_loss = concept_loss(pred, batch['uv_unposed']) kl = kl_loss(mean, logvar) - reconstr_loss = l2_loss + reconstr_loss = l2_mesh_loss if config['use_extra_losses']: - reconstr_loss = reconstr_loss + l2_mesh_loss + (e_loss + c_loss) * config['reg_loss_coeff'] + reconstr_loss = reconstr_loss + (e_loss + c_loss) * config['reg_loss_coeff'] if config['use_norm_loss']: reconstr_loss = reconstr_loss + n_loss * config['reg_loss_coeff'] @@ -256,13 +256,12 @@ def plot_everything(batch: dict, type: str): pred, mean, logvar = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) - mesh_pred = uv2mesh(pred) + mesh_pred = pred.numpy() orig_img = uv_orig uv_static_img = batch['uv_static'] uv_body_posed_img = batch['uv_body_posed'] - pred_img = pred - img = tf.concat((uv_static_img, uv_body_posed_img, orig_img, pred_img), axis=2) + img = tf.concat((uv_static_img, uv_body_posed_img, orig_img), axis=2) imgs = [wandb.Image((img[i] + 1) / 2.0) for i in range(4)] F = np.load("custom_utils/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy") diff --git a/models/gnn.py b/models/gnn.py index ad94aaa..4cd2b01 100644 --- a/models/gnn.py +++ b/models/gnn.py @@ -34,29 +34,39 @@ class Smoother(tf.keras.Model): x = self.conv6([x, self.a], training=training) return x +import numpy as np +import scipy.sparse as sp class GNNDecoder(tf.keras.Model): def __init__(self, ): super().__init__() self.a = sp_matrix_to_sp_tensor(sio.loadmat("custom_utils/ssmpl0.mat")['L'].tocoo()) - self.a_dense = sio.loadmat("custom_utils/ssmpl0.mat")['L'].todense() - + # self.a_dense = sio.loadmat("custom_utils/ssmpl0.mat")['L'].todense() + # self.a_dense = np.ones((100,100), dtype=np.float32) + # self.a = sp_matrix_to_sp_tensor(sp.coo_matrix(self.a_dense)) + self.flatten = Flatten() - self.conv1 = GATConv(channels=1024, attn_heads=4, concat_heads=True, dropout_rate=0.0, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) - self.conv2 = GATConv(channels=512, attn_heads=4, concat_heads=True, dropout_rate=0.1, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) - self.conv3 = GATConv(channels=256, attn_heads=4, concat_heads=True, dropout_rate=0.1, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) + # self.conv1 = GATConv(channels=4, attn_heads=1, concat_heads=True, dropout_rate=0.0, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) + # self.conv2 = GATConv(channels=4, attn_heads=1, concat_heads=True, dropout_rate=0.1, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) + # self.conv3 = GATConv(channels=4, attn_heads=1, concat_heads=True, dropout_rate=0.1, return_attn_coef=False, add_self_loops=True, activation=None, use_bias=True) + self.dense1 = Dense(512) + self.dense2 = Dense(256) + self.conv1 = GCNConv(256, use_bias=True) + self.conv2 = GCNConv(256, use_bias=True) + self.conv3 = GeneralConv(channels=128, batch_norm=True, dropout=0.1, aggregate='sum', activation='prelu', use_bias=True) self.conv4 = GeneralConv(channels=128, batch_norm=True, dropout=0.1, aggregate='sum', activation='prelu', use_bias=True) self.conv5 = GeneralConv(channels=64, batch_norm=True, dropout=0.0, aggregate='sum', activation='prelu', use_bias=True) - self.conv6 = GeneralConv(channels=3, batch_norm=False, dropout=0.0, aggregate='sum', activation='tanh', use_bias=True) + self.conv6 = GCNConv(3, activation='tanh', use_bias=False) def call(self, inputs, training=None): x = inputs x = self.flatten(x) x = tf.tile(x[:,None,:], [1,14475,1]) - - x = self.conv1([x, self.a_dense], training=training) - x = self.conv2([x, self.a_dense], training=training) - x = self.conv3([x, self.a_dense], training=training) + x = self.dense1(x) + x = self.dense2(x) + x = self.conv1([x, self.a], training=training) + x = self.conv2([x, self.a], training=training) + x = self.conv3([x, self.a], training=training) x = self.conv4([x, self.a], training=training) x = self.conv5([x, self.a], training=training) x = self.conv6([x, self.a], training=training)