diff --git a/configs/config_abl_all_post.py b/configs/config_abl_all_post.py index c45040a..1be281a 100644 --- a/configs/config_abl_all_post.py +++ b/configs/config_abl_all_post.py @@ -20,8 +20,8 @@ ablation_count = [config['use_skinning'], config['remove_global_orientation'], c assert ablation_count == 1 or ablation_count == 0 config['gpu'] = '5' -config['name'] = 'iccv_weights_test_3' -config['notes'] = '' +config['name'] = 'iccv_weights_test_5' +config['notes'] = 'smaller weight loss coeff, 1e-6, convolutional weight learning' config['avg_garment_path'] = 'custom_data/train_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv train_no_shape_one_hot_avg_uv if config['use_skinning']: @@ -93,7 +93,8 @@ config['cvae']['encoder_depths'] = [3, 3, 3, 9, 3] config['cvae']['encoder_dims'] = [32, 64, 128, 256, 64] config['cvae']['decoder_depths'] = config['cvae']['encoder_depths'] config['cvae']['decoder_dims'] = config['cvae']['encoder_dims'] - +config['cvae']['decoder_weights_depths'] = [3, 3, 3, 5, 3] +config['cvae']['decoder_weights_dims'] = [24, 32, 64, 128, 256] config['cond_ae_static'] = dict() config['cond_ae_static']['is_label'] = False diff --git a/convnext/models/convnext_tf.py b/convnext/models/convnext_tf.py index 5759ecd..e0fc3a4 100644 --- a/convnext/models/convnext_tf.py +++ b/convnext/models/convnext_tf.py @@ -222,7 +222,7 @@ class ConvNeXtReverse(tf.keras.Model): def __init__(self, depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], kernel_size=7, - drop_path_rate=0., layer_scale_init_value=1e-6, head_init_scale=1., + drop_path_rate=0., layer_scale_init_value=1e-6, head_init_scale=1., learn_weights=False, ): super().__init__() self.upsample_layers = [] # stem and 3 intermediate downsampling conv layers @@ -255,27 +255,20 @@ class ConvNeXtReverse(tf.keras.Model): self.stages.append(stage) cur += depths[i] - self.last = Dense(3, activation='tanh') + if not learn_weights: + self.last = Dense(3, activation='tanh') + else: + self.last = Activation('softmax') + # self.last = Dense(24, activation='softmax') - self.weights_model = tf.keras.Sequential([Block(dim=dims[-1]*2, kernel_size=kernel_size), - Block(dim=dims[-1]*2, kernel_size=kernel_size), - Block(dim=dims[-1]*2, kernel_size=kernel_size), - Flatten(), - Dense(128 * 24), - Reshape((24, 128)), - Dense(14475), - Lambda(lambda x: tf.transpose(x, (0, 2, 1))), - Lambda(lambda x: tf.nn.softmax(x, axis=-1))]) def call(self, x, training): - weights = self.weights_model(x, training=training) - for i in reversed(range(len(self.upsample_layers))): x = self.upsample_layers[i](x, training=training) x = self.stages[i](x, training=training) x = self.last(x, training=training) - return x, weights + return x class ConvNeXtReverseWithSkip(tf.keras.Model): diff --git a/cvae.py b/cvae.py index 74eb621..266f6f1 100644 --- a/cvae.py +++ b/cvae.py @@ -38,7 +38,7 @@ else: from losses import collision_ratio, kl_loss, normal_loss #, concept_loss # from losses import collision_penalty as collision_penalty from losses import collision_loss as collision_penalty -from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights +from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, UV2MeshLabel, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights from dataset import DatasetFactory, StaticDataset from custom_utils.smpl import SMPL from models.cvae_model import CVAENew as CVAE @@ -62,6 +62,7 @@ def init_wandb(): run.log_artifact(artifact) uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_adaptive=False, is_smpl_orig=config['is_cape']) +uv2mesh_label = UV2MeshLabel(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) def kl_loss(mean, logvar): @@ -71,10 +72,10 @@ def kl_loss(mean, logvar): return kl_loss -def blend_weights_reg(pred_w): +def blend_weights_reg(pred_w, gender=None): orig = smpl.weights l1 = tf.reduce_mean(tf.abs(pred_w - orig[None, 0, :, :])) - return l1 * 1e-3 + return l1 # 3d loss on posed !!!!!!!!!!!! @@ -91,6 +92,7 @@ def train_step(batch): pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True) mesh_pred = uv2mesh(pred) + mesh_weigths = uv2mesh_label(weights) if config['use_skinning']: shape = batch['pose_shape'][:, 72:] @@ -100,7 +102,12 @@ def train_step(batch): mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped) else: - mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights) + shape = batch['pose_shape'][:, 72:] + gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32) + shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]) + + mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] + mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths) # is the output already masked? @@ -112,9 +119,9 @@ def train_step(batch): n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) # conc_loss = concept_loss(pred, batch['uv_unposed']) kl = kl_loss(mean, logvar) - weights_l = blend_weights_reg(weights) + weights_l = blend_weights_reg(mesh_weigths) - reconstr_loss = l2_mesh_loss + weights_l + reconstr_loss = l2_mesh_loss + 1e-6 * weights_l if config['use_extra_losses']: if 'block_collision_loss' in config.keys() and config['block_collision_loss'] == True: reconstr_loss = reconstr_loss + (e_loss) * config['reg_loss_coeff'] @@ -150,6 +157,7 @@ def valid_step(batch): pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) mesh_pred = uv2mesh(pred) + mesh_weigths = uv2mesh_label(weights) if config['use_skinning']: shape = batch['pose_shape'][:, 72:] @@ -159,7 +167,12 @@ def valid_step(batch): mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped) else: - mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights) + shape = batch['pose_shape'][:, 72:] + gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32) + shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]) + + mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] + mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths) # is the output already masked? @@ -171,9 +184,9 @@ def valid_step(batch): n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) # conc_loss = concept_loss(pred, batch['uv_unposed']) kl = kl_loss(mean, logvar) - weights_l = blend_weights_reg(weights) + weights_l = blend_weights_reg(mesh_weigths) - reconstr_loss = l2_mesh_loss + weights_l + reconstr_loss = l2_mesh_loss + 1e-6 * weights_l if config['use_extra_losses']: reconstr_loss = reconstr_loss + (e_loss + c_loss) * config['reg_loss_coeff'] if config['use_norm_loss']: @@ -261,6 +274,7 @@ def plot_everything(batch: dict, type: str): pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False) mesh_pred = uv2mesh(pred).numpy() + mesh_weigths = uv2mesh_label(weights) orig_img = uv_orig uv_static_img = batch['uv_static'] @@ -285,7 +299,12 @@ def plot_everything(batch: dict, type: str): mesh_pred[i, mask[i], :] += shape_offsets.numpy()[i, mask[i], :] mesh_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred).numpy() else: - mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights).numpy() + shape = batch['pose_shape'][:, 72:] + gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32) + shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])]) + + mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis] + mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths).numpy() gt = [mesh_orig[i].numpy()[mask[i], :] for i in range(pred.shape[0])] garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask) @@ -305,9 +324,9 @@ def plot_everything(batch: dict, type: str): collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(garment_posed))] # figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], garment_posed[i], faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) } - figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) } - figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) } - figs_weights = { type + '/weights_' + str(i): plot_weights(weights[0,:,i], body[0], F) for i in range(24) } + figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) } + figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) } + figs_weights = { type + '/weights_' + str(i): plot_weights(mesh_weigths[0,:,i], body[0], F) for i in range(24) } # figs_weights = { type + '/weights_' + str(i): plot_weights(smpl.weights[0,:,i], body[0], F) for i in range(24) } mean_hist = wandb.Histogram(sequence=mean.numpy().mean(axis=0)) diff --git a/models/base.py b/models/base.py index e9cfd67..122c18f 100644 --- a/models/base.py +++ b/models/base.py @@ -171,13 +171,13 @@ class AEEncoderNew(tf.keras.Model): class DecoderNew(tf.keras.Model): - def __init__(self, depths, dims, kernel_size=7, is_label=False): + def __init__(self, depths, dims, kernel_size=7, is_label=False, learn_weights=False): super(DecoderNew, self).__init__() self.is_label = is_label if not is_label: - self.decoder = ConvNeXtReverse(depths=depths, dims=dims, kernel_size=kernel_size) + self.decoder = ConvNeXtReverse(depths=depths, dims=dims, kernel_size=kernel_size, learn_weights=learn_weights) else: self.decoder = ConvNeXtReverseLabels(depths=depths, dims=dims, nr_of_classes=7) diff --git a/models/cvae_model.py b/models/cvae_model.py index b4cf805..c708ef1 100644 --- a/models/cvae_model.py +++ b/models/cvae_model.py @@ -181,6 +181,7 @@ class CVAENew(tf.keras.Model): self.encoder = VariationalEncoderNew(config['cvae']['encoder_depths'], config['cvae']['encoder_dims'], kernel_size=config['cvae']['kernel_size']) self.decoder = DecoderNew(config['cvae']['decoder_depths'], config['cvae']['decoder_dims'], config['cvae']['kernel_size']) + self.decoder_weights = DecoderNew(config['cvae']['decoder_weights_depths'], config['cvae']['decoder_weights_dims'], config['cvae']['kernel_size'], learn_weights=True) self.reshape = tf.keras.layers.Reshape((8, 4, 64)) @@ -213,7 +214,8 @@ class CVAENew(tf.keras.Model): z = tf.concat([eps, z_cond], axis=-1) - x, weights = self.decoder(z, training=False) + x = self.decoder(z, training=False) + weights = self.decoder_weights(z, training=False) if self.avg_garment is not None: x = x + self.avg_garment[tf.newaxis,:,:,:] @@ -271,7 +273,8 @@ class CVAENew(tf.keras.Model): z = tf.concat([z, z_cond], axis=-1) - x, weights = self.decoder(z, training=training) + x = self.decoder(z, training=training) + weights = self.decoder_weights(z, training=training) if self.avg_garment is not None: x = x + self.avg_garment[tf.newaxis,:,:,:]