luni/cvpr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convolutional weights model

Browse Source
This commit is contained in:
luni
2023-02-17 13:11:35 +00:00
parent 61a609c8f7
commit 33eff16c8c
5 changed files with 50 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cvae.py
+32 -13
View File
@@ -38,7 +38,7 @@ else:
from losses import collision_ratio, kl_loss, normal_loss #, concept_loss
# from losses import collision_penalty as collision_penalty
from losses import collision_loss as collision_penalty
from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights
from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, UV2MeshLabel, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights
from dataset import DatasetFactory, StaticDataset
from custom_utils.smpl import SMPL
from models.cvae_model import CVAENew as CVAE
@@ -62,6 +62,7 @@ def init_wandb():
run.log_artifact(artifact)
uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_adaptive=False, is_smpl_orig=config['is_cape'])
uv2mesh_label = UV2MeshLabel(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
def kl_loss(mean, logvar):
@@ -71,10 +72,10 @@ def kl_loss(mean, logvar):
return kl_loss
def blend_weights_reg(pred_w):
def blend_weights_reg(pred_w, gender=None):
orig = smpl.weights
l1 = tf.reduce_mean(tf.abs(pred_w - orig[None, 0, :, :]))
return l1 * 1e-3
return l1
# 3d loss on posed !!!!!!!!!!!!
@@ -91,6 +92,7 @@ def train_step(batch):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True)
mesh_pred = uv2mesh(pred)
mesh_weigths = uv2mesh_label(weights)
if config['use_skinning']:
shape = batch['pose_shape'][:, 72:]
@@ -100,7 +102,12 @@ def train_step(batch):
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
else:
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights)
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths)
# is the output already masked?
@@ -112,9 +119,9 @@ def train_step(batch):
n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
# conc_loss = concept_loss(pred, batch['uv_unposed'])
kl = kl_loss(mean, logvar)
weights_l = blend_weights_reg(weights)
weights_l = blend_weights_reg(mesh_weigths)
reconstr_loss = l2_mesh_loss + weights_l
reconstr_loss = l2_mesh_loss + 1e-6 * weights_l
if config['use_extra_losses']:
if 'block_collision_loss' in config.keys() and config['block_collision_loss'] == True:
reconstr_loss = reconstr_loss + (e_loss) * config['reg_loss_coeff']
@@ -150,6 +157,7 @@ def valid_step(batch):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
mesh_pred = uv2mesh(pred)
mesh_weigths = uv2mesh_label(weights)
if config['use_skinning']:
shape = batch['pose_shape'][:, 72:]
@@ -159,7 +167,12 @@ def valid_step(batch):
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
else:
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights)
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths)
# is the output already masked?
@@ -171,9 +184,9 @@ def valid_step(batch):
n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
# conc_loss = concept_loss(pred, batch['uv_unposed'])
kl = kl_loss(mean, logvar)
weights_l = blend_weights_reg(weights)
weights_l = blend_weights_reg(mesh_weigths)
reconstr_loss = l2_mesh_loss + weights_l
reconstr_loss = l2_mesh_loss + 1e-6 * weights_l
if config['use_extra_losses']:
reconstr_loss = reconstr_loss + (e_loss + c_loss) * config['reg_loss_coeff']
if config['use_norm_loss']:
@@ -261,6 +274,7 @@ def plot_everything(batch: dict, type: str):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
mesh_pred = uv2mesh(pred).numpy()
mesh_weigths = uv2mesh_label(weights)
orig_img = uv_orig
uv_static_img = batch['uv_static']
@@ -285,7 +299,12 @@ def plot_everything(batch: dict, type: str):
mesh_pred[i, mask[i], :] += shape_offsets.numpy()[i, mask[i], :]
mesh_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred).numpy()
else:
mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights).numpy()
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths).numpy()
gt = [mesh_orig[i].numpy()[mask[i], :] for i in range(pred.shape[0])]
garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask)
@@ -305,9 +324,9 @@ def plot_everything(batch: dict, type: str):
collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(garment_posed))]
# figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], garment_posed[i], faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs_weights = { type + '/weights_' + str(i): plot_weights(weights[0,:,i], body[0], F) for i in range(24) }
figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) }
figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) }
figs_weights = { type + '/weights_' + str(i): plot_weights(mesh_weigths[0,:,i], body[0], F) for i in range(24) }
# figs_weights = { type + '/weights_' + str(i): plot_weights(smpl.weights[0,:,i], body[0], F) for i in range(24) }
mean_hist = wandb.Histogram(sequence=mean.numpy().mean(axis=0))