luni/cvpr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convolutional weights model

Browse Source
This commit is contained in:
luni
2023-02-17 13:11:35 +00:00
parent 61a609c8f7
commit 33eff16c8c
5 changed files with 50 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/config_abl_all_post.py
+4 -3
View File
@@ -20,8 +20,8 @@ ablation_count = [config['use_skinning'], config['remove_global_orientation'], c
assert ablation_count == 1 or ablation_count == 0
config['gpu'] = '5'
config['name'] = 'iccv_weights_test_3'
config['notes'] = ''
config['name'] = 'iccv_weights_test_5'
config['notes'] = 'smaller weight loss coeff, 1e-6, convolutional weight learning'
config['avg_garment_path'] = 'custom_data/train_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv train_no_shape_one_hot_avg_uv
if config['use_skinning']:
@@ -93,7 +93,8 @@ config['cvae']['encoder_depths'] = [3, 3, 3, 9, 3]
config['cvae']['encoder_dims'] = [32, 64, 128, 256, 64]
config['cvae']['decoder_depths'] = config['cvae']['encoder_depths']
config['cvae']['decoder_dims'] = config['cvae']['encoder_dims']
config['cvae']['decoder_weights_depths'] = [3, 3, 3, 5, 3]
config['cvae']['decoder_weights_dims'] = [24, 32, 64, 128, 256]
config['cond_ae_static'] = dict()
config['cond_ae_static']['is_label'] = False
convnext/models/convnext_tf.py
+6 -13
View File
@@ -222,7 +222,7 @@ class ConvNeXtReverse(tf.keras.Model):
def __init__(self,
depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], kernel_size=7,
drop_path_rate=0., layer_scale_init_value=1e-6, head_init_scale=1.,
drop_path_rate=0., layer_scale_init_value=1e-6, head_init_scale=1., learn_weights=False,
):
super().__init__()
self.upsample_layers = [] # stem and 3 intermediate downsampling conv layers
@@ -255,27 +255,20 @@ class ConvNeXtReverse(tf.keras.Model):
self.stages.append(stage)
cur += depths[i]
if not learn_weights:
self.last = Dense(3, activation='tanh')
else:
self.last = Activation('softmax')
# self.last = Dense(24, activation='softmax')
self.weights_model = tf.keras.Sequential([Block(dim=dims[-1]*2, kernel_size=kernel_size),
Block(dim=dims[-1]*2, kernel_size=kernel_size),
Block(dim=dims[-1]*2, kernel_size=kernel_size),
Flatten(),
Dense(128 * 24),
Reshape((24, 128)),
Dense(14475),
Lambda(lambda x: tf.transpose(x, (0, 2, 1))),
Lambda(lambda x: tf.nn.softmax(x, axis=-1))])
def call(self, x, training):
weights = self.weights_model(x, training=training)
for i in reversed(range(len(self.upsample_layers))):
x = self.upsample_layers[i](x, training=training)
x = self.stages[i](x, training=training)
x = self.last(x, training=training)
return x, weights
return x
class ConvNeXtReverseWithSkip(tf.keras.Model):
cvae.py
+32 -13
View File
@@ -38,7 +38,7 @@ else:
from losses import collision_ratio, kl_loss, normal_loss #, concept_loss
# from losses import collision_penalty as collision_penalty
from losses import collision_loss as collision_penalty
from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights
from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, UV2MeshLabel, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision, plot_weights
from dataset import DatasetFactory, StaticDataset
from custom_utils.smpl import SMPL
from models.cvae_model import CVAENew as CVAE
@@ -62,6 +62,7 @@ def init_wandb():
run.log_artifact(artifact)
uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_adaptive=False, is_smpl_orig=config['is_cape'])
uv2mesh_label = UV2MeshLabel(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
def kl_loss(mean, logvar):
@@ -71,10 +72,10 @@ def kl_loss(mean, logvar):
return kl_loss
def blend_weights_reg(pred_w):
def blend_weights_reg(pred_w, gender=None):
orig = smpl.weights
l1 = tf.reduce_mean(tf.abs(pred_w - orig[None, 0, :, :]))
return l1 * 1e-3
return l1
# 3d loss on posed !!!!!!!!!!!!
@@ -91,6 +92,7 @@ def train_step(batch):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True)
mesh_pred = uv2mesh(pred)
mesh_weigths = uv2mesh_label(weights)
if config['use_skinning']:
shape = batch['pose_shape'][:, 72:]
@@ -100,7 +102,12 @@ def train_step(batch):
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
else:
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights)
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths)
# is the output already masked?
@@ -112,9 +119,9 @@ def train_step(batch):
n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
# conc_loss = concept_loss(pred, batch['uv_unposed'])
kl = kl_loss(mean, logvar)
weights_l = blend_weights_reg(weights)
weights_l = blend_weights_reg(mesh_weigths)
reconstr_loss = l2_mesh_loss + weights_l
reconstr_loss = l2_mesh_loss + 1e-6 * weights_l
if config['use_extra_losses']:
if 'block_collision_loss' in config.keys() and config['block_collision_loss'] == True:
reconstr_loss = reconstr_loss + (e_loss) * config['reg_loss_coeff']
@@ -150,6 +157,7 @@ def valid_step(batch):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
mesh_pred = uv2mesh(pred)
mesh_weigths = uv2mesh_label(weights)
if config['use_skinning']:
shape = batch['pose_shape'][:, 72:]
@@ -159,7 +167,12 @@ def valid_step(batch):
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
else:
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights)
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths)
# is the output already masked?
@@ -171,9 +184,9 @@ def valid_step(batch):
n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
# conc_loss = concept_loss(pred, batch['uv_unposed'])
kl = kl_loss(mean, logvar)
weights_l = blend_weights_reg(weights)
weights_l = blend_weights_reg(mesh_weigths)
reconstr_loss = l2_mesh_loss + weights_l
reconstr_loss = l2_mesh_loss + 1e-6 * weights_l
if config['use_extra_losses']:
reconstr_loss = reconstr_loss + (e_loss + c_loss) * config['reg_loss_coeff']
if config['use_norm_loss']:
@@ -261,6 +274,7 @@ def plot_everything(batch: dict, type: str):
pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
mesh_pred = uv2mesh(pred).numpy()
mesh_weigths = uv2mesh_label(weights)
orig_img = uv_orig
uv_static_img = batch['uv_static']
@@ -285,7 +299,12 @@ def plot_everything(batch: dict, type: str):
mesh_pred[i, mask[i], :] += shape_offsets.numpy()[i, mask[i], :]
mesh_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred).numpy()
else:
mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred, weights).numpy()
shape = batch['pose_shape'][:, 72:]
gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
mesh_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths).numpy()
gt = [mesh_orig[i].numpy()[mask[i], :] for i in range(pred.shape[0])]
garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask)
@@ -305,9 +324,9 @@ def plot_everything(batch: dict, type: str):
collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(garment_posed))]
# figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], garment_posed[i], faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(not config['use_skinning'] and config['remove_global_orientation'])) for i in range(4) }
figs_weights = { type + '/weights_' + str(i): plot_weights(weights[0,:,i], body[0], F) for i in range(24) }
figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) }
figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i], flip_xy=(config['remove_global_orientation'])) for i in range(4) }
figs_weights = { type + '/weights_' + str(i): plot_weights(mesh_weigths[0,:,i], body[0], F) for i in range(24) }
# figs_weights = { type + '/weights_' + str(i): plot_weights(smpl.weights[0,:,i], body[0], F) for i in range(24) }
mean_hist = wandb.Histogram(sequence=mean.numpy().mean(axis=0))
models/base.py
+2 -2
View File
@@ -171,13 +171,13 @@ class AEEncoderNew(tf.keras.Model):
class DecoderNew(tf.keras.Model):
def __init__(self, depths, dims, kernel_size=7, is_label=False):
def __init__(self, depths, dims, kernel_size=7, is_label=False, learn_weights=False):
super(DecoderNew, self).__init__()
self.is_label = is_label
if not is_label:
self.decoder = ConvNeXtReverse(depths=depths, dims=dims, kernel_size=kernel_size)
self.decoder = ConvNeXtReverse(depths=depths, dims=dims, kernel_size=kernel_size, learn_weights=learn_weights)
else:
self.decoder = ConvNeXtReverseLabels(depths=depths, dims=dims, nr_of_classes=7)
models/cvae_model.py
+5 -2
View File
@@ -181,6 +181,7 @@ class CVAENew(tf.keras.Model):
self.encoder = VariationalEncoderNew(config['cvae']['encoder_depths'], config['cvae']['encoder_dims'], kernel_size=config['cvae']['kernel_size'])
self.decoder = DecoderNew(config['cvae']['decoder_depths'], config['cvae']['decoder_dims'], config['cvae']['kernel_size'])
self.decoder_weights = DecoderNew(config['cvae']['decoder_weights_depths'], config['cvae']['decoder_weights_dims'], config['cvae']['kernel_size'], learn_weights=True)
self.reshape = tf.keras.layers.Reshape((8, 4, 64))
@@ -213,7 +214,8 @@ class CVAENew(tf.keras.Model):
z = tf.concat([eps, z_cond], axis=-1)
x, weights = self.decoder(z, training=False)
x = self.decoder(z, training=False)
weights = self.decoder_weights(z, training=False)
if self.avg_garment is not None:
x = x + self.avg_garment[tf.newaxis,:,:,:]
@@ -271,7 +273,8 @@ class CVAENew(tf.keras.Model):
z = tf.concat([z, z_cond], axis=-1)
x, weights = self.decoder(z, training=training)
x = self.decoder(z, training=training)
weights = self.decoder_weights(z, training=training)
if self.avg_garment is not None:
x = x + self.avg_garment[tf.newaxis,:,:,:]