luni/cvpr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit all

Browse Source
This commit is contained in:
luni
2023-01-25 12:25:59 +00:00
parent 1bf224841f
commit 7470ca9fb6
139 changed files with 82069 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files
auto_encoder_curv.py
+60 -20
View File
@@ -1,5 +1,6 @@
import os
from configs.config_ae_curv import config
import sys
from configs.config_ae_curv_post import config
os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
import tensorflow as tf
@@ -20,6 +21,7 @@ import argparse
import wandb
import random
import numpy as np
import cmapy as cm
from custom_utils.custom_utils import UV2Mesh, UV2MeshLabel, get_summary_html, maskFacesBatch
@@ -29,7 +31,7 @@ import plotly.express as ex
from tf_utils import check_tf_version, allow_memory_growth
from dataset import DatasetFactory, StaticDataset
from custom_utils.custom_utils import batch_to_dict
from models.cond_encoder import AutoEncoder
from models.cond_encoder import AutoEncoderNew as AutoEncoder
testing = False
@@ -39,12 +41,13 @@ def init_wandb():
artifact = wandb.Artifact('code', type='code')
artifact.add_file('auto_encoder_curv.py')
artifact.add_file('losses.py')
artifact.add_file('configs/config_base.py')
artifact.add_file('configs/config_ae_curv.py')
artifact.add_file('models/base.py')
artifact.add_file('models/cond_encoder.py')
artifact.add_file('dataset.py')
artifact.add_file('convnext/models/convnext_tf.py')
config_file_names = [key for key in sys.modules.keys() if key.startswith('configs.')]
for config_file_name in config_file_names:
artifact.add_file('/'.join(config_file_name.split('.')) + '.py')
run.log_artifact(artifact)
uv2mesh = UV2MeshLabel(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
@@ -99,16 +102,20 @@ def train_step(batch):
ce_uv_loss = scce(tf.math.argmax(batch['uv_curvature'], axis=-1), pred)
ms_ssim = tf.reduce_mean(tf.image.ssim(pred, batch['uv_curvature'], max_val=1))
loss = ce_uv_loss
pred_mesh = tf.math.argmax(uv2mesh(pred), axis=-1)
mesh_pred = uv2mesh(pred) * batch['mesh_mask'][:,:,tf.newaxis]
mesh_orig = tf.cast(uv2mesh(tf.math.argmax(batch['uv_curvature'], axis=-1, output_type=tf.int32)), tf.float32) * batch['mesh_mask']
ce_mesh_loss = scce(mesh_orig, mesh_pred)
loss = ce_uv_loss + ce_mesh_loss
mean_iou = get_mean_iou(tf.math.argmax(batch['uv_curvature'], axis=-1), tf.math.argmax(pred, axis=-1))
mean_iou_mesh = get_mean_iou(tf.cast(batch['mesh_curvature'], tf.float32) * batch['mesh_mask'], tf.cast(pred_mesh, tf.float32) * batch['mesh_mask'])
dice = dice_dense(pred, batch['uv_curvature'])
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return { 'train/loss': loss, 'train/ce_uv_loss': ce_uv_loss, 'train/ssim': ms_ssim, 'train/mean_iou': mean_iou, 'train/mean_iou_mesh': mean_iou_mesh, 'train/dice': dice }
return { 'train/loss': loss, 'train/ce_uv_loss': ce_uv_loss, 'train/ssim': ms_ssim, 'train/mean_iou': mean_iou,
'train/dice': dice, 'train/ce_mesh_loss': ce_mesh_loss }
@@ -118,14 +125,17 @@ def valid_step(batch):
ce_uv_loss = scce(tf.math.argmax(batch['uv_curvature'], axis=-1), pred)
ms_ssim = tf.reduce_mean(tf.image.ssim(pred, batch['uv_curvature'], max_val=1))
loss = ce_uv_loss
pred_mesh = tf.math.argmax(uv2mesh(pred), axis=-1)
mesh_pred = uv2mesh(pred) * batch['mesh_mask'][:,:,tf.newaxis]
mesh_orig = tf.cast(uv2mesh(tf.math.argmax(batch['uv_curvature'], axis=-1, output_type=tf.int32)), tf.float32) * batch['mesh_mask']
ce_mesh_loss = scce(mesh_orig, mesh_pred)
loss = ce_uv_loss + ce_mesh_loss
mean_iou = get_mean_iou(tf.math.argmax(batch['uv_curvature'], axis=-1), tf.math.argmax(pred, axis=-1))
mean_iou_mesh = get_mean_iou(tf.cast(batch['mesh_curvature'], tf.float32) * batch['mesh_mask'], tf.cast(pred_mesh, tf.float32) * batch['mesh_mask'])
dice = dice_dense(pred, batch['uv_curvature'])
return { 'valid/loss': loss, 'valid/ce_uv_loss': ce_uv_loss, 'valid/ssim': ms_ssim, 'valid/mean_iou': mean_iou, 'valid/mean_iou_mesh': mean_iou_mesh, 'valid/dice': dice }
return { 'valid/loss': loss, 'valid/ce_uv_loss': ce_uv_loss, 'valid/ssim': ms_ssim, 'valid/mean_iou': mean_iou,
'valid/dice': dice, 'valid/ce_mesh_loss': ce_mesh_loss }
def plot_everything(batch: dict, type: str):
@@ -134,7 +144,14 @@ def plot_everything(batch: dict, type: str):
orig_img = tf.math.argmax(batch['uv_curvature'], axis=-1, output_type=tf.dtypes.int32)
pred_img = tf.math.argmax(pred, axis=-1, output_type=tf.dtypes.int32)
img = tf.concat((orig_img, pred_img), axis=2).numpy()
imgs = { type + '/images': [wandb.Image(img[i,:,:,np.newaxis] / 6.0) for i in range(img.shape[0])] }
colors = np.array([cm.color('jet', 1 - i) for i in np.linspace(0, 1, 6+1)])
color_img = np.zeros((img.shape[0], img.shape[1], img.shape[2], 3), np.uint8)
for i in range(7):
color_img[img == i, :] = colors[i]
color_img = [wandb.Image(color_img[i]) for i in range(4)]
imgs = { type + '/images': [wandb.Image(img[i,:,:,np.newaxis] / 6.0) for i in range(img.shape[0])], type + '/color_images': color_img }
return imgs
@@ -159,7 +176,7 @@ def train():
to_log = { **to_log, **train_figs, **valid_figs }
if step == 0:
to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
wandb.log(to_log, step=step)
print(step)
@@ -202,18 +219,41 @@ if __name__ == '__main__':
factory = DatasetFactory()
dataset_train = factory.get_dataset(folder=config['train_dir'], batch_size=config['batch_size'], signature=config['signature_dynamic'], use_filled=config['use_filled'], resolutions=[config['resolution']], in_memory=config['dataset_in_memory'], is_cape=config['is_cape'])
static_dataset_train = StaticDataset(folder=config['train_dir'], signature=config['signature_static'], use_filled=config['use_filled'], resolutions=[config['resolution']], in_memory=True, is_cape=config['is_cape'])
dataset_valid = factory.get_dataset(folder=config['valid_dir'], batch_size=config['batch_size'], signature=config['signature_dynamic'], use_filled=config['use_filled'], resolutions=[config['resolution']], in_memory=config['dataset_in_memory'], is_cape=config['is_cape'])
static_dataset_valid = StaticDataset(folder=config['valid_dir'], signature=config['signature_static'], use_filled=config['use_filled'], resolutions=[config['resolution']], in_memory=True, is_cape=config['is_cape'])
dataset_train = factory.get_dataset(folder=config['train_dir'],
batch_size=config['batch_size'],
signature=config['signature_dynamic'],
resolutions=[config['resolution']],
in_memory=config['dataset_in_memory'],
use_no_orient=config['remove_global_orientation'],
frames_file=config['frames_file'] if 'frames_file' in config.keys() else None,
is_cape=config['is_cape'])
static_dataset_train = StaticDataset(folder=config['train_dir'],
signature=config['signature_static'],
resolutions=[config['resolution']],
in_memory=True,
is_cape=config['is_cape'])
dataset_valid = factory.get_dataset(folder=config['valid_dir'],
batch_size=config['batch_size'],
signature=config['signature_dynamic'],
resolutions=[config['resolution']],
in_memory=config['dataset_in_memory'],
use_no_orient=config['remove_global_orientation'],
frames_file=config['frames_file_valid'] if 'frames_file_valid' in config.keys() else None,
is_cape=config['is_cape'])
static_dataset_valid = StaticDataset(folder=config['valid_dir'],
signature=config['signature_static'],
resolutions=[config['resolution']],
in_memory=True,
is_cape=config['is_cape'])
model = AutoEncoder(config['cond_ae'])
optimizer = tf.optimizers.Adam(learning_rate=config['lr'])
ckpt_enc = tf.train.Checkpoint(model=model.encoder)
manager_enc = tf.train.CheckpointManager(ckpt_enc, 'checkpoints/' + config['name'] + '_enc', max_to_keep=20)
manager_enc = tf.train.CheckpointManager(ckpt_enc, 'checkpoints/post/' + config['name'] + '_enc', max_to_keep=20)
ckpt_dec = tf.train.Checkpoint(model=model.decoder)
manager_dec = tf.train.CheckpointManager(ckpt_dec, 'checkpoints/' + config['name'] + '_dec', max_to_keep=20)
manager_dec = tf.train.CheckpointManager(ckpt_dec, 'checkpoints/post/' + config['name'] + '_dec', max_to_keep=20)
if testing:
test()