new weight trials, 10_3 didn't run

This reverts commit ac27319860.

aaa_helpers.txt

@@ -1,4 +1,6 @@
-nohup python cvae.py --name="lhunor_weights_test_6" > nohup/iccv/weights_test_6.out 2>&1 &
+nohup python cvae.py --name="lhunor_weights_test_10_2" > nohup/iccv/weights_test_10_2.out 2>&1 &
+nohup python evaluate_ae.py --name iccv_weights_test_6 > nohup/iccv/evaluate_iccv_weights_test_6.out 2>&1 &
+nohup python calculate_error.py --name iccv_weights_test_6 > nohup/iccv/error_iccv_weights_test_6.out 2>&1 &
 
 
 one final convolutional layer for smoothing, on the final coordinates

compare_models_saved.py

@@ -0,0 +1,188 @@
+import os
+import sys
+from configs.config_abl_all_post import config as config_a
+
+os.environ["CUDA_VISIBLE_DEVICES"] = '5'
+import tensorflow as tf
+
+gpus = tf.config.experimental.list_physical_devices('GPU')
+if gpus:
+    try:
+        # Currently, memory growth needs to be the same across GPUs
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True)
+        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
+        print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
+    except RuntimeError as e:
+        # Memory growth must be set before GPUs have been initialized
+        print(e)
+
+import glob
+import time
+import argparse
+import wandb
+import tqdm
+import random
+import numpy as np
+import plotly.graph_objects as go
+import plotly.express as ex
+from PIL import Image
+from postprocessing import fix_collision
+
+if config_a['is_skirt']:
+    from losses import edge_loss_skirt as edge_loss
+else:
+    from losses import edge_loss
+
+
+
+from losses import collision_ratio, kl_loss #, concept_loss
+# from losses import collision_penalty as collision_penalty
+from losses import collision_loss as collision_penalty
+from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision
+from dataset import DatasetFactory, StaticDataset
+from custom_utils.smpl import SMPL
+from models.cvae_model import CVAE
+from custom_utils.supersmpl_io import writeOBJ
+from plotly.subplots import make_subplots
+
+
+# uv2mesh = UV2Mesh(res=256, is_skirt=config_a['is_skirt'], is_adaptive=False)
+
+
+
+def plot(mesh_gt, pred_a, pred_b, body, mask):
+    fig = make_subplots(rows=1, cols=3, start_cell="top-left", specs = [ [{ 'type': 'scene' } for _ in range(3)] ])
+    
+    _, faces_garment = _maskFaces(np.zeros((14475, 3), np.float32), F, mask)
+    faces = np.concatenate((F, faces_garment + 14475), axis=0)
+    mesh_gt = np.concatenate((body, mesh_gt[mask, :]), axis=0)
+    pred_a = np.concatenate((body, pred_a), axis=0)
+    pred_b = np.concatenate((body, pred_b), axis=0)
+    colors = np.concatenate((np.ones_like(body, dtype=np.int32)*100, np.ones_like(mesh_gt, dtype=np.int32)*230), axis=0)
+    # colors = np.ones_like(mesh_gt, dtype=np.int32)*230
+
+    fig.add_trace(go.Mesh3d(
+            x=mesh_gt[:,0],
+            y=mesh_gt[:,2],
+            z=mesh_gt[:,1],
+            i = faces[:,0],
+            j = faces[:,1],
+            k = faces[:,2],
+            text="Ground Truth",
+            vertexcolor=colors,
+        ),
+        row=1, col=1)
+
+    fig.add_trace(go.Mesh3d(
+            x=pred_a[:,0],
+            y=pred_a[:,2],
+            z=pred_a[:,1],
+            i = faces[:,0],
+            j = faces[:,1],
+            k = faces[:,2],
+            text="Pred A",
+            vertexcolor=colors,
+        ),
+        row=1, col=2)
+
+    fig.add_trace(go.Mesh3d(
+            x=pred_b[:,0],
+            y=pred_b[:,2],
+            z=pred_b[:,1],
+            i = faces[:,0],
+            j = faces[:,1],
+            k = faces[:,2],
+            text="Pred B",
+            vertexcolor=colors,
+        ),
+        row=1, col=3)
+
+
+
+    fig.update_layout(scene_aspectmode='manual', scene_aspectratio=dict(x=1, y=1, z=1), 
+                    scene2_aspectmode='manual', scene2_aspectratio=dict(x=1, y=1, z=1), 
+                    scene3_aspectmode='manual', scene3_aspectratio=dict(x=1, y=1, z=1), 
+                    )
+    helper = dict(xaxis = dict(nticks=4, range=[-1,1],),
+                yaxis = dict(nticks=4, range=[-1,1],),
+                zaxis = dict(nticks=4, range=[-1,1],),)
+    fig.update_layout(scene = helper,
+                        scene2 = helper,
+                        scene3 = helper,
+                        )
+
+    return fig
+
+
+
+if __name__ == '__main__':
+    run = wandb.init(project='plots', name=config_a['name'], notes=config_a['notes'], dir='/wandb')
+
+    parser = argparse.ArgumentParser(description='')
+    parser.add_argument('--name', type=str)
+    args = vars(parser.parse_args())
+
+    skirt_modifier = '_skirt' if config_a['is_skirt'] else ''
+
+    folder = config_a['test_dir']
+    # smpl = SMPL(use_one_hot=False)
+
+    F = np.load("custom_utils/smpl_faces" + skirt_modifier + ".npy").astype(np.int32)
+
+    iteration = ''
+
+    # load_model(model, 'checkpoints/' + 'abl_all_120k')
+
+
+    factory = DatasetFactory()
+    dataset_a = factory.get_dataset(folder=config_a['test_dir'], 
+                                       batch_size=config_a['batch_size'], 
+                                       signature=config_a['signature_dynamic'], 
+                                       resolutions=[config_a['resolution']], 
+                                       in_memory=False, 
+                                       do_repeat=False,
+                                       use_one_hot=config_a['use_one_hot'],
+                                       use_no_orient=config_a['remove_global_orientation'],
+                                       shuffle=False,
+                                       testing=True,
+                                    #    frames_file='frames_abl.txt',
+                                       )
+    static_dataset_a = StaticDataset(folder=config_a['test_dir'], 
+                                        signature=config_a['signature_static'], 
+                                        resolutions=[config_a['resolution']], 
+                                        in_memory=True)
+
+    iterator_a = iter(dataset_a)
+    
+
+    faces_folder = '/home/workspace/data/results/support/faces' + ('_skirt' if config_a['is_skirt'] else '')
+    ids = os.listdir(faces_folder)
+    ids = [id.split('.')[0] for id in ids]
+    faces = {id : np.load(faces_folder + '/' + id + '.npy') for id in ids }
+
+    results_folder = '/home/workspace/data/results/post'
+
+    count = 0
+    while True:
+        batch = iterator_a.get_next()
+
+        batch = static_dataset_a.expand_batch(batch_to_dict(batch, config_a['signature_dynamic']))
+
+        mask = batch['mesh_mask'].numpy().astype(bool)
+        gt = batch['mesh_posed'].numpy()
+        body = batch['mesh_body_posed'].numpy()
+
+        for i in range(mask.shape[0]):
+            id, frame_nr = batch['id'][i].numpy().decode("utf-8"), batch['frame_nr'][i].numpy().decode("utf-8")
+            pred_a = np.load(results_folder + '/al_all_post-/' + id + '_' + frame_nr + '.npy')
+            pred_b = np.load(results_folder + '/iccv_weights_test_6-/' + id + '_' + frame_nr + '.npy')
+            # pred_c = np.load(results_folder + '/iccv_weights_test_6-/' + id + '_' + frame_nr + '.npy')
+
+            fig = plot(gt[i], pred_a, pred_b, body[i], mask[i])
+
+            wandb.log({f'{id}_{frame_nr}': fig}, step = count)
+            count += 1
+
+            print(count)
+

configs/config_abl_all_post.py

@@ -19,13 +19,13 @@ config['only_curvatures'] =           False
 ablation_count = [config['use_skinning'], config['remove_global_orientation'], config['use_curvature'], config['use_extra_losses'], config['use_offsets']].count(False)
 assert ablation_count == 1 or ablation_count == 0
 
-config['gpu'] = '5'
-config['name'] = 'iccv_weights_test_6'
-config['notes'] = 'smaller weight loss coeff, 1e-6, convolutional weight learning, continued from _5'
+config['gpu'] = '6'
+config['name'] = 'iccv_weights_test_10_3'
+config['notes'] = 'smaller weight loss coeff, 1e-6, convolutional weight learning, add gnn but smaller, same as 10 but training from zero and no avg garment'
 
-config['avg_garment_path'] = 'custom_data/train_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv  train_no_shape_one_hot_avg_uv
-if config['use_skinning']:
-  assert (config['use_offsets'] and config['avg_garment_path'] != '') or (not config['use_offsets'] and config['avg_garment_path'] == '')
+config['avg_garment_path'] = ''# 'custom_data/train_no_shape_avg_uv.npy' #custom_data/train_no_shape_avg_uv.npy # train_no_shape_avg_uv  train_no_shape_one_hot_avg_uv
+# if config['use_skinning']:
+#   assert (config['use_offsets'] and config['avg_garment_path'] != '') or (not config['use_offsets'] and config['avg_garment_path'] == '')
 
 config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
 
@@ -53,7 +53,7 @@ if config['is_cape']:
   config['smpl_res'] = 3638
 
 
-config['pretrained_model'] = 'checkpoints/post/iccv_weights_test_5'
+config['pretrained_model'] = ''
 
 ###############    
 config['lr'] = 0.0001#                 0.0001

convnext/models/convnext_tf.py

@@ -259,6 +259,7 @@ class ConvNeXtReverse(tf.keras.Model):
             self.last = Dense(3, activation='tanh')
         else:
             self.last = Activation('softmax')
+            # self.last = lambda x, training: x
             # self.last = Dense(24, activation='softmax')
 
 

cvae.py

@@ -42,6 +42,7 @@ from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mes
 from dataset import DatasetFactory, StaticDataset
 from custom_utils.smpl import SMPL
 from models.cvae_model import CVAENew as CVAE
+from models.gnn import Smoother
 
 
 def init_wandb():
@@ -92,7 +93,9 @@ def train_step(batch):
         pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=True)
 
         mesh_pred = uv2mesh(pred)
+        mesh_pred = smoother(mesh_pred, training=True)
         mesh_weigths = uv2mesh_label(weights)
+        mesh_weigths = smoother_weights(mesh_weigths, training=True)
 
         if config['use_skinning']:
             shape = batch['pose_shape'][:, 72:]
@@ -137,8 +140,8 @@ def train_step(batch):
 
     # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
     metric = tf.reduce_sum(tf.math.sqrt(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (mesh_orig - mesh_pred_posed) ** 2, axis=-1))) / tf.reduce_sum(batch['mesh_mask'])
-    gradients = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
+    gradients = tape.gradient(loss, model.trainable_variables + smoother.trainable_variables + smoother_weights.trainable_variables)
+    optimizer.apply_gradients(zip(gradients, model.trainable_variables + smoother.trainable_variables + smoother_weights.trainable_variables))
     return { 'train/loss': loss, 'train/edge_loss': e_loss, 'train/collision_loss': c_loss, 'train/euclid_error': metric,
              'train/reconstr_loss': reconstr_loss, 'train/collision_ratio': c_ratio,
              'train/mean': mean, 'train/logvar': logvar, 'train/kl_loss': kl, 'train/var': np.e ** logvar,
@@ -157,7 +160,9 @@ def valid_step(batch):
     pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
 
     mesh_pred = uv2mesh(pred)
+    mesh_pred = smoother(mesh_pred, training=False)
     mesh_weigths = uv2mesh_label(weights)
+    mesh_weigths = smoother_weights(mesh_weigths, training=False)
 
     if config['use_skinning']:
         shape = batch['pose_shape'][:, 72:]
@@ -273,8 +278,10 @@ def plot_everything(batch: dict, type: str):
 
     pred, mean, logvar, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
     
-    mesh_pred = uv2mesh(pred).numpy()
+    mesh_pred = uv2mesh(pred)
+    mesh_pred = smoother(mesh_pred, training=False).numpy()
     mesh_weigths = uv2mesh_label(weights)
+    mesh_weigths = smoother_weights(mesh_weigths, training=False)
     
     orig_img = uv_orig
     uv_static_img = batch['uv_static']
@@ -282,7 +289,10 @@ def plot_everything(batch: dict, type: str):
     pred_img = pred
     img = tf.concat((uv_static_img, uv_body_posed_img, orig_img, pred_img), axis=2)
     imgs = [wandb.Image((img[i] + 1) / 2.0) for i in range(4)]
-    
+    # min max normalization of weights
+    weights = (weights - tf.reduce_min(weights, axis=[1,2,3], keepdims=True)) / (tf.reduce_max(weights, axis=[1,2,3], keepdims=True) - tf.reduce_min(weights, axis=[1,2,3], keepdims=True))
+    weights_imgs = [wandb.Image(np.concatenate((np.concatenate([weights[b,:,:,i].numpy() for i in range(12)], axis=-1), np.concatenate([weights[b,:,:,i].numpy() for i in range(12,24)], axis=-1)), axis=0)) for b in range(4)]
+
     F = np.load("custom_utils/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy")
     mask = batch['mesh_mask'].numpy().astype(bool)
 
@@ -332,7 +342,7 @@ def plot_everything(batch: dict, type: str):
     mean_hist = wandb.Histogram(sequence=mean.numpy().mean(axis=0))
     logvar_hist = wandb.Histogram(sequence=logvar.numpy().mean(axis=0))
 
-    return { **figs, **figs_coll, **figs_weights, **{ type + '/images': imgs, type + '/mean_hist': mean_hist, type + '/logvar_hist': logvar_hist } }
+    return { **figs, **figs_coll, **figs_weights, **{ type + '/images': imgs, type + '/weight_images': weights_imgs, type + '/mean_hist': mean_hist, type + '/logvar_hist': logvar_hist } }
 
 
 def sample(folder): 
@@ -547,6 +557,8 @@ if __name__ == '__main__':
                                         is_cape=config['is_cape'])
 
     model = CVAE(config)
+    smoother = Smoother(features=24, activation_middle='tanh', activation_final='tanh', learn_weights=False)
+    smoother_weights = Smoother(features=3, activation_middle=None, activation_final='softmax', learn_weights=True)
 
     optimizer = tf.optimizers.Adam(learning_rate=config['lr'])
     balancing_activated_step = 0
@@ -571,9 +583,10 @@ if __name__ == '__main__':
 
         train_loss = train_step(batch)
 
-        data_valid = valid_iterator.get_next()
-        data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
-        valid_loss = valid_step(data_valid)
+        if step % 25 == 0:
+            data_valid = valid_iterator.get_next()
+            data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
+            valid_loss = valid_step(data_valid)
 
         to_log = { **train_loss, **valid_loss, **{ 'current_step': step} }
         if step % 5000 == 0 or (step < 5000 and step % 500 == 0):

evaluate_ae.py

@@ -39,7 +39,7 @@ else:
 from losses import collision_ratio, kl_loss #, concept_loss
 # from losses import collision_penalty as collision_penalty
 from losses import collision_loss as collision_penalty
-from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision
+from custom_utils.custom_utils import batch_to_dict, load_model, plot_3d, UV2Mesh, UV2MeshLabel, get_summary_html, maskFacesBatch, _maskFaces, plot_3d_collision
 from dataset import DatasetFactory, StaticDataset
 from custom_utils.smpl import SMPL
 from models.cvae_model import CVAENew as CVAE
@@ -48,6 +48,7 @@ from custom_utils.supersmpl_io import writeOBJ
 
 
 uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_adaptive=False, is_smpl_orig=config['is_cape'])
+uv2mesh_label = UV2MeshLabel(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
 
 @tf.function
 def step(batch):
@@ -56,9 +57,10 @@ def step(batch):
     else:
         uv_orig =  batch['uv_posed']
     
-    pred, _, _ = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
+    pred, _, _, weights = model([uv_orig, batch['uv_static'], batch['uv_body_posed'], batch['uv_curvature'], batch['uv_mask']], training=False)
 
     mesh_pred = uv2mesh(pred)
+    mesh_weigths = uv2mesh_label(weights)
 
     if config['use_skinning']:
         shape = batch['pose_shape'][:, 72:]
@@ -68,7 +70,13 @@ def step(batch):
         mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
         mesh_pred_posed = smpl.run_garment(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)
     else:
-        mesh_pred_posed = mesh_pred
+        shape = batch['pose_shape'][:, 72:]
+        gender = tf.cast(tf.greater_equal(shape[:, 10] , 0.5), tf.int32)
+        shape_offsets = tf.stack([tf.gather(tf.tensordot(shape[i,:10][tf.newaxis, :], smpl.shapedirs[gender[i]], axes=[[1], [2]])[0,:,:], smpl.to_keep, axis=0) for i in range(shape.shape[0])])
+
+        mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,tf.newaxis]
+        mesh_pred_posed = smpl.run_garment_with_weights(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped, mesh_weigths)
+
         
     # mesh_pred_posed_fixed = fix_collision(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
 

models/gnn.py

@@ -0,0 +1,34 @@
+import os
+import sys
+
+import scipy.io as sio
+import tensorflow as tf
+from spektral.layers import GCNConv, GeneralConv
+from spektral.utils import sp_matrix_to_sp_tensor
+from tensorflow.keras.layers import (Conv2D, Conv2DTranspose, Dense, DepthwiseConv2D,
+                                     GlobalAveragePooling2D, Layer,
+                                     LayerNormalization, Flatten, Reshape, UpSampling2D)
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+class Smoother(tf.keras.Model):
+    def __init__(self, features=64, activation_middle='tanh', activation_final='tanh', learn_weights=False):
+        super().__init__()
+        self.a = sp_matrix_to_sp_tensor(sio.loadmat("custom_utils/ssmpl0.mat")['L'].tocoo())
+        self.conv1 = GCNConv(features, activation=activation_middle, use_bias=False)
+        self.conv2 = GCNConv(features, activation=activation_middle, use_bias=False)
+        # self.conv3 = GCNConv(features, activation=activation_middle, use_bias=False)
+        # self.conv4 = GCNConv(64, activation='tanh', use_bias=False)
+        # self.conv5 = GCNConv(64, activation='tanh', use_bias=False)
+        self.conv6 = GCNConv(3 if not learn_weights else 24, activation=activation_final, use_bias=False)
+
+    def call(self, inputs, training=None):
+        x = inputs
+        x = self.conv1([x, self.a], training=training)
+        x = self.conv2([x, self.a], training=training)
+        # x = self.conv3([x, self.a], training=training)
+        # x = self.conv4([x, self.a], training=training)
+        # x = self.conv5([x, self.a], training=training)
+        x = self.conv6([x, self.a], training=training)
+        return x