train with fixed reg losses

configs/ae_norm_1.py

@@ -0,0 +1,73 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '0'
+config['name'] = 'norm_torch_2'
+config['notes'] = ' ' 
+config['tags'] = ['torch', 'norm'] 
+config['script'] = 'scripts/ae_norm_train.py'
+
+config['pretrained_model'] = ''
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-local/127/train'
+config['valid_dir'] = '/data-local/127/valid'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = None
+config['valid_frames'] = None
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = False
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['cond_ae'] = dict()
+config['cond_ae']['device'] = config['device']
+config['cond_ae']['is_label'] = False
+config['cond_ae']['latent_dim'] = config['latent_dim']
+config['cond_ae']['kernel_size'] = 7
+config['cond_ae']['bottleneck_size'] = 40 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 64, 3),
+    CNBlockConfig(64, 128, 9),
+    CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9),
+    CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3),
+    CNBlockConfig(128, 64, 9),
+    CNBlockConfig(64, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['cond_ae']['encoder_block_settings'] = block_setting
+config['cond_ae']['decoder_block_settings'] = block_setting_reverse
+config['cond_ae']['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['cond_ae']['upscale_output'] = False
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'uv_mask',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_normals',
+                        }

configs/direct_3.py

@@ -0,0 +1,82 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '0'
+config['name'] = 'direct_3'
+config['notes'] = 'with normals input' 
+config['tags'] = ['torch', 'direct'] 
+config['script'] = 'scripts/direct_train.py'
+
+config['pretrained_model'] = ''
+config['use_curvature'] = False
+config['use_normals'] = True
+
+# assert that only use curvature or normals
+assert not (config['use_curvature'] and config['use_normals'])
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-local/127/train'
+config['valid_dir'] = '/data-local/127/valid'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = None
+config['valid_frames'] = None
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = False
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+config['input_features'] = 9
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['is_label'] = False
+config['kernel_size'] = 7
+config['bottleneck_size'] = 64 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 128, 3),
+    CNBlockConfig(128, 256, 9),
+    CNBlockConfig(256, config['bottleneck_size'], 9),
+    CNBlockConfig(config['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['bottleneck_size'], 256, 3),
+    CNBlockConfig(256, 128, 9),
+    CNBlockConfig(128, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['encoder_block_settings'] = block_setting
+config['decoder_block_settings'] = block_setting_reverse
+config['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['reg_loss_coeff'] = 0.1
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'pose_shape',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_body_posed',
+                        'mesh_posed',
+                        'uv_unposed',
+                        'mesh_unposed',
+                        'uv_static',
+                        'uv_mask',
+                        'uv_body_posed',
+                        }

configs/direct_4.py

@@ -0,0 +1,82 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '5'
+config['name'] = 'direct_5'
+config['notes'] = 'with normals input but only single garment sequence' 
+config['tags'] = ['torch', 'direct'] 
+config['script'] = 'scripts/direct_train.py'
+
+config['pretrained_model'] = ''
+config['use_curvature'] = False
+config['use_normals'] = True
+
+# assert that only use curvature or normals
+assert not (config['use_curvature'] and config['use_normals'])
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-local/127/train'
+config['valid_dir'] = '/data-local/127/train'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = 'frames_00761.txt'
+config['valid_frames'] = 'frames_00761.txt'
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = True
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+config['input_features'] = 9
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['is_label'] = False
+config['kernel_size'] = 7
+config['bottleneck_size'] = 64 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 128, 3),
+    CNBlockConfig(128, 256, 9),
+    CNBlockConfig(256, config['bottleneck_size'], 9),
+    CNBlockConfig(config['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['bottleneck_size'], 256, 3),
+    CNBlockConfig(256, 128, 9),
+    CNBlockConfig(128, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['encoder_block_settings'] = block_setting
+config['decoder_block_settings'] = block_setting_reverse
+config['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['reg_loss_coeff'] = 0.1
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'pose_shape',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_body_posed',
+                        'mesh_posed',
+                        'uv_unposed',
+                        'mesh_unposed',
+                        'uv_static',
+                        'uv_mask',
+                        'uv_body_posed',
+                        }

configs/direct_6.py

@@ -0,0 +1,83 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '4'
+config['name'] = 'direct_6'
+config['notes'] = 'with normals input but only single garment sequence with msssim and l2 mesh loss only' 
+config['tags'] = ['torch', 'direct'] 
+config['script'] = 'scripts/direct_train.py'
+
+config['pretrained_model'] = ''
+config['use_curvature'] = False
+config['use_normals'] = True
+config['use_skinning'] = True
+
+# assert that only use curvature or normals
+assert not (config['use_curvature'] and config['use_normals'])
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-local/127/train'
+config['valid_dir'] = '/data-local/127/train'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = 'frames_00761.txt'
+config['valid_frames'] = 'frames_00761.txt'
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = True
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+config['input_features'] = 9
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['is_label'] = False
+config['kernel_size'] = 7
+config['bottleneck_size'] = 64 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 128, 3),
+    CNBlockConfig(128, 256, 9),
+    CNBlockConfig(256, config['bottleneck_size'], 9),
+    CNBlockConfig(config['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['bottleneck_size'], 256, 3),
+    CNBlockConfig(256, 128, 9),
+    CNBlockConfig(128, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['encoder_block_settings'] = block_setting
+config['decoder_block_settings'] = block_setting_reverse
+config['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['reg_loss_coeff'] = 0.1
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'pose_shape',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_body_posed',
+                        'mesh_posed',
+                        'uv_unposed',
+                        'mesh_unposed',
+                        'uv_static',
+                        'uv_mask',
+                        'uv_body_posed',
+                        }

configs/direct_7.py

@@ -0,0 +1,87 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '0'
+config['name'] = 'direct_9'
+config['notes'] = 'with normals input whole dataset with msssim and l2 mesh loss and reg losses and without skinning' 
+config['tags'] = ['torch', 'direct'] 
+config['script'] = 'scripts/direct_train.py'
+
+config['pretrained_model'] = ''
+config['use_curvature'] = False
+config['use_normals'] = True
+config['use_skinning'] = False
+
+# assert that only use curvature or normals
+assert not (config['use_curvature'] and config['use_normals'])
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-local/127/train'
+config['valid_dir'] = '/data-local/127/train'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = None
+config['valid_frames'] = None
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = False
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+config['input_features'] = 9
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['is_label'] = False
+config['kernel_size'] = 7
+config['bottleneck_size'] = 64 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 128, 3),
+    CNBlockConfig(128, 256, 9),
+    CNBlockConfig(256, config['bottleneck_size'], 9),
+    CNBlockConfig(config['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['bottleneck_size'], 256, 3),
+    CNBlockConfig(256, 128, 9),
+    CNBlockConfig(128, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['encoder_block_settings'] = block_setting
+config['decoder_block_settings'] = block_setting_reverse
+config['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['reg_loss_coeff'] = 0.1
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'pose_shape',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_body_posed',
+                        'mesh_posed',
+                        'uv_static',
+                        'uv_mask',
+                        'uv_body_posed',
+                        }
+
+if config['use_skinning']:
+  config['signature'].add('uv_unposed')
+  config['signature'].add('mesh_unposed')
+else:
+  config['signature'].add('uv_posed')

configs/direct_8.py

@@ -0,0 +1,87 @@
+from utils.convnext_utils import CNBlockConfig
+
+config = dict()
+
+config['gpu'] = '6'
+config['name'] = 'direct_8'
+config['notes'] = 'with normals input but only single garment sequence with msssim and l2 mesh loss only and without skinning and remasked' 
+config['tags'] = ['torch', 'direct'] 
+config['script'] = 'scripts/direct_train.py'
+
+config['pretrained_model'] = ''
+config['use_curvature'] = False
+config['use_normals'] = True
+config['use_skinning'] = False
+
+# assert that only use curvature or normals
+assert not (config['use_curvature'] and config['use_normals'])
+
+config['batch_size'] = 4
+config['train_dir'] = '/data-net/dataset/00761'
+config['valid_dir'] = '/data-net/dataset/00761'
+config['test_dir'] = '/data-net/dataset/test_ready'
+config['train_frames'] = None
+config['valid_frames'] = None
+
+config['resolution'] = 512
+config['is_skirt'] = False
+config['dataset_in_memory'] = True
+config['is_cape'] = False
+config['remove_global_orientation'] = True
+config['device'] = 'cuda'
+config['input_features'] = 9
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['latent_dim'] = 128
+
+config['bottleneck_scale'] = 2
+
+config['is_label'] = False
+config['kernel_size'] = 7
+config['bottleneck_size'] = 64 * config['bottleneck_scale']
+
+block_setting = [
+    CNBlockConfig(16, 32, 3),
+    CNBlockConfig(32, 128, 3),
+    CNBlockConfig(128, 256, 9),
+    CNBlockConfig(256, config['bottleneck_size'], 9),
+    CNBlockConfig(config['bottleneck_size'], None, 3),
+]
+
+block_setting_reverse = [
+    CNBlockConfig(config['bottleneck_size'], 256, 3),
+    CNBlockConfig(256, 128, 9),
+    CNBlockConfig(128, 32, 9),
+    CNBlockConfig(32, 16, 3),
+    CNBlockConfig(16, 3, 3),
+    # CNBlockConfig(3, 3, 3),
+]
+
+config['encoder_block_settings'] = block_setting
+config['decoder_block_settings'] = block_setting_reverse
+config['activation'] = 'tanh'
+
+
+config['lr'] = 0.0001
+config['reg_loss_coeff'] = 0.1
+
+config['signature'] = { 'id',
+                        'frame_nr',
+                        'pose_shape',
+                        'uv_normals',
+                        'mesh_mask',
+                        'mesh_body_posed',
+                        'mesh_posed',
+                        'uv_static',
+                        'uv_mask',
+                        'uv_body_posed',
+                        }
+
+if config['use_skinning']:
+  config['signature'].add('uv_unposed')
+  config['signature'].add('mesh_unposed')
+else:
+  config['signature'].add('uv_posed')

data/calculate_normals.py

@@ -0,0 +1,112 @@
+import os
+import sys
+import numpy as np
+import trimesh as tm
+import scipy.io as sio
+import plotly.graph_objects as go
+import plotly.express as px
+from tqdm import tqdm
+import torch
+from scipy.spatial.transform import Rotation as R
+import argparse
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
+
+from data.cloth3d_io import readOBJ, readPC2
+from data.utils import Frame
+from smpl.smpl_torch import SMPL
+
+
+def calculate_normals(vertices, mesh_file, poses):
+    _, faces, _, _ = readOBJ(mesh_file)
+    rodr = smpl.rodrigues(torch.tensor(poses.T.reshape(-1, 1, 3), dtype=torch.float32)).numpy().reshape(vertices.shape[0], -1, 3, 3)
+
+    normals = []
+    for i in range(vertices.shape[0]):
+        r = R.from_matrix(rodr[i, 0, :, :])
+        # assert r.as_matrix() == rodr[frame, 0, :, :]
+        out = r.apply(vertices[i], inverse=True)
+
+        # create a plotly mesh plot of out and vertices [i]
+        # fig = go.Figure(data=[
+        #     go.Mesh3d(
+        #         x=out[:, 0], y=out[:, 2], z=out[:, 1],
+        #         # i, j and k give the vertices of triangles
+        #         i=faces[:, 0], j=faces[:, 1], k=faces[:, 2],
+        #         name='y',
+        #         showscale=False
+        #     ),
+        #     go.Mesh3d(
+        #         x=vertices[i][:, 0], y=vertices[i][:, 1], z=vertices[i][:, 2],
+        #         # i, j and k give the vertices of triangles
+        #         i=faces[:, 0], j=faces[:, 1], k=faces[:, 2],
+        #         name='x',
+        #         showscale=False
+        #     )
+        # ])
+        # fig.show()
+    
+        mesh = tm.Trimesh(vertices=out, faces=faces, process=False, validate=False)
+        normals.append(mesh.vertex_normals)
+    return np.stack(normals)
+
+
+def get_frames(folder):
+    uv_static_folder = os.path.join(folder, 'pose_shape') 
+    frames = sorted(os.listdir(uv_static_folder))
+
+    frames = [frame.split('.')[0] for frame in frames] # .split('_')[0]
+    frames = [Frame(frame.split('_')[0], frame.split('_')[1]) for frame in frames]
+    return frames
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--folder", help = "end of range", default='')
+    parser.add_argument("-f_obj", "--folder_object", help = "end of range", default='/')
+    parser.add_argument("-s", "--start", help = "start of range", default=0)
+    parser.add_argument("-e", "--end", help = "end of range", default=10)
+    parser.add_argument("-n", "--name", help = "end of range", default='lhunor')
+    args = parser.parse_args()
+
+    obj_folder = args.folder_object # '/data-net/dataset/cloth3d/train'
+    folder = args.folder #'/data-local/127/train'
+
+    os.makedirs(os.path.join(folder, 'mesh_normals'), exist_ok=True)
+    frames = get_frames(folder)
+    smpl = SMPL('cpu', 'smpl')
+
+    ids = {frame.id for frame in frames}
+    ids = sorted(list(ids))[int(args.start):int(args.end)]
+
+    for id in tqdm(ids):
+        # find obj files in the folder
+        garment_obj_folder = os.path.join(obj_folder, id[:5])
+        garments = [file.split('.')[0] for file in os.listdir(garment_obj_folder) if file.endswith('.obj')]
+
+        if len(id) == 5:
+            garment = garments[0]
+        elif len(id) == 6:
+            garment = garments[1]
+        else:
+            raise ValueError('Invalid frame id')
+
+        mesh_file = os.path.join(garment_obj_folder, f'{garment}.obj')
+        vertices =readPC2(os.path.join(garment_obj_folder, f'{garment}.pc16'), True)['V']
+        poses = sio.loadmat(os.path.join(garment_obj_folder, f'info.mat'))['poses']
+        normals = calculate_normals(vertices, mesh_file, poses)
+
+        map = sio.loadmat(os.path.join(garment_obj_folder, f'{garment}_map.mat'))['map'][0]
+        mask = np.load(os.path.join(folder, 'mesh_mask', f'{id}.npy'))
+
+        mesh_normals = normals[:,map,:]
+        mesh_normals = mesh_normals * mask[None,:, None]
+
+        for i in range(vertices.shape[0]):
+            # np.savez(os.path.join(folder, 'mesh_normals', f'{id}_{i}.npz'), mesh_normals=mesh_normals)
+            np.savez_compressed(os.path.join(folder, 'mesh_normals', f'{id}_{i}.npz'), mesh_normals=mesh_normals[i])
+
+        # pass
+
+        # print(frame.id, frame.frame_nr)
+

data/convert_vto.py

@@ -1,259 +0,0 @@
-import numpy as np
-import os
-import sys
-import pickle
-import plotly.express as ex
-import plotly.graph_objects as go
-import torch
-import open3d as o3d
-import trimesh
-# from vedo import trimesh as trimesh_plot
-from scipy.spatial import KDTree
-from scipy.sparse import lil_matrix
-from scipy.optimize import minimize
-
-sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
-
-from smpl.smpl_torch import SMPL
-
-# import mask faces
-from custom_utils.custom_utils import _maskFaces
-
-smpl = SMPL(path='smpl')
-
-def energy_function(params, source_points, target_points, indices):
-    num_points = source_points.shape[0]
-    num_dims = source_points.shape[1]
-    
-    # Reshape the parameters
-    weights = params[:num_points * num_dims].reshape((num_points, num_dims))
-    translation = params[num_points * num_dims:num_points * num_dims + num_dims]
-    control_points = params[num_points * num_dims + num_dims:].reshape((-1, num_dims))
-    
-    # Compute the transformed source points
-    transformed_points = source_points + np.dot(weights, control_points.T)
-    
-    # Compute the TPS energy
-    tps_energy = 0.5 * np.sum(np.square(target_points[indices] - transformed_points))
-    
-    return tps_energy
-
-def non_rigid(source_points, target_points):
-    tree = KDTree(target_points)
-    distances, indices = tree.query(source_points)
-
-    # Initialize the optimization parameters
-    num_points = source_points.shape[0]
-    num_dims = source_points.shape[1]
-    initial_params = np.zeros(num_points * num_dims + num_dims + num_points)
-
-    # Define the optimization bounds
-    bounds = [(None, None)] * (num_points * num_dims)
-    bounds += [(None, None)] * num_dims
-    bounds += [(None, None)] * (num_points * num_dims)
-
-    # Perform the optimization
-    result = minimize(energy_function, initial_params, method='L-BFGS-B',
-                    args=(source_points, target_points, indices), bounds=bounds)
-
-    # Get the optimized parameters
-    optimized_params = result.x
-
-    # Reshape the parameters
-    optimized_weights = optimized_params[:num_points * num_dims].reshape((num_points, num_dims))
-    optimized_translation = optimized_params[num_points * num_dims:num_points * num_dims + num_dims]
-    optimized_control_points = optimized_params[num_points * num_dims + num_dims:].reshape((-1, num_dims))
-
-    transformed_points = source_points + np.dot(optimized_weights, optimized_control_points.T)
-    return transformed_points
-
-
-def register_smpl():
-    # generate body with zero pose and shape
-    pose = torch.zeros((1, 72))
-    shape = torch.zeros((1, 11))
-    smpl_out = smpl(pose, shape)[0][0]
-
-    with open('/data-net/code/vto-dataset/tshirt/simulations/tshirt_shape00_55_27.pkl', "rb") as f:
-        data = pickle.load(f)
-        garment = data['vertices'][100]
-        faces = data['faces']
-
-    # rotate garment x degrees along y axis
-    # convert degrees to radians
-    degrees = 107
-    theta = degrees * np.pi / 180
-    rot = np.array([[np.cos(theta), 0, np.sin(theta)],
-                    [0, 1, 0],
-                    [-np.sin(theta), 0, np.cos(theta)]])
-    garment = np.dot(rot, garment.T).T
-
-    # rotate garment x degrees along x axis
-    # convert degrees to radians
-    degrees = 5
-    theta = degrees * np.pi / 180
-    rot = np.array([[1, 0, 0],
-                    [0, np.cos(theta), -np.sin(theta)],
-                    [0, np.sin(theta), np.cos(theta)]])
-    garment = np.dot(rot, garment.T).T
-
-    # rotate garment x degrees along z axis
-    # convert degrees to radians
-    degrees = -5
-    theta = degrees * np.pi / 180
-    rot = np.array([[np.cos(theta), -np.sin(theta), 0],
-                    [np.sin(theta), np.cos(theta), 0],
-                    [0, 0, 1]])
-    garment = np.dot(rot, garment.T).T
-
-
-    garment[:,0] -= 0.06
-    garment[:,1] -= 0.03
-    garment[:,2] -= 0.015
-    
-    # visualize the body and garment
-    # fig = go.Figure(data=[go.Mesh3d(x=smpl_out[:,0], y=smpl_out[:,1], z=smpl_out[:,2], i=smpl.f[:,0], j=smpl.f[:,1], k=smpl.f[:,2])])
-    # fig.add_trace(go.Mesh3d(x=garment[:,0], y=garment[:,1], z=garment[:,2], i=faces[:,0], j=faces[:,1], k=faces[:,2]))
-    # fig.update_layout(scene=dict(xaxis=dict(range=[-1,1],), yaxis=dict(range=[-1,1],), zaxis=dict(range=[-1,1],),),)
-    # fig.show()
-
-    # non_rigid(garment, smpl_out.numpy())
-
-    # Load the source and target meshes
-    source_mesh = o3d.geometry.TriangleMesh(o3d.utility.Vector3dVector(garment), o3d.utility.Vector3iVector(faces))
-    target_mesh = o3d.geometry.TriangleMesh(o3d.utility.Vector3dVector(smpl_out), o3d.utility.Vector3iVector(smpl.f))
-    # flip source and target mesh
-    source_mesh, target_mesh = target_mesh, source_mesh
-
-    source_mesh.compute_vertex_normals()
-    source_mesh = source_mesh.filter_smooth_laplacian(number_of_iterations=10)
-    # plot the smoothed mesh with plotly
-    # fig = go.Figure(data=[go.Mesh3d(x=np.asarray(source_mesh.vertices)[:,0], y=np.asarray(source_mesh.vertices)[:,1], z=np.asarray(source_mesh.vertices)[:,2], i=np.asarray(source_mesh.triangles)[:,0], j=np.asarray(source_mesh.triangles)[:,1], k=np.asarray(source_mesh.triangles)[:,2])])
-    # fig.show()
-
-    # Perform initial alignment (translation and/or scaling)
-    # You can modify this step based on your specific needs
-    # For example, you can manually set an initial transformation
-    init_transformation = [[1.0, 0.0, 0.0, 0.0],
-                        [0.0, 1.0, 0.0, 0.0],
-                        [0.0, 0.0, 1.0, 0.0],
-                        [0.0, 0.0, 0.0, 1.0]]
-
-    # Create the ICP object and set the parameters
-    icp = o3d.pipelines.registration.TransformationEstimationPointToPoint()
-    criteria = o3d.pipelines.registration.ICPConvergenceCriteria(max_iteration=2000)
-        
-    # Perform ICP registration
-    reg_result = o3d.pipelines.registration.registration_icp(o3d.geometry.PointCloud(source_mesh.vertices),
-                                                             o3d.geometry.PointCloud(target_mesh.vertices),
-                                                             0.075,
-                                                             init_transformation, icp, criteria)
-
-    # Get the aligned source mesh after registration
-    aligned_mesh = source_mesh.transform(reg_result.transformation)
-
-    # Visualize the aligned mesh and target mesh with plotly
-    fig = go.Figure(data=[go.Mesh3d(x=np.asarray(aligned_mesh.vertices)[:,0], y=np.asarray(aligned_mesh.vertices)[:,1], z=np.asarray(aligned_mesh.vertices)[:,2], i=np.asarray(aligned_mesh.triangles)[:,0], j=np.asarray(aligned_mesh.triangles)[:,1], k=np.asarray(aligned_mesh.triangles)[:,2])])
-    fig.add_trace(go.Mesh3d(x=np.asarray(target_mesh.vertices)[:,0], y=np.asarray(target_mesh.vertices)[:,1], z=np.asarray(target_mesh.vertices)[:,2], i=np.asarray(target_mesh.triangles)[:,0], j=np.asarray(target_mesh.triangles)[:,1], k=np.asarray(target_mesh.triangles)[:,2]))
-    fig.update_layout(scene=dict(xaxis=dict(range=[-1,1],), yaxis=dict(range=[-1,1],), zaxis=dict(range=[-1,1],),),)
-    fig.show()
-
-    # Get the vertices of the aligned source mesh and target mesh
-    aligned_source_vertices = np.asarray(aligned_mesh.vertices)
-    target_vertices = np.asarray(target_mesh.vertices)
-
-    # Perform nearest neighbor search to find corresponding vertex indices
-    kd_tree = o3d.geometry.KDTreeFlann(target_mesh)
-    correspondences = []
-    matched = set()
-    for i in range(len(aligned_source_vertices)):
-        k = 1
-        _, idx, _ = kd_tree.search_knn_vector_3d(aligned_source_vertices[i], k)
-        # while idx[-1] in matched:
-        #     k += 1
-        #     _, idx, _ = kd_tree.search_knn_vector_3d(aligned_source_vertices[i], k)
-        # matched.add(idx[-1])
-        correspondences.append((i, idx[-1]))
-
-    # correspondences is a list of tuples where each tuple contains the corresponding vertex indices
-    # For example, correspondences = [(0, 10), (1, 5), (2, 15), ...]
-
-    # Visualize the correspondences with plotly
-    # fig = go.Figure(data=[go.Scatter3d(x=aligned_source_vertices[:,0], y=aligned_source_vertices[:,1], z=aligned_source_vertices[:,2], mode='markers', marker=dict(color='red', size=1))])
-    # fig.add_trace(go.Scatter3d(x=target_vertices[:,0], y=target_vertices[:,1], z=target_vertices[:,2], mode='markers', marker=dict(color='blue', size=1)))
-    # for corr in correspondences:
-    #     fig.add_trace(go.Scatter3d(x=[aligned_source_vertices[corr[0],0], target_vertices[corr[1],0]], y=[aligned_source_vertices[corr[0],1], target_vertices[corr[1],1]], z=[aligned_source_vertices[corr[0],2], target_vertices[corr[1],2]], mode='lines', line=dict(color='green', width=1)))
-    # fig.update_layout(scene=dict(xaxis=dict(range=[-1,1],), yaxis=dict(range=[-1,1],), zaxis=dict(range=[-1,1],),),)
-    # fig.show()
-
-    registered = np.zeros((14475, 3))
-    mask = np.zeros((14475), dtype=bool)
-    for i in range(len(correspondences)):
-        registered[correspondences[i][1]] = aligned_source_vertices[correspondences[i][0]]
-        mask[correspondences[i][1]] = True
-
-    # F = np.load(f'smpl/smpl_faces.npy').astype(np.int32)
-
-    # registered_masked, masked_faces = _maskFaces(registered, np.asarray(target_mesh.triangles), mask)
-    registered_masked, masked_faces = registered, np.asarray(target_mesh.triangles)
-
-    # plot aligned source mesh with the target topology
-    fig = go.Figure(data=[go.Mesh3d(x=registered_masked[:,0], y=registered_masked[:,1], z=registered_masked[:,2], i=masked_faces[:,0], j=masked_faces[:,1], k=masked_faces[:,2])])
-    fig.update_layout(scene=dict(xaxis=dict(range=[-1,1],), yaxis=dict(range=[-1,1],), zaxis=dict(range=[-1,1],),),)
-    fig.show()
-
-    return smpl
-
-
-def convert_vto(folder, out_folder):
-    os.makedirs(out_folder, exist_ok=True)
-    os.makedirs(os.path.join(out_folder, 'mesh_posed_no_orient'), exist_ok=True)
-    os.makedirs(os.path.join(out_folder, 'mesh_posed_body_no_orient'), exist_ok=True)
-    os.makedirs(os.path.join(out_folder, 'mesh_mask'), exist_ok=True)
-    os.makedirs(os.path.join(out_folder, 'pose_shape'), exist_ok=True)
-    F = np.load(f'smpl/smpl_faces.npy').astype(np.int32)
-
-    files = os.listdir(folder)
-    # files = ['/data-net/code/vto-dataset/tshirt/simulations/tshirt_shape00_55_27.pkl']
-    for file in files:
-        # read pickle files and get vetices and faces from them
-        with open(os.path.join(folder, file), "rb") as f:
-            data = pickle.load(f)
-            vertices = data['vertices']
-            faces = data['faces']
-            pose = data['pose']
-            shape = data['shape']
-            trans = data['translation']
-            id = file.split('.')[0]
-            # pose[:, :3] = 0
-            id = id.replace('_', '-')
-            subject = id.split('-')[2]
-            sequence = '_'.join(id.split('-')[2:])
-            # cmu_data = np.load(f'/data-net/code/vto-garment-collisions/assets/CMU/{subject}/{sequence}_poses.npz')
-            # shape = np.tile(cmu_data['betas'], (shape.shape[0], 1))
-            # pose = cmu_data['poses']
-
-            body = smpl(torch.tensor(pose), torch.tensor(np.concatenate((shape, np.zeros((shape.shape[0],1),dtype=np.float32)),axis=1)))[0].numpy()
-            pose_shape = np.concatenate((pose, shape, np.zeros((shape.shape[0],1),dtype=np.float32)),axis=1)
-            
-            # for i in range(vertices.shape[0]):
-            #     np.savez_compressed(os.path.join(out_folder, 'mesh_posed_body_no_orient', f'{id}_{i}.npz'), mesh_posed_body_no_orient=body[i])
-            #     np.save(os.path.join(out_folder, 'pose_shape', f'{id}_{i}.npy'), pose_shape[i])
-
-
-            # plot mesh with plotly
-            frame = 0
-            fig = go.Figure(data=[go.Mesh3d(x=vertices[frame,:,0], y=vertices[frame,:,1], z=vertices[frame,:,2], i=faces[:,0], j=faces[:,1], k=faces[:,2])])
-            fig.add_trace(go.Mesh3d(x=body[frame,:,0], y=body[frame,:,1], z=body[frame,:,2], i=F[:,0], j=F[:,1], k=F[:,2]))
-            fig.update_layout(scene=dict(xaxis=dict(range=[-1,1],), yaxis=dict(range=[-1,1],), zaxis=dict(range=[-1,1],),),)
-            fig.show()
-
-            
-            pass
-    pass
-
-
-if __name__ == '__main__':
-    # register_smpl()
-    convert_vto('/data-net/code/vto-dataset/dress/simulations', '/data-net/code/vto-dataset/dress_converted')
-    pass

data/dataset.py

@@ -175,6 +175,9 @@ class Dataset(Dataset):
         if 'mesh_curvature' in self.signature:
             mesh_curvature = np.load(os.path.join(self.folder, 'mesh_curvature', frame.id + separator + frame.frame_nr + '.npz')); mesh_curvature = mesh_curvature['mesh_curvature']
             item['mesh_curvature'] = mesh_curvature.astype(np.int32)
+        if 'mesh_normals' in self.signature:
+            mesh_normals = np.load(os.path.join(self.folder, 'mesh_normals', frame.id + separator + frame.frame_nr + '.npz')); mesh_normals = mesh_normals['mesh_normals']
+            item['mesh_normals'] = mesh_normals
 
         for res in self.resolutions:
             res_string = ''
@@ -203,6 +206,10 @@ class Dataset(Dataset):
                 uv_posed = np.load(os.path.join(self.folder, 'uv_posed_no_orient' + res_string, frame.id + separator + frame.frame_nr + '.npz')); uv_posed = uv_posed['uv_posed_no_orient']
                 item['uv_posed' + res_string] = uv_posed
 
+            if 'uv_normals' + res_string in self.signature:
+                uv_normals = np.load(os.path.join(self.folder, 'uv_normals' + res_string, frame.id + separator + frame.frame_nr + '.npz')); uv_normals = uv_normals['uv_normals']
+                item['uv_normals' + res_string] = uv_normals
+
 
         data = { key: (torch.from_numpy(value) if not (type(value) == str or type(value) == torch.Tensor) else value) for key, value in item.items()}
 

data/dataset_vto.py

@@ -1,40 +0,0 @@
-import os
-import sys
-
-import argparse
-parser = argparse.ArgumentParser(description='')
-parser.add_argument('--debug', action='store_true')
-args = vars(parser.parse_args())
-
-if args['debug']:
-    os.environ["CUDA_VISIBLE_DEVICES"] = '0'
-
-import math
-import time
-import torch
-import random
-import pytorch3d
-import torchvision as tv
-import numpy as np
-import scipy.io as sio
-import plotly.express as ex
-from torch.utils.data import Dataset
-
-sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
-
-from data.utils import Frame
-
-class Dataset(Dataset):
-    def __init__(self, folder):
-        pass
-
-    def __len__(self):
-        return len(self.frames)
-
-
-    def __getitem__(self, idx):
-        frame : Frame = self.frames[idx]
-
-
-if __name__ == '__main__':
-    from torch.utils.data import DataLoader

docker/docker-compose.yml

@@ -34,6 +34,7 @@ services:
       - '/data-net/guest/lhunor/data:/data-net'
       - '/data-fast/126-data1/lhunor/data:/data-local/126'
       - '/data-fast/127-data1/lhunor/data:/data-local/127'
+      - '/data-fast/127-data2/lhunor/data:/data-local/127_2'
       - '/data-fast/128-data1/lhunor/data:/data-local/128'
     deploy:
       resources:

losses/losses.py

@@ -18,12 +18,12 @@ from torch_cluster import knn
 smpl_to_keep_orig = sio.loadmat('smpl/keep_remove.mat')['keep'].reshape(-1)
 smpl_to_keep_orig = smpl_to_keep_orig[smpl_to_keep_orig < 6890]
 
-to_keep = sio.loadmat('smpl/keep_remove.mat')['keep'].reshape(-1)
-F_coll = torch.tensor(np.load("smpl/smpl_faces.npy")[to_keep,:], device='cuda')
-F_coll_skirt = torch.tensor(np.load("smpl/smpl_faces_skirt.npy")[to_keep,:], device='cuda')
+# to_keep = sio.loadmat('smpl/keep_remove.mat')['keep'].reshape(-1)
+F_coll = torch.tensor(np.load("smpl/smpl_faces.npy"), device='cuda')
+F_coll_skirt = torch.tensor(np.load("smpl/smpl_faces_skirt.npy"), device='cuda')
 F_coll_orig = torch.tensor(np.load("smpl/smpl_faces_orig.npy")[:6890,:], device='cuda')
-F_norm = torch.tensor(np.tile(np.load("smpl/smpl_faces.npy")[to_keep,:][None,:,:], [4,1,1]), device='cuda')
-F_norm_skirt = torch.tensor(np.tile(np.load("smpl/smpl_faces_skirt.npy")[to_keep,:][None,:,:], [4,1,1]), device='cuda')
+F_norm = torch.tensor(np.tile(np.load("smpl/smpl_faces.npy")[None,:,:], [4,1,1]), device='cuda')
+F_norm_skirt = torch.tensor(np.tile(np.load("smpl/smpl_faces_skirt.npy")[None,:,:], [4,1,1]), device='cuda')
 F_norm_orig = torch.tensor(np.tile(np.load("smpl/smpl_faces_orig.npy")[:6890,:][None,:,:], [4,1,1]), device='cuda')
 F_norm_orig = F_norm_orig[:,smpl_to_keep_orig,:]
 

run.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.direct_1 import config
+from configs.direct_6 import config
 import argparse
 
 # parser = argparse.ArgumentParser(description='')

run_parallel.py

@@ -4,24 +4,27 @@ import os
 
 if __name__ == '__main__':
 
-    folder = '/home/workspace/data/dataset/cloth3d/train'
+    # folder = '/home/workspace/data/dataset/cloth3d/train'
     # folder = '/home/workspace/local-data/cape_processed/train/male'
     # folder = '/home/workspace/local-data/valid_skirt'
     # folder = '/home/workspace/local-data/valid'
     # ids = os.listdir(folder)
     # ids = os.listdir(folder + '/mesh_posed_no_orient')#[280000:]
     skirt = 0
+    obj_folder = '/data-net/dataset/cloth3d/train'
+    folder = '/data-local/127/train'
 
-    count = int(500 * 1e3)
-    step = 10000
+    count = 0
+    step = 100
 
     # !!!!!!! change skirt parameter in script
 
-    while count < 510 * 1e3:
+    while count <1200:
         # os.system('nohup python convert_to_uv_cape.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_2_cape_train_male_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_2_cape_train_male_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')
         # os.system('nohup python convert_to_uv.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_unposed_onehot_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_train_unposed_onehot_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')
         # os.system('nohup python convert_to_uv_local.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_local_unposed_onehot_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_train_local_unposed_onehot_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')
-        os.system('nohup python data/convert_to_uv.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_2_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv/train_2_' + str(count) + '_' + str(count+step) + '_again.out' + ' 2>&1 &')
+        # os.system('nohup python data/convert_to_uv.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_2_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv/train_2_' + str(count) + '_' + str(count+step) + '_again.out' + ' 2>&1 &')
+        os.system(f'nohup python data/calculate_normals.py -f {folder} -f_obj {obj_folder}  -s {count} -e {(count+step)} -n normals_train_calc_{count}_{(count+step)} > nohup/normals/normals_train_calc_{count}_{(count+step)}.out' + ' 2>&1 &')
         # os.system('nohup python convert_to_uv_local_single.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_train_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_train_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')
         # os.system('nohup python convert_to_uv_local_single.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'to_uv_valid_skirt_local_unposed_onehot_3_' + str(count) + '_' + str(count+step) + ' > nohup/to_uv_valid_skirt_local_unposed_onehot_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')
         # os.system('nohup python downscale.py -f ' + folder + ' -s ' + str(count) + ' -e ' + str(count + step) + ' -n ' + 'downscale_valid_posed_no_orient_' + str(count) + '_' + str(count+step) + ' > nohup/downscale_valid_posed_no_orient_' + str(count) + '_' + str(count+step) + '.out' + ' 2>&1 &')

scripts/ae_norm_train.py

@@ -0,0 +1,218 @@
+import os
+import sys
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
+
+import argparse
+import importlib
+
+parser = argparse.ArgumentParser(description='')
+parser.add_argument('--name', type=str)
+parser.add_argument('--config', type=str)
+parser.add_argument('--debug', action='store_true')
+args = vars(parser.parse_args())
+
+config_module = importlib.import_module(args['config'])
+config = config_module.config
+if args['debug']:
+    from configs.ae_norm_1 import config
+os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
+if args['debug']:
+    os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
+
+import time
+import wandb
+import random
+import numpy as np
+import torch
+from torch import nn
+# from torchmetrics import MultiScaleStructuralSimilarityIndexMeasure as MSSSIM
+from custom_utils.custom_utils import UV2Mesh, get_summary_html, maskFacesBatch, get_summary_str
+import plotly.graph_objects as go
+import plotly.express as ex
+import torch.optim as optim
+
+if config['is_skirt']:
+    from losses.losses import edge_loss_skirt as edge_loss
+else:
+    from losses.losses import edge_loss
+
+from data.dataset import Dataset
+# from torch.utils.data import DataLoader
+from custom_utils.custom_utils import plot_3d
+from models.cond_encoder import AutoEncoder
+from utils.checkpoints import CheckpointSaver, load_model
+from custom_utils.custom_utils import Accumulator
+from utils.utils import InfiniteDataLoader as DataLoader
+
+from pytorch_msssim import MS_SSIM
+
+
+def init_wandb():
+    run = wandb.init(project='ae', name=config['name'], notes=config['notes'], dir='/data-local/127_2/temp', tags=config['tags'])
+    artifact = wandb.Artifact('code', type='code')
+    artifact.add_file('scripts/ae_norm_train.py')
+    artifact.add_file('losses/losses.py')
+    artifact.add_file('models/base.py')
+    artifact.add_file('models/cond_encoder.py')
+    artifact.add_file('models/convnext.py')
+    artifact.add_file('data/dataset.py')
+    config_file_names = [key for key in sys.modules.keys()  if key.startswith('configs.')]
+    for config_file_name in config_file_names:
+        file_name = '/'.join(config_file_name.split('.')) + '.py'
+        if os.path.exists(file_name):
+            artifact.add_file(file_name)
+    run.log_artifact(artifact)
+
+uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
+
+
+
+def train_step(batch):
+    model.train()
+    pred = model([batch['uv_normals'], batch['uv_mask']]) * 1.1
+
+    ms_ssim = torch.mean(_MSSSIM(pred + 1.1, batch['uv_normals'] + 1.1))
+    loss = 1e3 * (1 - ms_ssim)
+
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+
+    model.eval()
+    with torch.inference_mode():
+        mesh_pred = uv2mesh(pred)
+        mesh_orig = batch['mesh_normals']
+        l1_loss = torch.mean(torch.sum((torch.abs(batch['uv_normals'] - pred)), axis=1))
+        metric = torch.mean(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred) ** 2, axis=-1)))
+    
+    return { 'train/loss': loss, 'train/l1_loss': l1_loss, 'train/euclid_error': metric, 'train/ms_ssim': ms_ssim, }
+        
+
+def valid_step(batch):
+    model.eval()
+
+    with torch.inference_mode():
+        pred = model([batch['uv_normals'], batch['uv_mask']]) * 1.1
+
+        mesh_pred = uv2mesh(pred)
+        mesh_orig = batch['mesh_normals']
+
+        l1_loss = torch.mean(torch.sum((torch.abs(batch['uv_normals'] - pred)), axis=1))
+        ms_ssim = torch.mean(_MSSSIM(pred + 1.1, batch['uv_normals'] + 1.1))
+        loss = 1e3 * (1 - ms_ssim)
+
+        metric = torch.mean(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred) ** 2, axis=-1)))
+    
+    return { 'valid/loss': loss, 'valid/l1_loss': l1_loss, 'valid/euclid_error': metric, 'valid/ms_ssim': ms_ssim, }
+       
+
+def plot_everything(batch: dict, type: str):
+    model.eval()
+    pred = model([batch['uv_normals'], batch['uv_mask']]).detach() * 1.1
+    
+    # mesh_pred = uv2mesh(pred)
+    # mesh_orig = batch['mesh_normals']
+    
+    orig_img = torch.permute(batch['uv_normals'], [0,2,3,1])
+    pred_img = torch.permute(pred, [0,2,3,1])
+    img = torch.concat((orig_img, pred_img), axis=2)
+    imgs = [wandb.Image((img[i].cpu().numpy() + 1.1) / 2.2) for i in range(4)]
+
+    # F = np.load("smpl/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy")
+    # garment_unposed, faces = mesh_pred.cpu().numpy(), F
+    # gt = [mesh_orig[i].cpu().numpy() for i in range(pred.shape[0])]
+    
+    # body = uv2mesh(batch['uv_body_posed']).cpu().numpy()
+    # garment_posed = garment_unposed
+    # collision = [np.concatenate((body[i], garment_posed[i]), axis=0) for i in range(len(garment_posed))]
+    # collision_faces = [np.concatenate((F, faces + (3638 if config['is_cape'] else 14475)), axis=0) for i in range(len(garment_posed))]
+    # collision_color = [np.concatenate((np.ones_like(body[i], dtype=np.int32)*100, np.ones_like(garment_posed[i], dtype=np.int32)*230), axis=0) for i in range(len(garment_posed))]
+
+    # figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces, collision[i], collision_faces[i], collision_color[i]) for i in range(4) }
+    figs = {}
+
+    return { **figs, **{ type + '/images': imgs } }
+
+
+def train():
+    valid_iterator = iter(valid_dataloader)
+
+    step = 0
+    for batch in train_dataloader:
+        train_loss = train_step(batch)
+        train_accumulator.update(train_loss)
+
+        if step % 25 == 0:
+            data_valid = next(valid_iterator)
+            valid_loss = valid_step(data_valid)
+            valid_accumulator.update(valid_loss)
+
+        if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
+            to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
+            if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
+                train_figs = plot_everything(batch, 'train')
+                valid_figs = plot_everything(data_valid, 'valid')
+                to_log = { **to_log, **train_figs, **valid_figs }
+            if step == 0:
+                to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
+
+            if not args['debug']:
+                wandb.log(to_log, step=step)
+            train_accumulator.reset()
+            valid_accumulator.reset()
+            print(step)
+
+        if step % 10000 == 0:
+            ckpt_saver_enc(model.encoder, step, optimizer, scaler=None)
+            ckpt_saver_dec(model.decoder, step, optimizer, scaler=None)
+
+        step += 1
+
+
+class Scale(torch.nn.Module):
+    def __init__(self, scale_factor):
+        super().__init__()
+        self.scale_factor = scale_factor
+
+    def forward(self, x):
+        return x * self.scale_factor
+
+
+if __name__ == '__main__':
+    if not args['debug']:
+        init_wandb()
+
+    skirt_modifier = '_skirt' if config['is_skirt'] else ''
+
+    dataset_train = Dataset(config['train_dir'], config['signature'], device=config['device'], frames_file=config['train_frames'], use_cache=config['dataset_in_memory'], is_cape=config['is_cape'])
+    dataset_valid = Dataset(config['valid_dir'], config['signature'], device=config['device'], frames_file=config['valid_frames'], use_cache=config['dataset_in_memory'], is_cape=config['is_cape'])
+    train_dataloader = DataLoader(dataset_train, batch_size=4, drop_last=True, shuffle=True)
+    valid_dataloader = DataLoader(dataset_valid, batch_size=4, drop_last=True, shuffle=True)
+    train_accumulator = Accumulator()
+    valid_accumulator = Accumulator()
+
+    model : AutoEncoder = AutoEncoder(config['cond_ae'])
+    optimizer = optim.Adam(model.parameters(), lr=config['lr'])
+
+    # _MSSSIM = MSSSIM(data_range=2).to(config['device'])
+    _MSSSIM = MS_SSIM(data_range=2.2, size_average=True, channel=3)
+
+    if config['pretrained_model'] != '':
+        load_model(model.encoder, config['pretrained_model'] + '_enc')
+        load_model(model.decoder, config['pretrained_model'] + '_dec')
+
+    ckpt_saver_enc = CheckpointSaver('checkpoints/iccv/torch/' + config['name'] + '_enc', top_n=5)
+    ckpt_saver_dec = CheckpointSaver('checkpoints/iccv/torch/' + config['name'] + '_dec', top_n=5)
+
+    run_name = config['name']
+    print(f'started training: {run_name}')
+
+    if not args['debug']:
+        # model = torch.compile(model)
+        wandb.watch(model, log='all', log_freq=10000, log_graph=True)
+
+    train()
+
+
+

scripts/direct_train.py

@@ -13,7 +13,7 @@ args = vars(parser.parse_args())
 config_module = importlib.import_module(args['config'])
 config = config_module.config
 if args['debug']:
-    from configs.direct_1 import config
+    from configs.direct_7 import config
     # from configs.ae_static_orig_2 import config
 os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
@@ -52,7 +52,7 @@ from pytorch_msssim import MS_SSIM
 
 
 def init_wandb():
-    run = wandb.init(project='iccv', name=config['name'], notes=config['notes'], dir='/data-local/127/temp', tags=config['tags'])
+    run = wandb.init(project='iccv', name=config['name'], notes=config['notes'], dir='/data-local/127_2/temp', tags=config['tags'])
     artifact = wandb.Artifact('code', type='code')
     artifact.add_file('scripts/direct_train.py')
     artifact.add_file('losses/losses.py')
@@ -76,26 +76,37 @@ def train_step(batch):
     # with torch.autocast(device_type='cuda', dtype=torch.float16, enabled=use_amp):
     if config['use_curvature']:
         input = torch.cat([batch['uv_curvature'], batch['uv_static']], axis=1)
+    elif config['use_normals']:
+        input = torch.cat([batch['uv_normals'], batch['uv_body_posed'], batch['uv_static']], axis=1)
     else:
         input = torch.cat([batch['uv_body_posed'], batch['uv_static']], axis=1)
     pred = model([input, batch['uv_mask']])
 
     mesh_pred = uv2mesh(pred)
-    mesh_orig = batch['mesh_unposed']
-    shape = batch['pose_shape'][:, 72:]
-    gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
-    shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])])
-    mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,None]
-    mesh_pred_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)[0]
- 
+    if config['use_skinning']:
+        mesh_orig = batch['mesh_unposed']
+        uv_orig = batch['uv_unposed']
+        shape = batch['pose_shape'][:, 72:]
+        gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
+        shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])])
+        mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,None]
+        mesh_pred_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)[0]
+    else:
+        mesh_orig = batch['mesh_posed']
+        uv_orig = batch['uv_posed']
+        mesh_pred_posed = mesh_pred
+
+    l2_mesh_loss = torch.mean(1e3 * torch.mean(torch.sum(batch['mesh_mask'][:,:,None] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
+
     e_loss = torch.mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
     c_loss, c_ratio = collision_loss(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
     c_loss, c_ratio = torch.mean(c_loss), torch.mean(c_ratio)
     # l2_loss = torch.mean(1e3 * torch.mean(torch.sum(((batch['uv_unposed'] - pred) ** 2), axis=-1), axis=[1, 2]))
-    l2_mesh_loss = torch.mean(1e3 * torch.mean(torch.sum(batch['mesh_mask'][:,:,None] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
     n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
+  
 
-    loss = l2_mesh_loss + (e_loss + c_loss + n_loss) * config['reg_loss_coeff']
+    ms_ssim = torch.mean(_MSSSIM(pred + 1, uv_orig + 1))
+    loss = l2_mesh_loss + (1 - ms_ssim) * 1e3 + (e_loss + c_loss + n_loss) * config['reg_loss_coeff']
 
     optimizer.zero_grad()
     loss.backward()
@@ -107,9 +118,8 @@ def train_step(batch):
 
     model.eval()
     with torch.inference_mode():
-        ms_ssim = torch.mean(_MSSSIM(pred + 1, batch['uv_unposed'] + 1))
         metric = torch.sum(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred.detach()) ** 2, axis=-1))) / torch.sum(batch['mesh_mask'])
-    
+
     return { 'train/loss': loss, 'train/edge_loss': e_loss, 'train/euclid_error': metric, 'train/collision_ratio': c_ratio,
              'train/ms_ssim': ms_ssim, 'train/collision_loss': c_loss, 'train/l2_mesh_loss': l2_mesh_loss,
              'train/normal_loss': n_loss }
@@ -121,16 +131,25 @@ def valid_step(batch):
     with torch.inference_mode():
         if config['use_curvature']:
             input = torch.cat([batch['uv_curvature'], batch['uv_static']], axis=1)
+        elif config['use_normals']:
+            input = torch.cat([batch['uv_normals'], batch['uv_body_posed'], batch['uv_static']], axis=1)
         else:
             input = torch.cat([batch['uv_body_posed'], batch['uv_static']], axis=1)
+
         pred = model([input, batch['uv_mask']])
         mesh_pred = uv2mesh(pred)
-        mesh_orig = batch['mesh_unposed']
-        shape = batch['pose_shape'][:, 72:]
-        gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
-        shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])])
-        mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,None]
-        mesh_pred_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)[0]
+        if config['use_skinning']:
+            mesh_orig = batch['mesh_unposed']
+            uv_orig = batch['uv_unposed']
+            shape = batch['pose_shape'][:, 72:]
+            gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
+            shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])])
+            mesh_pred_shaped = mesh_pred + shape_offsets * batch['mesh_mask'][:,:,None]
+            mesh_pred_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], mesh_pred_shaped)[0]
+        else:
+            mesh_orig = batch['mesh_posed']
+            uv_orig = batch['uv_posed']
+            mesh_pred_posed = mesh_pred
     
         e_loss = torch.mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
         c_loss, c_ratio = collision_loss(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
@@ -139,8 +158,8 @@ def valid_step(batch):
         l2_mesh_loss = torch.mean(1e3 * torch.mean(torch.sum(batch['mesh_mask'][:,:,None] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
         n_loss = normal_loss(mesh_pred_posed, batch['mesh_posed'], batch['mesh_mask'], is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
 
-        ms_ssim = torch.mean(_MSSSIM(pred + 1, batch['uv_unposed'] + 1))
-        loss = l2_mesh_loss + (e_loss + c_loss + n_loss) * config['reg_loss_coeff']
+        ms_ssim = torch.mean(_MSSSIM(pred + 1, uv_orig + 1))
+        loss = l2_mesh_loss + (1 - ms_ssim) * 1e3 + (e_loss + c_loss + n_loss) * config['reg_loss_coeff']
 
         metric = torch.sum(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred) ** 2, axis=-1))) / torch.sum(batch['mesh_mask'])
     
@@ -154,49 +173,64 @@ def plot_everything(batch: dict, type: str):
     with torch.inference_mode():
         if config['use_curvature']:
             input = torch.cat([batch['uv_curvature'], batch['uv_static']], axis=1)
+        elif config['use_normals']:
+            input = torch.cat([batch['uv_normals'], batch['uv_body_posed'], batch['uv_static']], axis=1)
         else:
             input = torch.cat([batch['uv_body_posed'], batch['uv_static']], axis=1)
         pred = model([input, batch['uv_mask']]).detach()
 
     mesh_pred = uv2mesh(pred).cpu().numpy()
-    mesh_orig = batch['mesh_unposed'].cpu().numpy()
+    if config['use_skinning']:
+        mesh_orig = batch['mesh_unposed'].cpu().numpy()
+        uv_orig = batch['uv_unposed']
+    else:
+        mesh_orig = batch['mesh_posed'].cpu().numpy()
+        uv_orig = batch['uv_posed']
     
-    orig_img = torch.permute(batch['uv_unposed'], [0,2,3,1])
+    orig_img = torch.permute(uv_orig, [0,2,3,1])
     uv_static_img = torch.permute(batch['uv_static'], [0,2,3,1])
     uv_body_posed_img = torch.permute(batch['uv_body_posed'], [0,2,3,1])
+    uv_normals_img = (torch.permute(batch['uv_normals'], [0,2,3,1]) + 1.1) / 2.2
     pred_img = torch.permute(pred, [0,2,3,1])
-    img = torch.concat((uv_static_img, uv_body_posed_img, orig_img, pred_img), axis=2)
+    img = torch.concat((uv_static_img, uv_body_posed_img, uv_normals_img, orig_img, pred_img), axis=2)
     imgs = [wandb.Image((img[i].cpu().numpy() + 1) / 2.0) for i in range(4)]
     
     F = np.load("smpl/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy")
     mask = batch['mesh_mask'].cpu().numpy().astype(bool)
 
-    shape = batch['pose_shape'][:, 72:]
-    gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
-    shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])]).cpu().numpy()
+    if config['use_skinning']:
+        shape = batch['pose_shape'][:, 72:]
+        gender = torch.greater_equal(shape[:, 10] , 0.5).to(torch.long)
+        shape_offsets = torch.stack([torch.tensordot(shape[i,:10][None, :], smpl.shapedirs[gender[i]], dims=[[1], [2]])[0,smpl.to_keep,:] for i in range(shape.shape[0])]).cpu().numpy()
 
-    garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask)
-    garment_unposed = [garment_unposed[i] + shape_offsets[i, mask[i], :] for i in range(mask.shape[0])]
-    gt = [mesh_orig[i][mask[i], :] + shape_offsets[i, mask[i], :] for i in range(pred.shape[0])]
+        garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask)
+        garment_unposed = [garment_unposed[i] + shape_offsets[i, mask[i], :] for i in range(mask.shape[0])]
+        gt = [mesh_orig[i][mask[i], :] + shape_offsets[i, mask[i], :] for i in range(pred.shape[0])]
 
-    for i in range(mesh_orig.shape[0]):
-        mesh_pred[i, mask[i], :] += shape_offsets[i, mask[i], :]
-    mesh_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], torch.tensor(mesh_pred, device=config['device']))[0].cpu().numpy()
+        for i in range(mesh_orig.shape[0]):
+            mesh_pred[i, mask[i], :] += shape_offsets[i, mask[i], :]
+        mesh_posed = smpl(batch['pose_shape'][:, :72], batch['pose_shape'][:, 72:], torch.tensor(mesh_pred, device=config['device']))[0].cpu().numpy()
+    else:
+        garment_unposed, faces = maskFacesBatch(mesh_pred, F, mask)
+        mesh_posed = mesh_pred
+        gt = [mesh_orig[i][mask[i], :] for i in range(pred.shape[0])]
 
     gt_posed = [batch['mesh_posed'].cpu().numpy()[i, mask[i], :] for i in range(mask.shape[0])]
-
     mesh_posed_fixed = fix_collision(torch.tensor(mesh_posed, device=config['device']), batch['mesh_body_posed'], torch.tensor(mask, device=config['device'])).cpu().numpy()
     body = batch['mesh_body_posed'].cpu().numpy()
 
     garment_posed = [mesh_posed[i, mask[i], :] for i in range(body.shape[0])]
     collision = [np.concatenate((body[i], garment_posed[i]), axis=0) for i in range(len(garment_posed))]
-    collision_faces = [np.concatenate((F, faces[i] + config['smpl_res']), axis=0) for i in range(len(garment_posed))]
     collision_color = [np.concatenate((np.ones_like(body[i], dtype=np.int32)*100, np.ones_like(garment_posed[i], dtype=np.int32)*230), axis=0) for i in range(len(garment_posed))]
 
     garment_posed_fixed = [mesh_posed_fixed[i, mask[i], :] for i in range(body.shape[0])]
     collision_fixed = [np.concatenate((body[i], garment_posed_fixed[i]), axis=0) for i in range(len(garment_posed))]
     collision_gt = [np.concatenate((body[i], gt_posed[i]), axis=0) for i in range(len(garment_posed))]
 
+    # faces_ = np.load('/data-net/dataset/00761/faces/00761.npy')
+    # faces = [faces_ for i in range(len(garment_posed))]
+    collision_faces = [np.concatenate((F, faces[i] + config['smpl_res']), axis=0) for i in range(len(garment_posed))]
+
     figs = { type + "/garment_" + str(i): plot_3d(gt[i], garment_unposed[i], faces[i], collision[i], collision_faces[i], collision_color[i]) for i in range(4) }
     figs_coll = { type + "/garment_" + str(i) + '_collision': plot_3d_collision(collision_gt[i], collision[i], collision_fixed[i], collision_faces[i], collision_color[i]) for i in range(4) }
 
@@ -273,7 +307,7 @@ if __name__ == '__main__':
 
     if not args['debug']:
         # model = torch.compile(model)
-        wandb.watch(model, log='all', log_freq=1000, log_graph=True)
+        wandb.watch(model, log='all', log_freq=10000, log_graph=True)
 
     train()
 

smpl/smpl_torch.py

@@ -206,9 +206,9 @@ class SMPL(Module):
         )
       )
     
-    joints = torch.matmul(results, torch.transpose(torch.concat([J, torch.ones((J.shape[0], 24, 1), dtype=torch.float32, device=J.device)], axis=-1)[:,:,None,:],2,3))[:,:,:3,0]
+    joints = torch.matmul(results, torch.transpose(torch.concat([J, torch.ones((J.shape[0], 24, 1), dtype=torch.float32, device=J.get_device())], axis=-1)[:,:,None,:],2,3))[:,:,:3,0]
             
-    T = torch.stack([torch.tensordot(results[i], self.weights_all[torch.greater(g, 0.5).to(torch.long)], dims=([0], [1])).permute(2, 0, 1) for i, g in enumerate(betas[:,-1])])
+    T = torch.stack([torch.tensordot(results[i], self.weights[torch.greater(g, 0.5).to(torch.long)], dims=([0], [1])).permute(2, 0, 1) for i, g in enumerate(betas[:,-1])])
     rest_shape_h = torch.cat(
       (v_posed, torch.ones((batch_num, v_posed.shape[1], 1), dtype=torch.float32).to(self.device)), dim=2
     )
@@ -230,4 +230,4 @@ class SMPL(Module):
     if remove and verts is None:
       result = result[:,self.to_keep,:]
 
-    return result, joints
+    return result, joints, v_shaped, v_posed

utils/postprocess.py

@@ -22,7 +22,7 @@ from losses.losses import _nearest_neighbour
 """ COLLISION FUNCTIONS START """
 # F = np.load("smpl/smpl_faces.npy")
 to_keep = sio.loadmat('smpl/keep_remove.mat')['keep'].reshape(-1)
-F_fix_coll = torch.tensor(np.tile(np.load("smpl/smpl_faces.npy")[to_keep,:][None,:,:], [4,1,1]), device='cuda')
+F_fix_coll = torch.tensor(np.tile(np.load("smpl/smpl_faces.npy")[None,:,:], [4,1,1]), device='cuda')
 smpl_to_keep_orig_post = sio.loadmat('smpl/keep_remove.mat')['keep'].reshape(-1)
 smpl_to_keep_orig_post = smpl_to_keep_orig_post[smpl_to_keep_orig_post < 6890]
 F_fix_coll_orig = torch.tensor(np.tile(np.load("smpl/smpl_faces_orig.npy")[:6890,:][None,:,:], [4,1,1]), device='cuda')[:,smpl_to_keep_orig_post,:]