diff --git a/configs/iccvw/ae_body_1_10p.py b/configs/iccvw/ae_body_1_10p.py new file mode 100644 index 0000000..84d88e5 --- /dev/null +++ b/configs/iccvw/ae_body_1_10p.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '5' +config['name'] = 'body_iccvw_1_10p' +config['notes'] = ' ' +config['tags'] = ['torch', 'body'] +config['script'] = 'scripts/ae_body_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt_10p.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train_10p.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0001 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_mask', + 'mesh_body_posed', + } \ No newline at end of file diff --git a/configs/iccvw/ae_body_1_25p.py b/configs/iccvw/ae_body_1_25p.py new file mode 100644 index 0000000..9c285ca --- /dev/null +++ b/configs/iccvw/ae_body_1_25p.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '5' +config['name'] = 'body_iccvw_1_25p' +config['notes'] = ' ' +config['tags'] = ['torch', 'body'] +config['script'] = 'scripts/ae_body_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt_25p.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train_25p.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0001 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_mask', + 'mesh_body_posed', + } \ No newline at end of file diff --git a/configs/iccvw/ae_body_1_50p.py b/configs/iccvw/ae_body_1_50p.py new file mode 100644 index 0000000..c10ee2d --- /dev/null +++ b/configs/iccvw/ae_body_1_50p.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '5' +config['name'] = 'body_iccvw_1_50p' +config['notes'] = ' ' +config['tags'] = ['torch', 'body'] +config['script'] = 'scripts/ae_body_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt_50p.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train_50p.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0001 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_mask', + 'mesh_body_posed', + } \ No newline at end of file diff --git a/configs/iccvw/ae_body_1_skirt.py b/configs/iccvw/ae_body_1_skirt.py new file mode 100644 index 0000000..f383361 --- /dev/null +++ b/configs/iccvw/ae_body_1_skirt.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '1' +config['name'] = 'body_iccvw_1_skirt' +config['notes'] = ' ' +config['tags'] = ['torch', 'body'] +config['script'] = 'scripts/ae_body_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = True +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 16 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0001 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'pose_shape', + 'uv_body_posed', + 'mesh_mask', + 'mesh_body_posed', + } \ No newline at end of file diff --git a/configs/iccvw/ae_norm_1_skirt.py b/configs/iccvw/ae_norm_1_skirt.py new file mode 100644 index 0000000..cd44125 --- /dev/null +++ b/configs/iccvw/ae_norm_1_skirt.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '4' +config['name'] = 'norm_iccvw_1_skirt' +config['notes'] = ' ' +config['tags'] = ['torch', 'norm'] +config['script'] = 'scripts/ae_norm_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = True +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 40 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, 128, 9), + CNBlockConfig(128, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 128, 3), + CNBlockConfig(128, 64, 9), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0001 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'uv_mask', + 'uv_normals', + 'mesh_mask', + 'mesh_normals', + } \ No newline at end of file diff --git a/configs/iccvw/ae_static_1_skirt.py b/configs/iccvw/ae_static_1_skirt.py new file mode 100644 index 0000000..127c5b8 --- /dev/null +++ b/configs/iccvw/ae_static_1_skirt.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '3' +config['name'] = 'static_iccvw_1_skirt' +config['notes'] = ' ' +config['tags'] = ['torch', 'static'] +config['script'] = 'scripts/ae_static_train.py' + +config['pretrained_model'] = '' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = True +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 8 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(8, 16, 3), + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 64, 3), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 8, 3), + CNBlockConfig(8, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.0005 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'uv_static', + 'mesh_static', + 'mesh_mask', + 'uv_mask', + } \ No newline at end of file diff --git a/configs/iccvw/ae_static_5.py b/configs/iccvw/ae_static_5.py new file mode 100644 index 0000000..24d1fd9 --- /dev/null +++ b/configs/iccvw/ae_static_5.py @@ -0,0 +1,78 @@ +from utils.convnext_utils import CNBlockConfig + +config = dict() + +config['gpu'] = '5' +config['name'] = 'static_iccvw_5' +config['notes'] = 'lower learning rate' +config['tags'] = ['torch', 'static'] +config['script'] = 'scripts/ae_static_train.py' + +config['pretrained_model'] = 'checkpoints/iccvw/static_iccvw_3' + +config['batch_size'] = 4 +config['train_dir'] = '/data-local/127/cloth3d_processed' +config['valid_dir'] = '/data-local/127/cloth3d_processed' +config['test_dir'] = '/data-net/dataset/test_ready' + +config['resolution'] = 512 +config['is_skirt'] = False +config['dataset_in_memory'] = False +config['is_cape'] = False +config['remove_global_orientation'] = True +config['device'] = 'cuda' + +if config['is_skirt']: + config['train_frames'] = 'frames_train_skirt.txt' + config['valid_frames'] = 'frames_valid_skirt.txt' +else: + config['train_frames'] = 'frames_train.txt' + config['valid_frames'] = 'frames_valid.txt' + +config['smpl_res'] = 14475 +if config['is_cape']: + config['smpl_res'] = 3638 + +config['latent_dim'] = 128 + +config['bottleneck_scale'] = 2 + +config['cond_ae'] = dict() +config['cond_ae']['device'] = config['device'] +config['cond_ae']['is_label'] = False +config['cond_ae']['latent_dim'] = config['latent_dim'] +config['cond_ae']['kernel_size'] = 7 +config['cond_ae']['bottleneck_size'] = 8 * config['bottleneck_scale'] + +block_setting = [ + CNBlockConfig(8, 16, 3), + CNBlockConfig(16, 32, 3), + CNBlockConfig(32, 64, 3), + CNBlockConfig(64, config['cond_ae']['bottleneck_size'], 9), + CNBlockConfig(config['cond_ae']['bottleneck_size'], None, 3), +] + +block_setting_reverse = [ + CNBlockConfig(config['cond_ae']['bottleneck_size'], 64, 3), + CNBlockConfig(64, 32, 9), + CNBlockConfig(32, 16, 3), + CNBlockConfig(16, 8, 3), + CNBlockConfig(8, 3, 3), + # CNBlockConfig(3, 3, 3), +] + +config['cond_ae']['encoder_block_settings'] = block_setting +config['cond_ae']['decoder_block_settings'] = block_setting_reverse +config['cond_ae']['activation'] = 'tanh' + + +config['lr'] = 0.00005 +config['cond_ae']['upscale_output'] = False + +config['signature'] = { 'id', + 'frame_nr', + 'uv_static', + 'mesh_static', + 'mesh_mask', + 'uv_mask', + } \ No newline at end of file diff --git a/scripts/ae_body_train.py b/scripts/ae_body_train.py index e6dfa9a..37938cc 100644 --- a/scripts/ae_body_train.py +++ b/scripts/ae_body_train.py @@ -11,10 +11,12 @@ parser.add_argument('--config', type=str) parser.add_argument('--debug', action='store_true') args = vars(parser.parse_args()) -config_module = importlib.import_module(args['config']) -config = config_module.config if args['debug']: from configs.iccvw.ae_body_cape_m import config +else: + config_module = importlib.import_module(args['config']) + config = config_module.config + os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu'] if args['debug']: @@ -32,10 +34,7 @@ import plotly.graph_objects as go import plotly.express as ex import torch.optim as optim -if config['is_skirt']: - from losses.losses import edge_loss_skirt as edge_loss -else: - from losses.losses import edge_loss +from losses.losses import edge_loss from data.dataset import Dataset # from torch.utils.data import DataLoader @@ -65,7 +64,7 @@ def init_wandb(): artifact.add_file(file_name) run.log_artifact(artifact) -uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape']) +uv2mesh = UV2Mesh(res=256, is_skirt=False, is_smpl_orig=config['is_cape']) @@ -122,7 +121,7 @@ def plot_everything(batch: dict, type: str): img = torch.concat((orig_img, pred_img), axis=2) imgs = [wandb.Image((img[i].cpu().numpy() + 1) / 2.0) for i in range(4)] - F = np.load("smpl/smpl_faces" + skirt_modifier + ("_orig" if config['is_cape'] else "") + ".npy") + F = np.load("smpl/smpl_faces.npy") garment_unposed, faces = mesh_pred.cpu().numpy(), F gt = [mesh_orig[i].cpu().numpy() for i in range(pred.shape[0])] @@ -176,9 +175,7 @@ if __name__ == '__main__': if not args['debug']: init_wandb() - skirt_modifier = '_skirt' if config['is_skirt'] else '' - - uv_mask_body = np.load('custom_utils/mask_256x512' + ('_skirt' if config['is_skirt'] else '') + '.npy') + uv_mask_body = np.load('custom_utils/mask_256x512.npy') uv_mask_body = cv2.dilate(uv_mask_body.astype(np.float32), cv2.getStructuringElement(cv2.MORPH_RECT, (2 * 2 + 1, 2 * 2 + 1), (2, 2)), iterations=2) uv_mask_body = torch.from_numpy(uv_mask_body).float().to(config['device']) diff --git a/scripts/ae_static_train.py b/scripts/ae_static_train.py index d445d40..3e6de96 100644 --- a/scripts/ae_static_train.py +++ b/scripts/ae_static_train.py @@ -94,7 +94,7 @@ def train_step(batch): mesh_orig = batch['mesh_static'] l1_loss = torch.mean(torch.sum((torch.abs(batch['uv_static'] - pred)), axis=1)) e_loss = 1e-2 * torch.mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask'])) - metric = torch.sum(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred.detach()) ** 2, axis=-1))) / torch.sum(batch['mesh_mask']) + metric = torch.sum(torch.sqrt(torch.sum(batch['mesh_mask'][:,:,None] * (mesh_orig - mesh_pred) ** 2, axis=-1))) / torch.sum(batch['mesh_mask']) return { 'train/loss': loss, 'train/edge_loss': e_loss, 'train/l1_loss': l1_loss, 'train/euclid_error': metric, 'train/ms_ssim': ms_ssim, }