trial

2023-03-04 13:17:36 +00:00
parent 7c92e03b45
commit 8666d90312
5 changed files with 291 additions and 93 deletions
@@ -86,7 +86,7 @@ nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f_03223_SL" > nohup
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_cape_f_03223_SS" > nohup/iccv/pyr/iccv_pyr_64_cape_f_03223_SS.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_128_cape_f" > nohup/iccv/pyr/iccv_pyr_64_128_cape_f.out 2>&1 &

-nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_64_pre_cape_f" > nohup/iccv/pyr/iccv_pyr_64_pre_cape_f.out 2>&1 &
+nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_pre_cape_f" > nohup/iccv/pyr/iccv_pyr_32_64_128_pre_cape_f.out 2>&1 &

 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128" > nohup/iccv/pyr/iccv_pyr_32_64_128.out 2>&1 &
 nohup python cvae_pyramid.py --name="lhunor_iccv_pyr_32_64_128_skirt" > nohup/iccv/pyr/iccv_pyr_32_64_128_skirt.out 2>&1 &
@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_32_64_skin import config
+from configs.iccv.config_abl_pyr_32_64_128_skirt import config

 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 # import tensorflow as tf
@@ -0,0 +1,188 @@
+config = dict()
+
+config['gpu'] = '2'
+config['name'] = 'iccv_pyr_32_64_128_pre_cape_f'
+config['notes'] = 'load pretrained on cloth3d'
+config['tags'] = ['pyr', 'iccv', 'cape_f']
+
+# DON'T FORGET TO RESET SSIM LOSS USAGE
+# DON'T FORGET TO CHECK MODEL LOADING
+# DON'T FORGET TO CHECK IF STORE IN MEMORY
+# DON'T FORGET TO CHECK REG LOSS COEFF
+# DON'T FORGET TO LOAD FINAL UPSCALER UNLESS HAVE LEVEL 512
+# DON'T FORGET TO CHECK IF SKIRT
+# DON'T FORGET TO CHECK IF CAPE
+# DON'T FORGET TO CHECK IF USING DOWNSCALE NONE OR _2
+# DON'T FORGET TO CHECK UPSCALING and ENCODER CHECKPOINTS
+
+config['use_skinning'] =              True # and deshaping
+config['remove_global_orientation'] = True
+config['use_extra_losses'] =          True # edge, collision, 3d mesh
+config['use_offsets'] =               False
+config['use_norm_loss'] =             True
+config['use_curvature'] =             True
+
+config['pretrained_model'] = 'checkpoints/iccv/iccv_pyr_32_64_128'
+config['pretrained_model_iteration'] = ''
+
+config['pyramid'] = dict()
+config['pyramid']['resolutions'] = [32, 64, 128]
+config['pyramid']['pretrained_pyramid'] = ''
+config['pyramid']['final_upscale_enhancer'] = ''
+
+config['train_dir'] = '/home/workspace/local-data/cape_processed/train/female'
+config['valid_dir'] = '/home/workspace/local-data/cape_processed/test/female'
+config['test_dir'] = '/home/workspace/local-data_127/cape_processed/test/female'
+config['batch_size'] = 4
+config['is_skirt'] = False
+config['is_cape'] = True
+
+config['dataset_in_memory'] = False
+config['resolution'] = None
+
+config['smpl_res'] = 14475
+if config['is_cape']:
+  config['smpl_res'] = 3638
+
+config['lr'] = 0.00001
+config['starting_balancing_coeff'] = 0.001
+config['reg_loss_coeff'] = 0.1 # set to 0 if don't want to use edge+collision
+config['avg_garment_path'] = ''
+config['bottleneck_scale'] = 2
+
+config['signature_dynamic'] = { 'id': (), 'frame_nr': (),
+                          'pose_shape': (72 + 11,),
+                          'uv_body_posed': (512, 256, 3),
+                          'mesh_body_posed': (config['smpl_res'], 3,),
+                          'mesh_posed': (config['smpl_res'], 3,),
+                          'mesh_unposed': (config['smpl_res'], 3,),
+                          'uv_unposed': (512, 256, 3),
+                          'uv_curvature': (512, 256, 7),
+                        }
+config['signature_static'] = { 'uv_static': (512, 256, 3),
+                              'mesh_mask': (config['smpl_res'],),
+                              'uv_mask': (512, 256,),
+                              }
+
+config['pyramid']['is_label'] = False
+config['pyramid']['add_noise'] = False
+config['pyramid']['noise_amount'] = 0.0
+
+config['pyramid']['loss_weights'] = [1,1,1,1,1]
+
+for res in config['pyramid']['resolutions']:
+  if res != 512:
+      res_modifier = '_' + str(res)
+      config['signature_dynamic']['uv_body_posed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_unposed' + res_modifier] = (res, res//2, 3)
+      config['signature_dynamic']['uv_curvature' + res_modifier] = (res, res//2, 7)
+      config['signature_static']['uv_static' + res_modifier] = (res, res//2, 3)
+      config['signature_static']['uv_mask' + res_modifier] = (res, res//2)
+
+
+config['pyramid']['static_model'] = 'checkpoints/iccv/static_iccv_1_cape_f'
+config['pyramid']['body_model'] = 'checkpoints/iccv/body_iccv_1_cape_f'
+config['pyramid']['curvature_model'] = 'checkpoints/iccv/curv_iccv_1_cape_f'
+config['pyramid']['freeze_encoders'] = True
+
+
+config['pyramid']['levels'] = []
+for res in config['pyramid']['resolutions']:
+  if res == 512:
+    level = dict()
+    level['resolution'] = 512
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 7
+    level['encoder_depths'] = [3, 3, 9, 3]
+    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['decoder_input'] = (16, 8, 80)
+    level['decoder_depths'] = [3, 3, 9, 3]
+    level['decoder_dims'] = [256, 128, 64, 16]
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = ''
+    config['pyramid']['levels'].append(level)
+  elif res == 256:
+    level = dict()
+    level['resolution'] = 256
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9, 9]
+    level['encoder_dims'] = [32, 64, 128, level['latent_dim']]
+    level['decoder_input'] = (8, 4, 128)
+    level['decoder_depths'] = [9, 9, 9, 9]
+    level['decoder_dims'] = [256, 128, 64, 16]
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_'#'checkpoints/up_3_d256_512_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 128:
+    level = dict()
+    level['resolution'] = 128
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 5
+    level['encoder_depths'] = [9, 9, 9]
+    level['encoder_dims'] = [128, 128, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_128_2_cape_f'#'checkpoints/up_3_d128_256_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 64:
+    level = dict()
+    level['resolution'] = 64
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [9, 9]
+    level['encoder_dims'] = [64, level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_64_2_cape_f'#checkpoints/up_3_d64_128_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  elif res == 32:
+    level = dict()
+    level['resolution'] = 32
+    level['latent_dim'] = 64 * config['bottleneck_scale']
+    level['kernel_size'] = 3
+    level['encoder_depths'] = [16]
+    level['encoder_dims'] = [level['latent_dim']]
+    level['decoder_depths'] = level['encoder_depths']
+    level['decoder_dims'] = level['encoder_dims']
+    level['avg_garment_path'] = ''
+    level['upscale_enhancer'] = 'checkpoints/iccv/iccv_32_2_cape_f'#'checkpoints/up_3_d32_64_2'
+    level['upscale_enhancer_freeze'] = True
+    config['pyramid']['levels'].append(level)
+  else:
+    raise NotImplementedError
+
+config['cond_ae_static'] = dict()
+config['cond_ae_static']['is_label'] = False
+config['cond_ae_static']['kernel_size'] = 7
+config['cond_ae_static']['bottleneck_size'] = 8 * config['bottleneck_scale']
+config['cond_ae_static']['encoder_depths'] = [3, 3, 3, 27, 3]
+config['cond_ae_static']['encoder_dims'] = [8, 16, 32, 64, config['cond_ae_static']['bottleneck_size']]
+config['cond_ae_static']['decoder_depths'] = config['cond_ae_static']['encoder_depths']
+config['cond_ae_static']['decoder_dims'] = config['cond_ae_static']['encoder_dims']
+config['cond_ae_static']['activation'] = 'tanh'
+
+config['cond_ae_curv'] = dict()
+config['cond_ae_curv']['is_label'] = True
+config['cond_ae_curv']['kernel_size'] = 7
+config['cond_ae_curv']['bottleneck_size'] = 40 * config['bottleneck_scale']
+config['cond_ae_curv']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_curv']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_curv']['bottleneck_size']]
+config['cond_ae_curv']['decoder_depths'] = config['cond_ae_curv']['encoder_depths']
+config['cond_ae_curv']['decoder_dims'] = config['cond_ae_curv']['encoder_dims']
+config['cond_ae_curv']['activation'] = 'tanh'
+
+config['cond_ae_body'] = dict()
+config['cond_ae_body']['is_label'] = False
+config['cond_ae_body']['kernel_size'] = 7
+config['cond_ae_body']['bottleneck_size'] = 16 * config['bottleneck_scale']
+config['cond_ae_body']['encoder_depths'] = [3, 3, 9, 9, 3]
+config['cond_ae_body']['encoder_dims'] = [16, 32, 64, 128, config['cond_ae_body']['bottleneck_size']]
+config['cond_ae_body']['decoder_depths'] = config['cond_ae_body']['encoder_depths']
+config['cond_ae_body']['decoder_dims'] = config['cond_ae_body']['encoder_dims']
+config['cond_ae_body']['activation'] = 'tanh'
@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.iccv.config_abl_pyr_64_pre_cape_f import config
+from configs.iccv.config_abl_pyr_32_64_128_pre_cape_f import config

 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 import tensorflow as tf
@@ -220,20 +220,25 @@ def train_step(batch):
        out_levels = model([levels_input, conds], training=True)

        if config['use_skinning']:
-            mesh_orig, uv_orig = batch['mesh_unposed'], batch['uv_unposed']
+            mesh_orig = batch['mesh_unposed']
        else:
-            mesh_orig, uv_orig = batch['mesh_posed'], batch['uv_posed']
+            mesh_orig = batch['mesh_posed']

        output = dict()
        reconstr_loss_total = 0
        kl_total = 0

-        #for i, res in enumerate(config['pyramid']['resolutions']):
-        i = -1
-        res = config['pyramid']['resolutions'][-1]
+        for i, res in enumerate(config['pyramid']['resolutions']):
+        # i = -1
+            res = config['pyramid']['resolutions'][i] if i != len(config['pyramid']['resolutions']) - 1 else 512
            res_modifier = '_' + str(res) if res != 512 else ''
            pred, mean, logvar = out_levels[i]

+            if config['use_skinning']:
+                uv_orig = batch['uv_unposed' + res_modifier]
+            else:
+                uv_orig = batch['uv_posed' + res_modifier]
+
            mesh_pred = uv2mesh(pred)

            if config['use_skinning']:
@@ -251,7 +256,7 @@ def train_step(batch):
            e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
            c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
            c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio)
-        l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2]))
+            l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1) * batch['uv_mask' + res_modifier], axis=[1, 2]))
            # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
            # ms_ssim = tf.reduce_mean(tf.image.ssim(pred + 1, batch['uv_unposed'] + 1, max_val=3))
            l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
@@ -294,20 +299,25 @@ def valid_step(batch):


    if config['use_skinning']:
-        mesh_orig, uv_orig = batch['mesh_unposed'], batch['uv_unposed']
+        mesh_orig = batch['mesh_unposed']
    else:
-        mesh_orig, uv_orig = batch['mesh_posed'], batch['uv_posed']
+        mesh_orig = batch['mesh_posed']

    output = dict()
    reconstr_loss_total = 0
    kl_total = 0

-    #for i, res in enumerate(config['pyramid']['resolutions']):
-    i = -1
-    res = config['pyramid']['resolutions'][-1]
+    for i, res in enumerate(config['pyramid']['resolutions']):
+    # i = -1
+        res = config['pyramid']['resolutions'][i] if i != len(config['pyramid']['resolutions']) - 1 else 512
        res_modifier = '_' + str(res) if res != 512 else ''
        pred, mean, logvar = out_levels[i]

+        if config['use_skinning']:
+            uv_orig = batch['uv_unposed' + res_modifier]
+        else:
+            uv_orig = batch['uv_posed' + res_modifier]
+
        mesh_pred = uv2mesh(pred)

        if config['use_skinning']:
@@ -324,7 +334,7 @@ def valid_step(batch):
        e_loss = tf.reduce_mean(edge_loss(mesh_pred, mesh_orig, batch['mesh_mask']))
        c_loss, c_ratio = collision_penalty(mesh_pred_posed, batch['mesh_body_posed'], batch['mesh_mask'])
        c_loss, c_ratio = tf.reduce_mean(c_loss), tf.reduce_mean(c_ratio)
-    l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1), axis=[1, 2]))
+        l2_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(((uv_orig - pred) ** 2), axis=-1) * batch['uv_mask' + res_modifier], axis=[1, 2]))
        # ms_ssim = tf.reduce_mean(tf.image.ssim_multiscale(pred + 5, batch['uv_unposed'] + 5, max_val=10, power_factors=(0.5, 0.2, 0.1, 0.1, 0.1)))
        # ms_ssim = tf.reduce_mean(tf.image.ssim(pred + 1, batch['uv_unposed'] + 1, max_val=3))
        l2_mesh_loss = tf.reduce_mean(1e3 * tf.reduce_mean(tf.reduce_sum(batch['mesh_mask'][:,:,tf.newaxis] * (batch['mesh_posed'] - mesh_pred_posed) ** 2, axis=-1), axis=-1))
@@ -810,7 +820,7 @@ if __name__ == '__main__':
        if config['use_curvature']:
            model.curv_enc.trainable = False
    else:
-        load_model(model, config['pyramid']['pretrained_pyramid'], iteration=(config['pyramid']['pretrained_pyramid_iteration'] if 'pretrained_pyramid_iteration' in config.keys() else ''))
+        # load_model(model, config['pyramid']['pretrained_pyramid'], iteration=(config['pyramid']['pretrained_pyramid_iteration'] if 'pretrained_pyramid_iteration' in config.keys() else ''))
        model.static_enc.trainable = False
        model.body_enc.trainable = False
        if config['use_curvature']:
@@ -828,8 +838,8 @@ if __name__ == '__main__':
    #     load_model(model.upscale_enhancer, config['pyramid']['final_upscale_enhancer'])
    #     model.upscale_enhancer.trainable = False

-    for i in range(len(model.levels)-1):
-        model.levels[i].trainable = False
+    # for i in range(len(model.levels)-1):
+    #     model.levels[i].trainable = False

    # load_model(model.cond_encoder, config['pyramid']['pretrained_pyramid'])
    # load_model(model.levels[0], config['pyramid']['pretrained_pyramid'])
@@ -335,8 +335,8 @@ class PyramidNew(tf.keras.Model):
            pred = pred + levels_input[0][2] # uv_body_posed_32
        if 1 != len(self.levels):
            pred = self.levels[0].enhance(pred, levels_input[0][-2], training=training)  # levels_input[i][-2]=uv_mask_res
-            pred = tf.image.resize(pred, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
            out_levels.append([pred, mean, logvar])
+            pred = tf.image.resize(pred, (self.levels[1].size, self.levels[1].size//2), tf.image.ResizeMethod.BILINEAR)
            pred = pred * levels_input[1][-2][:,:,:,tf.newaxis]
        else:
            # pred = self.enhance(pred, levels_input[0][-2], training=training)