started condition trainings

auto_encoder_static.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from configs.config_ae_static import config
+from configs.iccv.config_ae_static_cape_f import config
 os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
 
 import tensorflow as tf
@@ -34,10 +34,11 @@ from tf_utils import check_tf_version, allow_memory_growth
 from dataset import DatasetFactory, StaticDataset
 from custom_utils.custom_utils import batch_to_dict, plot_3d
 from models.cond_encoder import AutoEncoderNew as AutoEncoder
+from custom_utils.custom_utils import Accumulator
 
 
 def init_wandb():
-    run = wandb.init(project='ae', name=config['name'], notes=config['notes'], dir='/home/workspace/local-data/temp')
+    run = wandb.init(project='ae', name=config['name'], notes=config['notes'], dir='/tmp', tags=config['tags'])
     artifact = wandb.Artifact('code', type='code')
     artifact.add_file('auto_encoder_static.py')
     artifact.add_file('losses.py')
@@ -47,7 +48,9 @@ def init_wandb():
     artifact.add_file('convnext/models/convnext_tf.py')
     config_file_names = [key for key in sys.modules.keys()  if key.startswith('configs.')]
     for config_file_name in config_file_names:
-        artifact.add_file('/'.join(config_file_name.split('.')) + '.py')
+        file_name = '/'.join(config_file_name.split('.')) + '.py'
+        if os.path.exists(file_name):
+            artifact.add_file(file_name)
     run.log_artifact(artifact)
 
 uv2mesh = UV2Mesh(res=256, is_skirt=config['is_skirt'], is_smpl_orig=config['is_cape'])
@@ -125,27 +128,32 @@ def train():
     for batch in dataset_train:
         batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic']))
         train_loss = train_step(batch)
+        train_accumulator.update(train_loss)
 
         data_valid = valid_iterator.get_next()
         data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
         valid_loss = valid_step(data_valid)
+        valid_accumulator.update(valid_loss)
 
-        to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
-        if step % 5000 == 0 or (step < 5000 and step % 500 == 0):
-            train_figs = plot_everything(batch, 'train')
-            valid_figs = plot_everything(data_valid, 'valid')
-            to_log = { **to_log, **train_figs, **valid_figs }
-        if step == 0:
-            to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
+        if step % 1000 == 0 or (step < 1000 and step % 10 == 0):
+            to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
+            if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
+                train_figs = plot_everything(batch, 'train')
+                valid_figs = plot_everything(data_valid, 'valid')
+                to_log = { **to_log, **train_figs, **valid_figs }
+            if step == 0:
+                to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
 
-            
-        wandb.log(to_log, step=step)
+                
+            wandb.log(to_log, step=step)
+            train_accumulator.reset()
+            valid_accumulator.reset()
 
         if step % 10000 == 0:
             manager_enc.save(checkpoint_number=step)
             manager_dec.save(checkpoint_number=step)
 
-        print(step)
+            print(step)
 
         step += 1
 
@@ -231,6 +239,9 @@ if __name__ == '__main__':
     model : AutoEncoder = AutoEncoder(config['cond_ae'])
     optimizer = tf.optimizers.Adam(learning_rate=config['lr'])
 
+    train_accumulator = Accumulator()
+    valid_accumulator = Accumulator()
+
     if config['pretrained_model'] != '':
         load_model(model.encoder, config['pretrained_model'] + '_enc')
         load_model(model.decoder, config['pretrained_model'] + '_dec')