improe mdoel saving/laoding

custom_utils/custom_utils.py

@@ -291,22 +291,6 @@ def sparse_to_tensor(L):
     return tf.SparseTensor(idx, data, shape)
 
 
-def load_model(model, checkpoint, disable_assert=False, assert_consumed_on=False, iteration=''):
-    ckpt = tf.train.Checkpoint(model=model)
-    manager = tf.train.CheckpointManager(ckpt, checkpoint, max_to_keep=1)
-    if not disable_assert:
-        assert manager.latest_checkpoint is not None, checkpoint
-        assert manager.latest_checkpoint.split('/')[-1] != 'ckpt-0', checkpoint
-    if assert_consumed_on:
-        ckpt.restore(manager.latest_checkpoint).assert_consumed()
-    else:
-        if iteration != '':
-            name_base = manager.latest_checkpoint.split('-')[0]
-            ckpt_ind = [i for i in range(len(manager.checkpoints)) if manager.checkpoints[i].endswith(name_base + '-' + iteration)][0]
-            ckpt.restore(manager.checkpoints[ckpt_ind]).assert_existing_objects_matched()#.assert_consumed()
-        else:
-            ckpt.restore(manager.latest_checkpoint).assert_existing_objects_matched()#.assert_consumed()
-
 
 def get_summary_str(model):
     # from torchview import draw_graph

utils/checkpoints.py

@@ -5,7 +5,7 @@ import logging
 
 
 class CheckpointSaver:
-    def __init__(self, dirpath, decreasing=False, top_n=5):
+    def __init__(self, dirpath, top_n=5):
         """
         dirpath: Directory path where to store all model weights 
         decreasing: If decreasing is `True`, then lower metric is better
@@ -14,25 +14,27 @@ class CheckpointSaver:
         if not os.path.exists(dirpath): os.makedirs(dirpath)
         self.dirpath = dirpath
         self.top_n = top_n
-        self.decreasing = decreasing
         self.top_model_paths = []
-        self.best_metric_val = np.Inf if decreasing else -np.Inf
         
-    def __call__(self, model, epoch, optimizer, metric_val=np.Inf):
-        model_path = os.path.join(self.dirpath, model.__class__.__name__ + f'_epoch{epoch}.pt')
-        save = metric_val<self.best_metric_val if self.decreasing else metric_val>self.best_metric_val
-        if save: 
-            logging.info(f"Current metric value better than {metric_val} better than best {self.best_metric_val}, saving model at {model_path}")
-            self.best_metric_val = metric_val
-            torch.save({ 'model': model.state_dict(), 'optimizer':optimizer }, model_path)
-            self.top_model_paths.append({'path': model_path, 'score': metric_val})
-            self.top_model_paths = sorted(self.top_model_paths, key=lambda o: o['score'], reverse=not self.decreasing)
+    def __call__(self, model, iteration, optimizer):
+        model_path = os.path.join(self.dirpath, f'iteration_{iteration}.pt')
+        torch.save({ 'model': model.state_dict(), 'optimizer': optimizer }, model_path)
+        self.top_model_paths.append({'path': model_path})
         if len(self.top_model_paths)>self.top_n: 
             self.cleanup()
     
     def cleanup(self):
-        to_remove = self.top_model_paths[self.top_n:]
+        to_remove = self.top_model_paths[:len(self.top_model_paths)-self.top_n]
         logging.info(f"Removing extra models.. {to_remove}")
         for o in to_remove:
             os.remove(o['path'])
-        self.top_model_paths = self.top_model_paths[:self.top_n]
+        self.top_model_paths = self.top_model_paths[len(self.top_model_paths)-self.top_n:]
+
+
+def load_model(model, checkpoint, optimizer=None, iteration=''):
+    # list of checkpoint files in the folder checkpoints
+    checkpoints = [os.path.join(checkpoint, f) for f in os.listdir(checkpoint) if f.endswith(f'{iteration}.pt')]
+    checkpoint = torch.load(checkpoints[-1])
+    model.load_state_dict(checkpoint['model'])
+    if optimizer is not None: optimizer.load_state_dict(checkpoint['optimizer'])
+