save
This commit is contained in:
+39
-39
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import sys
|
||||
from configs.iccv.config_abl_pyr_32_64_25p import config
|
||||
from configs.iccv.config_abl_pyr_32_64_128_256_512 import config
|
||||
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
|
||||
import tensorflow as tf
|
||||
@@ -730,12 +730,12 @@ def generate_results():
|
||||
@tf.function
|
||||
def speed_step(levels_input, conds):
|
||||
|
||||
return model.sample([levels_input, conds], training=False)
|
||||
return model.sample([levels_input, conds])
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
init_wandb()
|
||||
# init_wandb()
|
||||
parser = argparse.ArgumentParser(description='')
|
||||
parser.add_argument('--name', type=str, default='lhunor')
|
||||
args = vars(parser.parse_args())
|
||||
@@ -846,52 +846,52 @@ if __name__ == '__main__':
|
||||
for batch in dataset_train:
|
||||
batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic']))
|
||||
|
||||
# levels_input, conds = group_inputs_for_levels(batch, config)
|
||||
levels_input, conds = group_inputs_for_levels(batch, config)
|
||||
|
||||
# start = time.perf_counter()
|
||||
# out = speed_step(levels_input, conds)
|
||||
# t = time.perf_counter() - start
|
||||
# print(t)
|
||||
# times.append(t)
|
||||
start = time.perf_counter()
|
||||
out = speed_step(levels_input, conds)
|
||||
t = time.perf_counter() - start
|
||||
print(t)
|
||||
times.append(t)
|
||||
|
||||
# if step > 200:
|
||||
# break
|
||||
if step > 200:
|
||||
break
|
||||
|
||||
|
||||
train_loss = train_step(batch)
|
||||
train_accumulator.update(train_loss)
|
||||
# train_loss = train_step(batch)
|
||||
# train_accumulator.update(train_loss)
|
||||
|
||||
if step % 25 == 0:
|
||||
data_valid = valid_iterator.get_next()
|
||||
data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
|
||||
valid_loss = valid_step(data_valid)
|
||||
valid_accumulator.update(valid_loss)
|
||||
# if step % 25 == 0:
|
||||
# data_valid = valid_iterator.get_next()
|
||||
# data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
|
||||
# valid_loss = valid_step(data_valid)
|
||||
# valid_accumulator.update(valid_loss)
|
||||
|
||||
if train_loss['train/euclid_error'] < 0.015:
|
||||
if balancing_activated_step == 0:
|
||||
balancing_activated_step = step
|
||||
balancing_coeff = balancing_coeff + 1 / 20000
|
||||
# if train_loss['train/euclid_error'] < 0.015:
|
||||
# if balancing_activated_step == 0:
|
||||
# balancing_activated_step = step
|
||||
# balancing_coeff = balancing_coeff + 1 / 20000
|
||||
|
||||
if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
|
||||
to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
|
||||
if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
|
||||
train_figs = plot_everything(batch, 'train')
|
||||
valid_figs = plot_everything(data_valid, 'valid')
|
||||
to_log = { **to_log, **train_figs, **valid_figs }
|
||||
if step == 0:
|
||||
to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
|
||||
# if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
|
||||
# to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
|
||||
# if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
|
||||
# train_figs = plot_everything(batch, 'train')
|
||||
# valid_figs = plot_everything(data_valid, 'valid')
|
||||
# to_log = { **to_log, **train_figs, **valid_figs }
|
||||
# if step == 0:
|
||||
# to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
|
||||
|
||||
to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
|
||||
# to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
|
||||
|
||||
wandb.log(to_log, step=step)
|
||||
train_accumulator.reset()
|
||||
valid_accumulator.reset()
|
||||
# wandb.log(to_log, step=step)
|
||||
# train_accumulator.reset()
|
||||
# valid_accumulator.reset()
|
||||
|
||||
if step % 5000 == 0:
|
||||
manager.save(checkpoint_number=step)
|
||||
# if step % 5000 == 0:
|
||||
# manager.save(checkpoint_number=step)
|
||||
|
||||
print(step)
|
||||
# print(step)
|
||||
|
||||
step += 1
|
||||
# print('mean:')
|
||||
# print(np.mean(times[1:]))
|
||||
print('mean:')
|
||||
print(np.mean(times[10:]))
|
||||
Reference in New Issue
Block a user