This commit is contained in:
2023-03-07 22:02:55 +00:00
parent cbecf668df
commit cb76d36c4f
20 changed files with 626 additions and 168 deletions
+39 -39
View File
@@ -1,6 +1,6 @@
import os
import sys
from configs.iccv.config_abl_pyr_32_64_25p import config
from configs.iccv.config_abl_pyr_32_64_128_256_512 import config
os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu']
import tensorflow as tf
@@ -730,12 +730,12 @@ def generate_results():
@tf.function
def speed_step(levels_input, conds):
return model.sample([levels_input, conds], training=False)
return model.sample([levels_input, conds])
if __name__ == '__main__':
init_wandb()
# init_wandb()
parser = argparse.ArgumentParser(description='')
parser.add_argument('--name', type=str, default='lhunor')
args = vars(parser.parse_args())
@@ -846,52 +846,52 @@ if __name__ == '__main__':
for batch in dataset_train:
batch = static_dataset_train.expand_batch(batch_to_dict(batch, config['signature_dynamic']))
# levels_input, conds = group_inputs_for_levels(batch, config)
levels_input, conds = group_inputs_for_levels(batch, config)
# start = time.perf_counter()
# out = speed_step(levels_input, conds)
# t = time.perf_counter() - start
# print(t)
# times.append(t)
start = time.perf_counter()
out = speed_step(levels_input, conds)
t = time.perf_counter() - start
print(t)
times.append(t)
# if step > 200:
# break
if step > 200:
break
train_loss = train_step(batch)
train_accumulator.update(train_loss)
# train_loss = train_step(batch)
# train_accumulator.update(train_loss)
if step % 25 == 0:
data_valid = valid_iterator.get_next()
data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
valid_loss = valid_step(data_valid)
valid_accumulator.update(valid_loss)
# if step % 25 == 0:
# data_valid = valid_iterator.get_next()
# data_valid = static_dataset_valid.expand_batch(batch_to_dict(data_valid, config['signature_dynamic']))
# valid_loss = valid_step(data_valid)
# valid_accumulator.update(valid_loss)
if train_loss['train/euclid_error'] < 0.015:
if balancing_activated_step == 0:
balancing_activated_step = step
balancing_coeff = balancing_coeff + 1 / 20000
# if train_loss['train/euclid_error'] < 0.015:
# if balancing_activated_step == 0:
# balancing_activated_step = step
# balancing_coeff = balancing_coeff + 1 / 20000
if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
train_figs = plot_everything(batch, 'train')
valid_figs = plot_everything(data_valid, 'valid')
to_log = { **to_log, **train_figs, **valid_figs }
if step == 0:
to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
# if step % 1000 == 0 or (step < 1000 and step % 100 == 0):
# to_log = dict({ **train_loss, **valid_loss, **{ 'current_step': step } })
# if step % 5000 == 0 or (step < 5000 and step % 1000 == 0):
# train_figs = plot_everything(batch, 'train')
# valid_figs = plot_everything(data_valid, 'valid')
# to_log = { **to_log, **train_figs, **valid_figs }
# if step == 0:
# to_log = { **to_log, **{ 'model/summary': wandb.Html(get_summary_html(model)) } }
to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
# to_log = { **to_log, **{ 'train/balancing_coeff': balancing_coeff, 'train/gamma': gamma } }
wandb.log(to_log, step=step)
train_accumulator.reset()
valid_accumulator.reset()
# wandb.log(to_log, step=step)
# train_accumulator.reset()
# valid_accumulator.reset()
if step % 5000 == 0:
manager.save(checkpoint_number=step)
# if step % 5000 == 0:
# manager.save(checkpoint_number=step)
print(step)
# print(step)
step += 1
# print('mean:')
# print(np.mean(times[1:]))
print('mean:')
print(np.mean(times[10:]))