mirror of
https://github.com/zebrajr/faceswap.git
synced 2025-12-06 00:20:09 +01:00
lr-finder: Log error on NaN
This commit is contained in:
parent
0f27fe19cd
commit
b16b755d55
|
|
@ -132,7 +132,8 @@ class LearningRateFinder:
|
||||||
for idx in pbar:
|
for idx in pbar:
|
||||||
model_inputs, model_targets = self._feeder.get_batch()
|
model_inputs, model_targets = self._feeder.get_batch()
|
||||||
loss: list[float] = self._model.model.train_on_batch(model_inputs, y=model_targets)
|
loss: list[float] = self._model.model.train_on_batch(model_inputs, y=model_targets)
|
||||||
if np.isnan(loss[0]):
|
if any(np.isnan(x) for x in loss):
|
||||||
|
logger.warning("NaN detected! Exiting early")
|
||||||
break
|
break
|
||||||
self._on_batch_end(idx, loss[0])
|
self._on_batch_end(idx, loss[0])
|
||||||
self._update_description(pbar)
|
self._update_description(pbar)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user