From f5040b25db3e6e20dd58e439427353a488a341e4 Mon Sep 17 00:00:00 2001 From: Kang Sheng Date: Mon, 13 Jan 2025 19:33:44 +0800 Subject: [PATCH] add an extra validation that loss should not vary too much --- tests/trainer/test_trainer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 3b8e0b02d9ad..b4622bb247f8 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -857,6 +857,13 @@ def tokenize_function(examples): # max diff broken should be very off self.assertGreater(max(diff_broken), 2, f"Difference {max(diff_broken)} is not greater than 2") + loss_base = sum(base_loss_callback.losses) + loss_broken = sum(broken_loss_callback.losses) + + # mean/sum loss should not vary too much. + relative_diff = abs(loss_base - loss_broken) / max(loss_base, loss_broken) + self.assertLess(relative_diff, 0.1, f"Relative difference {relative_diff} is not within 0.1") + @slow def test_gradient_accumulation_loss_alignment_with_loss_func(self): set_seed(42)