From e02146943d3373020b7fa6e8acc31dc18b4201e4 Mon Sep 17 00:00:00 2001
From: Jonas-Jaeger <34666257+Jonas-Jaeger@users.noreply.github.com>
Date: Sun, 19 Apr 2020 05:07:15 +0200
Subject: [PATCH] Removed redundant computations in clip_gradients that slowed
 down the gradient clipping. (#1523)

Fixes #1522
---
 pytorch_lightning/trainer/training_tricks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_lightning/trainer/training_tricks.py b/pytorch_lightning/trainer/training_tricks.py
index f5523210ef..3364c9d305 100644
--- a/pytorch_lightning/trainer/training_tricks.py
+++ b/pytorch_lightning/trainer/training_tricks.py
@@ -40,7 +40,7 @@ class TrainerTrainingTricksMixin(ABC):
                 device = parameters[0].device
                 total_norm = torch.zeros([], device=device if parameters else None)
                 for p in parameters:
-                    param_norm = p.grad.data.norm(norm_type) ** norm_type
+                    param_norm = p.grad.data.pow(norm_type).sum()
                     total_norm.add_(param_norm)
                 total_norm = (total_norm ** (1. / norm_type))
             eps = EPSILON_FP16 if self.precision == 16 else EPSILON