diff --git a/spacy/training/loggers.py b/spacy/training/loggers.py
index f0ca7064a..467f1e36b 100644
--- a/spacy/training/loggers.py
+++ b/spacy/training/loggers.py
@@ -41,19 +41,10 @@ def console_logger(progress_bar: bool = False):
                 if progress is not None:
                     progress.update(1)
                 return
-            try:
-                losses = [
-                    "{0:.2f}".format(float(info["losses"][pipe_name]))
-                    for pipe_name in logged_pipes
-                ]
-            except KeyError as e:
-                raise KeyError(
-                    Errors.E983.format(
-                        dict="scores (losses)",
-                        key=str(e),
-                        keys=list(info["losses"].keys()),
-                    )
-                ) from None
+            losses = [
+                "{0:.2f}".format(float(info["losses"][pipe_name]))
+                for pipe_name in logged_pipes if pipe_name in info["losses"]
+            ]
 
             scores = []
             for col in score_cols:
diff --git a/spacy/training/loop.py b/spacy/training/loop.py
index 0d4414964..8f0aea6d4 100644
--- a/spacy/training/loop.py
+++ b/spacy/training/loop.py
@@ -184,7 +184,7 @@ def train_while_improving(
                 and hasattr(proc, "model")
                 and proc.model not in (True, False, None)
             ):
-                proc.model.finish_update(optimizer)
+                proc.finish_update(optimizer)
         optimizer.step_schedules()
         if not (step % eval_frequency):
             if optimizer.averages:
@@ -287,7 +287,8 @@ def update_meta(
         if metric is not None:
             nlp.meta["performance"][metric] = info["other_scores"].get(metric, 0.0)
     for pipe_name in nlp.pipe_names:
-        nlp.meta["performance"][f"{pipe_name}_loss"] = info["losses"][pipe_name]
+        if pipe_name in info["losses"]:
+            nlp.meta["performance"][f"{pipe_name}_loss"] = info["losses"][pipe_name]
 
 
 def create_before_to_disk_callback(