diff --git a/src/os_unix.c b/src/os_unix.c
index 91e302a9e6..e87331defc 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -1671,6 +1671,32 @@ x_error_check(Display *dpy UNUSED, XErrorEvent *error_event UNUSED)
     return 0;
 }
 
+/*
+ * Return TRUE when connection to the X server is desired.
+ */
+    static int
+x_connect_to_server(void)
+{
+    // No point in connecting if we are exiting or dying.
+    if (exiting || v_dying)
+	return FALSE;
+
+#if defined(FEAT_CLIENTSERVER)
+    if (x_force_connect)
+	return TRUE;
+#endif
+    if (x_no_connect)
+	return FALSE;
+
+    /* Check for a match with "exclude:" from 'clipboard'. */
+    if (clip_exclude_prog != NULL)
+    {
+	if (vim_regexec_prog(&clip_exclude_prog, FALSE, T_NAME, (colnr_T)0))
+	    return FALSE;
+    }
+    return TRUE;
+}
+
 #if defined(FEAT_X11) && defined(FEAT_XCLIPBOARD)
 # if defined(HAVE_SETJMP_H)
 /*
@@ -1716,7 +1742,8 @@ x_IOerror_handler(Display *dpy UNUSED)
     static void
 may_restore_clipboard(void)
 {
-    if (xterm_dpy_retry_count > 0)
+    // Only try restoring if we want the connection.
+    if (x_connect_to_server() && xterm_dpy_retry_count > 0)
     {
 	--xterm_dpy_retry_count;
 
@@ -1736,28 +1763,6 @@ may_restore_clipboard(void)
 }
 #endif
 
-/*
- * Return TRUE when connection to the X server is desired.
- */
-    static int
-x_connect_to_server(void)
-{
-#if defined(FEAT_CLIENTSERVER)
-    if (x_force_connect)
-	return TRUE;
-#endif
-    if (x_no_connect)
-	return FALSE;
-
-    /* Check for a match with "exclude:" from 'clipboard'. */
-    if (clip_exclude_prog != NULL)
-    {
-	if (vim_regexec_prog(&clip_exclude_prog, FALSE, T_NAME, (colnr_T)0))
-	    return FALSE;
-    }
-    return TRUE;
-}
-
 /*
  * Test if "dpy" and x11_window are valid by getting the window title.
  * I don't actually want it yet, so there may be a simpler call to use, but
diff --git a/src/version.c b/src/version.c
index 42ae469ea5..a49edb709c 100644
--- a/src/version.c
+++ b/src/version.c
@@ -799,6 +799,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    656,
 /**/
     655,
 /**/