patch 9.1.0514: Vim9: issue with comparing objects recursively

Problem:  Vim9: issue with comparing objects recursively
          (Yinzuo Jiang)
Solution: only set recursive == TRUE, when called from tv_equal(), not
          from typeval_compare_object(), refactor code into object_equal()
          function (LemonBoy)

The recursive flag in tv_equal should be set only when the caller is
tv_equal, meaning that the comparison depth is > 1. The comparison
predicates for other object types are all following this rule, except
for the object one, and that may cause some weird issues like causing
the max depth limit not to be initialized in some cases.

closes: #15076

Signed-off-by: LemonBoy <thatlemon@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

src/proto/vim9class.pro

@@ -40,6 +40,7 @@ int is_class_name(char_u *name, typval_T *rettv);
 void protected_method_access_errmsg(char_u *method_name);
 int object_empty(object_T *obj);
 int object_len(object_T *obj);
+int object_equal(object_T *o1, object_T *o2, int ic, int recursive);
 char_u *object2string(object_T *obj, char_u *numbuf, int copyID, int echo_style, int restore_copyID, int composite_val);
 int class_instance_of(class_T *cl, class_T *other_cl);
 void f_instanceof(typval_T *argvars, typval_T *rettv);

src/typval.c

@@ -1742,14 +1742,6 @@ typval_compare_object(
 	return OK;
     }
 
-    class_T *cl1 = tv1->vval.v_object->obj_class;
-    class_T *cl2 = tv2->vval.v_object->obj_class;
-    if (cl1 != cl2 || cl1 == NULL || cl2 == NULL)
-    {
-	*res = !res_match;
-	return OK;
-    }
-
     object_T *obj1 = tv1->vval.v_object;
     object_T *obj2 = tv2->vval.v_object;
     if (type == EXPR_IS || type == EXPR_ISNOT)
@@ -1758,14 +1750,7 @@ typval_compare_object(
 	return OK;
     }
 
-    for (int i = 0; i < cl1->class_obj_member_count; ++i)
-	if (!tv_equal((typval_T *)(obj1 + 1) + i,
-				 (typval_T *)(obj2 + 1) + i, ic, TRUE))
-	{
-	    *res = !res_match;
-	    return OK;
-	}
-    *res = res_match;
+    *res = object_equal(obj1, obj2, ic, FALSE) ? res_match : !res_match;
     return OK;
 }
 
@@ -2115,7 +2100,7 @@ tv_equal(
 
 	case VAR_OBJECT:
 	    ++recursive_cnt;
-	    (void)typval_compare_object(tv1, tv2, EXPR_EQUAL, ic, &r);
+	    r = object_equal(tv1->vval.v_object, tv2->vval.v_object, ic, TRUE);
 	    --recursive_cnt;
 	    return r;
 

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    514,
 /**/
     513,
 /**/

src/vim9class.c

@@ -3841,6 +3841,34 @@ object_len(object_T *obj)
     return tv_to_number(&rettv);
 }
 
+/*
+ * Return TRUE when two objects have exactly the same values.
+ */
+    int
+object_equal(
+	object_T *o1,
+	object_T *o2,
+	int		ic,	// ignore case for strings
+	int		recursive)  // TRUE when used recursively
+{
+    class_T *cl1, *cl2;
+
+    if (o1 == o2)
+	return TRUE;
+
+    cl1 = o1->obj_class;
+    cl2 = o2->obj_class;
+
+    if (cl1 != cl2 || cl1 == NULL || cl2 == NULL)
+	return FALSE;
+
+    for (int i = 0; i < cl1->class_obj_member_count; ++i)
+	if (!tv_equal((typval_T *)(o1 + 1) + i, (typval_T *)(o2 + 1) + i, ic, recursive))
+	    return FALSE;
+
+    return TRUE;
+}
+
 /*
  * Return a textual representation of object "obj"
  */