patch 8.1.0691: text properties are not adjusted for :substitute

Problem:    Text properties are not adjusted for :substitute.
Solution:   Adjust text properties as well as possible.

src/ex_cmds.c

@@ -5628,9 +5628,19 @@ do_sub(exarg_T *eap)
 		 * - original text up to match
 		 * - length of substituted part
 		 * - original text after match
+		 * Adjust text properties here, since we have all information
+		 * needed.
 		 */
 		if (nmatch == 1)
+		{
 		    p1 = sub_firstline;
+#ifdef FEAT_TEXT_PROP
+		    if (curbuf->b_has_textprop)
+			adjust_prop_columns(lnum, regmatch.startpos[0].col,
+			      sublen - 1 - (regmatch.endpos[0].col
+						  - regmatch.startpos[0].col));
+#endif
+		}
 		else
 		{
 		    p1 = ml_get(sub_firstlnum + nmatch - 1);
@@ -5732,11 +5742,12 @@ do_sub(exarg_T *eap)
 			STRMOVE(p1, p1 + 1);
 		    else if (*p1 == CAR)
 		    {
-			if (u_inssub(lnum) == OK)   /* prepare for undo */
+			if (u_inssub(lnum) == OK)   // prepare for undo
 			{
-			    *p1 = NUL;		    /* truncate up to the CR */
+			    colnr_T	plen = (colnr_T)(p1 - new_start + 1);
-			    ml_append(lnum - 1, new_start,
+
-					(colnr_T)(p1 - new_start + 1), FALSE);
+			    *p1 = NUL;		    // truncate up to the CR
+			    ml_append(lnum - 1, new_start, plen, FALSE);
 			    mark_adjust(lnum + 1, (linenr_T)MAXLNUM, 1L, 0L);
 			    if (subflags.do_ask)
 				appended_lines(lnum - 1, 1L);
@@ -5746,13 +5757,16 @@ do_sub(exarg_T *eap)
 				    first_line = lnum;
 				last_line = lnum + 1;
 			    }
-			    /* All line numbers increase. */
+#ifdef FEAT_TEXT_PROP
+			    adjust_props_for_split(lnum, plen, 1);
+#endif
+			    // all line numbers increase
 			    ++sub_firstlnum;
 			    ++lnum;
 			    ++line2;
-			    /* move the cursor to the new line, like Vi */
+			    // move the cursor to the new line, like Vi
 			    ++curwin->w_cursor.lnum;
-			    /* copy the rest */
+			    // copy the rest
 			    STRMOVE(new_start, p1 + 1);
 			    p1 = new_start - 1;
 			}

src/proto/textprop.pro

@@ -14,4 +14,5 @@ void f_prop_type_list(typval_T *argvars, typval_T *rettv);
 void clear_global_prop_types(void);
 void clear_buf_prop_types(buf_T *buf);
 void adjust_prop_columns(linenr_T lnum, colnr_T col, int bytes_added);
+void adjust_props_for_split(linenr_T lnum, int kept, int deleted);
 /* vim: set ft=c : */

src/testdir/test_textprop.vim

@@ -89,30 +89,34 @@ func SetupPropsInFirstLine()
   call setline(1, 'one two three')
   call prop_add(1, 1, {'length': 3, 'id': 11, 'type': 'one'})
   call prop_add(1, 5, {'length': 3, 'id': 12, 'type': 'two'})
-  call prop_add(1, 8, {'length': 5, 'id': 13, 'type': 'three'})
+  call prop_add(1, 9, {'length': 5, 'id': 13, 'type': 'three'})
   call prop_add(1, 1, {'length': 13, 'id': 14, 'type': 'whole'})
 endfunc
 
-let s:expected_props = [{'col': 1, 'length': 13, 'id': 14, 'type': 'whole', 'start': 1, 'end': 1},
+func Get_expected_props()
+  return [
+      \ {'col': 1, 'length': 13, 'id': 14, 'type': 'whole', 'start': 1, 'end': 1},
       \ {'col': 1, 'length': 3, 'id': 11, 'type': 'one', 'start': 1, 'end': 1},
       \ {'col': 5, 'length': 3, 'id': 12, 'type': 'two', 'start': 1, 'end': 1},
-      \ {'col': 8, 'length': 5, 'id': 13, 'type': 'three', 'start': 1, 'end': 1},
+      \ {'col': 9, 'length': 5, 'id': 13, 'type': 'three', 'start': 1, 'end': 1},
       \ ]
+endfunc
 
 func Test_prop_add()
   new
   call AddPropTypes()
   call SetupPropsInFirstLine()
-  call assert_equal(s:expected_props, prop_list(1))
+  let expected_props = Get_expected_props()
+  call assert_equal(expected_props, prop_list(1))
   call assert_fails("call prop_add(10, 1, {'length': 1, 'id': 14, 'type': 'whole'})", 'E966:')
   call assert_fails("call prop_add(1, 22, {'length': 1, 'id': 14, 'type': 'whole'})", 'E964:')
  
   " Insert a line above, text props must still be there.
   call append(0, 'empty')
-  call assert_equal(s:expected_props, prop_list(2))
+  call assert_equal(expected_props, prop_list(2))
   " Delete a line above, text props must still be there.
   1del
-  call assert_equal(s:expected_props, prop_list(1))
+  call assert_equal(expected_props, prop_list(1))
 
   " Prop without length or end column is zero length
   call prop_clear(1)
@@ -128,7 +132,7 @@ func Test_prop_remove()
   new
   call AddPropTypes()
   call SetupPropsInFirstLine()
-  let props = deepcopy(s:expected_props)
+  let props = Get_expected_props()
   call assert_equal(props, prop_list(1))
 
   " remove by id
@@ -236,7 +240,7 @@ func Test_prop_clear()
   new
   call AddPropTypes()
   call SetupPropsInFirstLine()
-  call assert_equal(s:expected_props, prop_list(1))
+  call assert_equal(Get_expected_props(), prop_list(1))
 
   call prop_clear(1)
   call assert_equal([], prop_list(1))
@@ -251,7 +255,7 @@ func Test_prop_clear_buf()
   call SetupPropsInFirstLine()
   let bufnr = bufnr('')
   wincmd w
-  call assert_equal(s:expected_props, prop_list(1, {'bufnr': bufnr}))
+  call assert_equal(Get_expected_props(), prop_list(1, {'bufnr': bufnr}))
 
   call prop_clear(1, 1, {'bufnr': bufnr})
   call assert_equal([], prop_list(1, {'bufnr': bufnr}))
@@ -265,7 +269,7 @@ func Test_prop_setline()
   new
   call AddPropTypes()
   call SetupPropsInFirstLine()
-  call assert_equal(s:expected_props, prop_list(1))
+  call assert_equal(Get_expected_props(), prop_list(1))
 
   call setline(1, 'foobar')
   call assert_equal([], prop_list(1))
@@ -280,7 +284,7 @@ func Test_prop_setbufline()
   call SetupPropsInFirstLine()
   let bufnr = bufnr('')
   wincmd w
-  call assert_equal(s:expected_props, prop_list(1, {'bufnr': bufnr}))
+  call assert_equal(Get_expected_props(), prop_list(1, {'bufnr': bufnr}))
 
   call setbufline(bufnr, 1, 'foobar')
   call assert_equal([], prop_list(1, {'bufnr': bufnr}))
@@ -290,6 +294,54 @@ func Test_prop_setbufline()
   bwipe!
 endfunc
 
+func Test_prop_substitute()
+  new
+  " Set first line to 'one two three'
+  call AddPropTypes()
+  call SetupPropsInFirstLine()
+  let expected_props = Get_expected_props()
+  call assert_equal(expected_props, prop_list(1))
+
+  " Change "n" in "one" to XX: 'oXXe two three'
+  s/n/XX/
+  let expected_props[0].length += 1
+  let expected_props[1].length += 1
+  let expected_props[2].col += 1
+  let expected_props[3].col += 1
+  call assert_equal(expected_props, prop_list(1))
+
+  " Delete "t" in "two" and "three" to XX: 'oXXe wo hree'
+  s/t//g
+  let expected_props[0].length -= 2
+  let expected_props[2].length -= 1
+  let expected_props[3].length -= 1
+  let expected_props[3].col -= 1
+  call assert_equal(expected_props, prop_list(1))
+
+  " Split the line by changing w to line break: 'oXXe ', 'o hree'
+  " The long prop is split and spans both lines.
+  " The props on "two" and "three" move to the next line.
+  s/w/\r/
+  let new_props = [
+	\ copy(expected_props[0]),
+	\ copy(expected_props[2]),
+	\ copy(expected_props[3]),
+	\ ]
+  let expected_props[0].length = 5
+  unlet expected_props[3]
+  unlet expected_props[2]
+  call assert_equal(expected_props, prop_list(1))
+
+  let new_props[0].length = 6
+  let new_props[1].col = 1
+  let new_props[1].length = 1
+  let new_props[2].col = 3
+  call assert_equal(new_props, prop_list(2))
+
+  call DeletePropTypes()
+  bwipe!
+endfunc
+
 " Setup a three line prop in lines 2 - 4.
 " Add short props in line 1 and 5.
 func Setup_three_line_prop()

src/textprop.c

@@ -18,6 +18,8 @@
  *
  * TODO:
  * - Adjust text property column and length when text is inserted/deleted.
+ *   -> a :substitute with a multi-line match
+ *   -> search for changed_bytes() from ex_cmds.c
  * - Perhaps we only need TP_FLAG_CONT_NEXT and can drop TP_FLAG_CONT_PREV?
  * - Add an arrray for global_proptypes, to quickly lookup a prop type by ID
  * - Add an arrray for b_proptypes, to quickly lookup a prop type by ID
@@ -346,6 +348,34 @@ get_text_props(buf_T *buf, linenr_T lnum, char_u **props, int will_change)
     return (int)(proplen / sizeof(textprop_T));
 }
 
+/*
+ * Set the text properties for line "lnum" to "props" with length "len".
+ * If "len" is zero text properties are removed, "props" is not used.
+ * Any existing text properties are dropped.
+ * Only works for the current buffer.
+ */
+    static void
+set_text_props(linenr_T lnum, char_u *props, int len)
+{
+    char_u *text;
+    char_u *newtext;
+    size_t textlen;
+
+    text = ml_get(lnum);
+    textlen = STRLEN(text) + 1;
+    newtext = alloc(textlen + len);
+    if (newtext == NULL)
+	return;
+    mch_memmove(newtext, text, textlen);
+    if (len > 0)
+	mch_memmove(newtext + textlen, props, len);
+    if (curbuf->b_ml.ml_flags & ML_LINE_DIRTY)
+	vim_free(curbuf->b_ml.ml_line_ptr);
+    curbuf->b_ml.ml_line_ptr = newtext;
+    curbuf->b_ml.ml_line_len = textlen + len;
+    curbuf->b_ml.ml_flags |= ML_LINE_DIRTY;
+}
+
     static proptype_T *
 find_type_by_id(hashtab_T *ht, int id)
 {
@@ -976,4 +1006,69 @@ adjust_prop_columns(
     }
 }
 
+/*
+ * Adjust text properties for a line that was split in two.
+ * "lnum" is the newly inserted line.  The text properties are now on the line
+ * below it.  "kept" is the number of bytes kept in the first line, while
+ * "deleted" is the number of bytes deleted.
+ */
+    void
+adjust_props_for_split(linenr_T lnum, int kept, int deleted)
+{
+    char_u	*props;
+    int		count;
+    garray_T    prevprop;
+    garray_T    nextprop;
+    int		i;
+    int		skipped = kept + deleted;
+
+    if (!curbuf->b_has_textprop)
+	return;
+    count = get_text_props(curbuf, lnum + 1, &props, FALSE);
+    ga_init2(&prevprop, sizeof(textprop_T), 10);
+    ga_init2(&nextprop, sizeof(textprop_T), 10);
+
+    // Get the text properties, which are at "lnum + 1".
+    // Keep the relevant ones in the first line, reducing the length if needed.
+    // Copy the ones that include the split to the second line.
+    // Move the ones after the split to the second line.
+    for (i = 0; i < count; ++i)
+    {
+	textprop_T  prop;
+	textprop_T *p;
+
+	// copy the prop to an aligned structure
+	mch_memmove(&prop, props + i * sizeof(textprop_T), sizeof(textprop_T));
+
+	if (prop.tp_col < kept && ga_grow(&prevprop, 1) == OK)
+	{
+	    p = ((textprop_T *)prevprop.ga_data) + prevprop.ga_len;
+	    *p = prop;
+	    if (p->tp_col + p->tp_len >= kept)
+		p->tp_len = kept - p->tp_col;
+	    ++prevprop.ga_len;
+	}
+
+	if (prop.tp_col + prop.tp_len >= skipped && ga_grow(&nextprop, 1) == OK)
+	{
+	    p = ((textprop_T *)nextprop.ga_data) + nextprop.ga_len;
+	    *p = prop;
+	    if (p->tp_col > skipped)
+		p->tp_col -= skipped - 1;
+	    else
+	    {
+		p->tp_len -= skipped - p->tp_col;
+		p->tp_col = 1;
+	    }
+	    ++nextprop.ga_len;
+	}
+    }
+
+    set_text_props(lnum, prevprop.ga_data, prevprop.ga_len * sizeof(textprop_T));
+    ga_clear(&prevprop);
+
+    set_text_props(lnum + 1, nextprop.ga_data, nextprop.ga_len * sizeof(textprop_T));
+    ga_clear(&nextprop);
+}
+
 #endif // FEAT_TEXT_PROP

src/version.c

@@ -799,6 +799,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    691,
 /**/
     690,
 /**/