patch 8.1.0675: text property column in screen columns is not practical

Problem:    Text property column is screen columns is not practical.
Solution:   Use byte values for the column.

runtime/doc/eval.txt

@@ -1,4 +1,4 @@
-*eval.txt*	For Vim version 8.1.  Last change: 2018 Dec 28
+*eval.txt*	For Vim version 8.1.  Last change: 2019 Jan 01
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -6689,18 +6689,19 @@ prompt_setprompt({buf}, {text})				*prompt_setprompt()*
 <
 						*prop_add()* *E965*
 prop_add({lnum}, {col}, {props})
-		Attach a text property at position {lnum}, {col}.  Use one for
+		Attach a text property at position {lnum}, {col}.  {col} is
-		the first column.
+		counted in bytes, use one for the first column.
 		If {lnum} is invalid an error is given. *E966*
 		If {col} is invalid an error is given. *E964*
 
 		{props} is a dictionary with these fields:
-		   length	length of text in characters, can only be used
+		   length	length of text in bytes, can only be used
 				for a property that does not continue in
-				another line
+				another line; can be zero
-		   end_lnum	line number for end of text
+		   end_lnum	line number for the end of text
-		   end_col	last column of the text; not used when
+		   end_col	column just after the text; not used when "length"
-				"length" is present
+				is present; when {col} and "end_col" are equal
+				this is a zero-width text property
 		   bufnr	buffer to add the property to; when omitted
 		   		the current buffer is used
 		   id		user defined ID for the property; when omitted
@@ -6709,11 +6710,12 @@ prop_add({lnum}, {col}, {props})
 		All fields except "type" are optional.
 
 		It is an error when both "length" and "end_lnum" or "end_col"
-		are passed.  Either use "length" or "end_col" for a property
+		are given.  Either use "length" or "end_col" for a property
 		within one line, or use "end_lnum" and "end_col" for a
 		property that spans more than one line.
-		When neither "length" nor "end_col" are passed the property
+		When neither "length" nor "end_col" are given the property
-		will apply to one character.
+		will be zero-width.  That means it will not be highlighted but
+		will move with the text, as a kind of mark.
 		The property can end exactly at the last character of the
 		text, or just after it.  In the last case, if text is appended
 		to the line, the text property size will increase, also when

src/proto/textprop.pro

@@ -13,4 +13,5 @@ void f_prop_type_get(typval_T *argvars, typval_T *rettv);
 void f_prop_type_list(typval_T *argvars, typval_T *rettv);
 void clear_global_prop_types(void);
 void clear_buf_prop_types(buf_T *buf);
+void adjust_prop_columns(linenr_T lnum, colnr_T col, int bytes_added);
 /* vim: set ft=c : */

src/structs.h

@@ -705,7 +705,7 @@ typedef struct memline
  */
 typedef struct textprop_S
 {
-    colnr_T	tp_col;		// start column (one based)
+    colnr_T	tp_col;		// start column (one based, in bytes)
     colnr_T	tp_len;		// length in bytes
     int		tp_id;		// identifier
     int		tp_type;	// property type

src/testdir/dumps/Test_textprop_01.dump

@@ -1,20 +1,6 @@
 | +0#af5f00255#ffffff0@1|1| >O+0#0000000&|n|e| +0&#ffff4012|t|w|o| +0&#ffffff0@63
-| +0#af5f00255&@1|2| |N+0#0000000#ffff4012|u|m|b|e|r| |1+0#4040ff13&|2|3| +0#0000000&|a|n|d| |t|h|e|n| |4+0#4040ff13&|5|6|7|.+0#0000000&| +0&#ffffff0@45
+| +0#af5f00255&@1|2| |N+0#0000000#ffff4012|u|m|b|é|r| |1+0#4040ff13&|2|3| +0#0000000&|ä|n|d| |t|h|œ|n| |4+0#4040ff13&|¾|7|.+0#0000000&| +0&#ffffff0@46
-| +0#af5f00255&@1|3| |T+0#0000000#ffff4012|h|r|e+0&#ffffff0@1| @65
+| +0#af5f00255&@1|3| |T+0#0000000#ffff4012|h|r+0&#ffffff0|e@1| @65
 |~+0#4040ff13&| @73
 |~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
-|~| @73
 | +0#0000000&@56|1|,|1| @10|A|l@1| 

src/testdir/test_textprop.vim

@@ -7,6 +7,8 @@ endif
 
 source screendump.vim
 
+" test length zero
+
 func Test_proptype_global()
   call prop_type_add('comment', {'highlight': 'Directory', 'priority': 123, 'start_incl': 1, 'end_incl': 1})
   let proptypes = prop_type_list()
@@ -112,6 +114,12 @@ func Test_prop_add()
   1del
   call assert_equal(s:expected_props, prop_list(1))
 
+  " Prop without length or end column is zero length
+  call prop_clear(1)
+  call prop_add(1, 5, {'type': 'two'})
+  let expected = [{'col': 5, 'length': 0, 'type': 'two', 'id': 0, 'start': 1, 'end': 1}]
+  call assert_equal(expected, prop_list(1))
+
   call DeletePropTypes()
   bwipe!
 endfunc
@@ -220,7 +228,7 @@ func Test_prop_multiline()
   call assert_equal([expect1], prop_list(1))
   let expect2 = {'col': 1, 'length': 10, 'type': 'comment', 'start': 0, 'end': 0, 'id': 0}
   call assert_equal([expect2], prop_list(2))
-  let expect3 = {'col': 1, 'length': 5, 'type': 'comment', 'start': 0, 'end': 1, 'id': 0}
+  let expect3 = {'col': 1, 'length': 4, 'type': 'comment', 'start': 0, 'end': 1, 'id': 0}
   call assert_equal([expect3], prop_list(3))
   call prop_clear(1, 3)
 
@@ -236,30 +244,31 @@ func Test_prop_multiline()
 
   bwipe!
 
-  " Test deleting the first line with a prop.
+  " Test deleting the first line of a multi-line prop.
   call Setup_three_line_prop()
+  let expect_short = {'col': 2, 'length': 1, 'type': 'comment', 'start': 1, 'end': 1, 'id': 0}
+  call assert_equal([expect_short], prop_list(1))
   let expect2 = {'col': 4, 'length': 4, 'type': 'comment', 'start': 1, 'end': 0, 'id': 0}
   call assert_equal([expect2], prop_list(2))
   2del
-  let expect_short = {'col': 2, 'length': 1, 'type': 'comment', 'start': 1, 'end': 1, 'id': 0}
   call assert_equal([expect_short], prop_list(1))
   let expect2 = {'col': 1, 'length': 6, 'type': 'comment', 'start': 1, 'end': 0, 'id': 0}
   call assert_equal([expect2], prop_list(2))
   bwipe!
 
-  " Test deleting the last line with a prop.
+  " Test deleting the last line of a multi-line prop.
   call Setup_three_line_prop()
   let expect3 = {'col': 1, 'length': 6, 'type': 'comment', 'start': 0, 'end': 0, 'id': 0}
   call assert_equal([expect3], prop_list(3))
-  let expect4 = {'col': 1, 'length': 5, 'type': 'comment', 'start': 0, 'end': 1, 'id': 0}
+  let expect4 = {'col': 1, 'length': 4, 'type': 'comment', 'start': 0, 'end': 1, 'id': 0}
   call assert_equal([expect4], prop_list(4))
   4del
-  let expect3 = {'col': 1, 'length': 6, 'type': 'comment', 'start': 0, 'end': 1, 'id': 0}
+  let expect3.end = 1
   call assert_equal([expect3], prop_list(3))
   call assert_equal([expect_short], prop_list(4))
   bwipe!
 
-  " Test appending a line below the text prop start.
+  " Test appending a line below the multi-line text prop start.
   call Setup_three_line_prop()
   let expect2 = {'col': 4, 'length': 4, 'type': 'comment', 'start': 1, 'end': 0, 'id': 0}
   call assert_equal([expect2], prop_list(2))
@@ -287,22 +296,23 @@ endfunc
 
 " screenshot test with textprop highlighting
 funct Test_textprop_screenshots()
-  if !CanRunVimInTerminal()
+  if !CanRunVimInTerminal() || &encoding != 'utf-8'
     return
   endif
   call writefile([
-	\ "call setline(1, ['One two', 'Number 123 and then 4567.', 'Three'])",
+	\ "call setline(1, ['One two', 'Numbér 123 änd thœn 4¾7.', 'Three'])",
 	\ "hi NumberProp ctermfg=blue",
 	\ "hi LongProp ctermbg=yellow",
 	\ "call prop_type_add('number', {'highlight': 'NumberProp'})",
 	\ "call prop_type_add('long', {'highlight': 'LongProp'})",
 	\ "call prop_add(1, 4, {'end_lnum': 3, 'end_col': 3, 'type': 'long'})",
-	\ "call prop_add(2, 8, {'length': 3, 'type': 'number'})",
+	\ "call prop_add(2, 9, {'length': 3, 'type': 'number'})",
-	\ "call prop_add(2, 21, {'length': 4, 'type': 'number'})",
+	\ "call prop_add(2, 24, {'length': 4, 'type': 'number'})",
 	\ "set number",
+	\ "hi clear SpellBad",
 	\ "set spell",
 	\], 'XtestProp')
-  let buf = RunVimInTerminal('-S XtestProp', {})
+  let buf = RunVimInTerminal('-S XtestProp', {'rows': 6})
   call VerifyScreenDump(buf, 'Test_textprop_01', {})
 
   " clean up

src/textprop.c

@@ -17,8 +17,8 @@
  * Text properties have a type, which can be used to specify highlighting.
  *
  * TODO:
- * - Perhaps we only need TP_FLAG_CONT_NEXT ?
+ * - Adjust text property column and length when text is inserted/deleted.
- * - Adjust text property column and length when text is inserted/deleted
+ * - Perhaps we only need TP_FLAG_CONT_NEXT and can drop TP_FLAG_CONT_PREV?
  * - Add an arrray for global_proptypes, to quickly lookup a prop type by ID
  * - Add an arrray for b_proptypes, to quickly lookup a prop type by ID
  * - Checking the text length to detect text properties is slow.  Use a flag in
@@ -198,12 +198,12 @@ f_prop_add(typval_T *argvars, typval_T *rettv UNUSED)
     {
 	long length = dict_get_number(dict, (char_u *)"length");
 
-	if (length < 1 || end_lnum > start_lnum)
+	if (length < 0 || end_lnum > start_lnum)
 	{
 	    EMSG2(_(e_invargval), "length");
 	    return;
 	}
-	end_col = start_col + length - 1;
+	end_col = start_col + length;
     }
     else if (dict_find(dict, (char_u *)"end_col", -1) != NULL)
     {
@@ -260,13 +260,13 @@ f_prop_add(typval_T *argvars, typval_T *rettv UNUSED)
 	}
 
 	if (lnum == end_lnum)
-	    length = end_col - col + 1;
+	    length = end_col - col;
 	else
 	    length = textlen - col + 1;
 	if (length > (long)textlen)
 	    length = textlen;	// can include the end-of-line
-	if (length < 1)
+	if (length < 0)
-	    length = 1;
+	    length = 0;		// zero-width property
 
 	// Allocate the new line with space for the new proprety.
 	newtext = alloc(buf->b_ml.ml_line_len + sizeof(textprop_T));
@@ -912,4 +912,14 @@ clear_buf_prop_types(buf_T *buf)
     buf->b_proptypes = NULL;
 }
 
+/*
+ * Adjust the columns of text properties in line "lnum" after position "col" to
+ * shift by "bytes_added" (can be negative).
+ */
+    void
+adjust_prop_columns(linenr_T lnum UNUSED, colnr_T col UNUSED, int bytes_added UNUSED)
+{
+    // TODO
+}
+
 #endif // FEAT_TEXT_PROP

src/version.c

@@ -799,6 +799,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    675,
 /**/
     674,
 /**/