patch 9.1.1077: included syntax items do not understand contains=TOP

Problem:  Syntax engine interpreted contains=TOP as matching nothing
          inside included files, since :syn-include forces HL_CONTAINED
          on for every included item. After 8.2.2761, interprets
          contains=TOP as contains=@INCLUDED, which is also not correct
          since it doesn't respect exclusions, and doesn't work if there
          is no @INCLUDED cluster.
Solution: revert patch 8.2.2761, instead track groups that have had
          HL_CONTAINED forced, and interpret contains=TOP and
          contains=CONTAINED using this. (Theodore Dubois)

fixes: #11277
closes: #16571

Signed-off-by: Theodore Dubois <tblodt@icloud.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

src/syntax.c

@@ -299,7 +299,7 @@ static void update_si_attr(int idx);
 static void check_keepend(void);
 static void update_si_end(stateitem_T *sip, int startcol, int force);
 static short *copy_id_list(short *list);
-static int in_id_list(stateitem_T *item, short *cont_list, struct sp_syn *ssp, int contained);
+static int in_id_list(stateitem_T *item, short *cont_list, struct sp_syn *ssp, int flags);
 static int push_current_state(int idx);
 static void pop_current_state(void);
 #ifdef FEAT_PROFILE
@@ -1943,7 +1943,7 @@ syn_current_attr(
 					? !(spp->sp_flags & HL_CONTAINED)
 					: in_id_list(cur_si,
 					    cur_si->si_cont_list, &spp->sp_syn,
-					    spp->sp_flags & HL_CONTAINED))))
+					    spp->sp_flags))))
 			{
 			    int r;
 
@@ -3269,7 +3269,7 @@ check_keyword_id(
 			: (cur_si == NULL
 			    ? !(kp->flags & HL_CONTAINED)
 			    : in_id_list(cur_si, cur_si->si_cont_list,
-				      &kp->k_syn, kp->flags & HL_CONTAINED)))
+				      &kp->k_syn, kp->flags)))
 		{
 		    *endcolp = startcol + kwlen;
 		    *flagsp = kp->flags;
@@ -4681,7 +4681,7 @@ syn_incl_toplevel(int id, int *flagsp)
 {
     if ((*flagsp & HL_CONTAINED) || curwin->w_s->b_syn_topgrp == 0)
 	return;
-    *flagsp |= HL_CONTAINED;
+    *flagsp |= HL_CONTAINED | HL_INCLUDED_TOPLEVEL;
     if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER)
     {
 	// We have to alloc this, because syn_combine_list() will free it.
@@ -5969,17 +5969,12 @@ get_id_list(
 		    break;
 		}
 		if (name[1] == 'A')
-		    id = SYNID_ALLBUT + current_syn_inc_tag;
+		    id = SYNID_ALLBUT;
 		else if (name[1] == 'T')
-		{
+		    id = SYNID_TOP;
-		    if (curwin->w_s->b_syn_topgrp >= SYNID_CLUSTER)
-			id = curwin->w_s->b_syn_topgrp;
 		else
-			id = SYNID_TOP + current_syn_inc_tag;
+		    id = SYNID_CONTAINED;
-		}
+		id += current_syn_inc_tag;
-		else
-		    id = SYNID_CONTAINED + current_syn_inc_tag;
-
 	    }
 	    else if (name[1] == '@')
 	    {
@@ -6127,7 +6122,7 @@ in_id_list(
     stateitem_T	*cur_si,	// current item or NULL
     short	*list,		// id list
     struct sp_syn *ssp,		// group id and ":syn include" tag of group
-    int		contained)	// group id is contained
+    int		flags)		// group flags
 {
     int		retval;
     short	*scl_list;
@@ -6135,6 +6130,7 @@ in_id_list(
     short	id = ssp->id;
     static int	depth = 0;
     int		r;
+    int		toplevel;
 
     // If ssp has a "containedin" list and "cur_si" is in it, return TRUE.
     if (cur_si != NULL && ssp->cont_in_list != NULL
@@ -6148,7 +6144,7 @@ in_id_list(
 	// cur_si->si_idx is -1 for keywords, these never contain anything.
 	if (cur_si->si_idx >= 0 && in_id_list(NULL, ssp->cont_in_list,
 		&(SYN_ITEMS(syn_block)[cur_si->si_idx].sp_syn),
-		  SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags & HL_CONTAINED))
+		  SYN_ITEMS(syn_block)[cur_si->si_idx].sp_flags))
 	    return TRUE;
     }
 
@@ -6160,7 +6156,14 @@ in_id_list(
      * inside anything.  Only allow not-contained groups.
      */
     if (list == ID_LIST_ALL)
-	return !contained;
+	return !(flags & HL_CONTAINED);
+
+    /*
+     * Is this top-level (i.e. not 'contained') in the file it was declared in?
+     * For included files, this is different from HL_CONTAINED, which is set
+     * unconditionally.
+     */
+    toplevel = !(flags & HL_CONTAINED) || (flags & HL_INCLUDED_TOPLEVEL);
 
     /*
      * If the first item is "ALLBUT", return TRUE if "id" is NOT in the
@@ -6179,13 +6182,13 @@ in_id_list(
 	else if (item < SYNID_CONTAINED)
 	{
 	    // TOP: accept all not-contained groups in the same file
-	    if (item - SYNID_TOP != ssp->inc_tag || contained)
+	    if (item - SYNID_TOP != ssp->inc_tag || !toplevel)
 		return FALSE;
 	}
 	else
 	{
 	    // CONTAINED: accept all contained groups in the same file
-	    if (item - SYNID_CONTAINED != ssp->inc_tag || !contained)
+	    if (item - SYNID_CONTAINED != ssp->inc_tag || toplevel)
 		return FALSE;
 	}
 	item = *++list;
@@ -6209,7 +6212,7 @@ in_id_list(
 	    if (scl_list != NULL && depth < 30)
 	    {
 		++depth;
-		r = in_id_list(NULL, scl_list, ssp, contained);
+		r = in_id_list(NULL, scl_list, ssp, flags);
 		--depth;
 		if (r)
 		    return retval;

src/testdir/test_syntax.vim

@@ -949,7 +949,7 @@ func Test_syn_contained_transparent()
 endfunc
 
 func Test_syn_include_contains_TOP()
-  let l:case = "TOP in included syntax means its group list name"
+  let l:case = "TOP in included syntax refers to top level of that included syntax"
   new
   syntax include @INCLUDED syntax/c.vim
   syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
@@ -964,6 +964,18 @@ func Test_syn_include_contains_TOP()
   bw!
 endfunc
 
+func Test_syn_include_contains_TOP_excluding()
+  new
+  syntax include @INCLUDED syntax/c.vim
+  syntax region FencedCodeBlockC start=/```c/ end=/```/ contains=@INCLUDED
+
+  call setline(1,  ['```c', '#if 0', 'int', '#else', 'int', '#if', '#endif', '```' ])
+  let l:expected = ["cCppOutElse", "cConditional"]
+  eval AssertHighlightGroups(6, 1, l:expected, 1)
+  syntax clear
+  bw!
+endfunc
+
 " This was using freed memory
 func Test_WinEnter_synstack_synID()
   autocmd WinEnter * call synstack(line("."), col("."))

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1077,
 /**/
     1076,
 /**/

src/vim.h

@@ -953,6 +953,7 @@ extern int (*dyn_libintl_wputenv)(const wchar_t *envstring);
 # define HL_TRANS_CONT	0x10000 // transparent item without contains arg
 # define HL_CONCEAL	0x20000 // can be concealed
 # define HL_CONCEALENDS	0x40000 // can be concealed
+# define HL_INCLUDED_TOPLEVEL 0x80000 // toplevel item in included syntax, allowed by contains=TOP
 #endif
 
 // Values for 'options' argument in do_search() and searchit()