patch 8.2.4804: expression in heredoc doesn't work for compiled function

Problem:    Expression in heredoc doesn't work for compiled function.
Solution:   Implement compiling the heredoc expressions. (Yegappan Lakshmanan,
            closes #10232)

runtime/doc/eval.txt

@@ -3247,8 +3247,6 @@ text...
 			expression evaluation fails, then the assignment fails.
 			once the "`=" has been found {expr} and a backtick
 			must follow.  {expr} cannot be empty.
-			Currenty, in a compiled function {expr} is evaluated
-			when compiling the function, THIS WILL CHANGE.
 
 			{endmarker} must not contain white space.
 			{endmarker} cannot start with a lower case character.

src/evalvars.c

@@ -673,16 +673,21 @@ eval_all_expr_in_str(char_u *str)
  *
  * The {marker} is a string. If the optional 'trim' word is supplied before the
  * marker, then the leading indentation before the lines (matching the
- * indentation in the 'cmd' line) is stripped.
+ * indentation in the "cmd" line) is stripped.
  *
  * When getting lines for an embedded script (e.g. python, lua, perl, ruby,
- * tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
+ * tcl, mzscheme), "script_get" is set to TRUE. In this case, if the marker is
  * missing, then '.' is accepted as a marker.
  *
+ * When compiling a heredoc assignment to a variable in a Vim9 def function,
+ * "vim9compile" is set to TRUE. In this case, instead of generating a list of
+ * string values from the heredoc, vim9 instructions are generated.  On success
+ * the returned list will be empty.
+ *
  * Returns a List with {lines} or NULL on failure.
  */
     list_T *
-heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
+heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile)
 {
     char_u	*theline = NULL;
     char_u	*marker;
@@ -696,6 +701,8 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
     int		comment_char = in_vim9script() ? '#' : '"';
     int		evalstr = FALSE;
     int		eval_failed = FALSE;
+    cctx_T	*cctx = vim9compile ? eap->cookie : NULL;
+    int		count = 0;
 
     if (eap->getline == NULL)
     {
@@ -816,6 +823,18 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 		    break;
 
 	str = theline + ti;
+	if (vim9compile)
+	{
+	    if (compile_heredoc_string(str, evalstr, cctx) == FAIL)
+	    {
+		vim_free(theline);
+		vim_free(text_indent);
+		return FAIL;
+	    }
+	    count++;
+	}
+	else
+	{
 	    if (evalstr)
 	    {
 		str = eval_all_expr_in_str(str);
@@ -832,9 +851,13 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 	    if (list_append_string(l, str, -1) == FAIL)
 		break;
 	}
+    }
     vim_free(theline);
     vim_free(text_indent);
 
+    if (vim9compile && cctx->ctx_skip != SKIP_YES && !eval_failed)
+	generate_NEWLIST(cctx, count, FALSE);
+
     if (eval_failed)
     {
 	// expression evaluation in the heredoc failed
@@ -986,7 +1009,7 @@ ex_let(exarg_T *eap)
 	long	cur_lnum = SOURCING_LNUM;
 
 	// HERE document
-	l = heredoc_get(eap, expr + 3, FALSE);
+	l = heredoc_get(eap, expr + 3, FALSE, FALSE);
 	if (l != NULL)
 	{
 	    rettv_list_set(&rettv, l);

src/ex_getln.c

@@ -4605,7 +4605,7 @@ script_get(exarg_T *eap UNUSED, char_u *cmd UNUSED)
 	return NULL;
     cmd += 2;
 
-    l = heredoc_get(eap, cmd, TRUE);
+    l = heredoc_get(eap, cmd, TRUE, FALSE);
     if (l == NULL)
 	return NULL;
 

src/proto/evalvars.pro

@@ -13,7 +13,7 @@ list_T *eval_spell_expr(char_u *badword, char_u *expr);
 int get_spellword(list_T *list, char_u **pp);
 void prepare_vimvar(int idx, typval_T *save_tv);
 void restore_vimvar(int idx, typval_T *save_tv);
-list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get);
+list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile);
 void ex_var(exarg_T *eap);
 void ex_let(exarg_T *eap);
 int ex_let_vars(char_u *arg_start, typval_T *tv, int copy, int semicolon, int var_count, int flags, char_u *op);

src/proto/vim9compile.pro

@@ -16,6 +16,7 @@ int may_get_next_line(char_u *whitep, char_u **arg, cctx_T *cctx);
 int may_get_next_line_error(char_u *whitep, char_u **arg, cctx_T *cctx);
 void fill_exarg_from_cctx(exarg_T *eap, cctx_T *cctx);
 int func_needs_compiling(ufunc_T *ufunc, compiletype_T compile_type);
+int compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx);
 int assignment_len(char_u *p, int *heredoc);
 void vim9_declare_error(char_u *name);
 int get_var_dest(char_u *name, assign_dest_T *dest, cmdidx_T cmdidx, int *option_scope, int *vimvaridx, type_T **type, cctx_T *cctx);

src/testdir/test_vim9_assign.vim

@@ -1821,10 +1821,31 @@ def Test_assign_lambda()
 enddef
 
 def Test_heredoc()
-  var lines =<< trim END # comment
+  # simple heredoc
-    text
+  var lines =<< trim END
+    var text =<< trim TEXT # comment
+      abc
+    TEXT
+    assert_equal(['abc'], text)
   END
-  assert_equal(['text'], lines)
+  v9.CheckDefAndScriptSuccess(lines)
+
+  # empty heredoc
+  lines =<< trim END
+     var text =<< trim TEXT
+     TEXT
+     assert_equal([], text)
+  END
+  v9.CheckDefAndScriptSuccess(lines)
+
+  # heredoc with a single empty line
+  lines =<< trim END
+     var text =<< trim TEXT
+
+     TEXT
+     assert_equal([''], text)
+  END
+  v9.CheckDefAndScriptSuccess(lines)
 
   v9.CheckDefFailure(['var lines =<< trim END X', 'END'], 'E488:')
   v9.CheckDefFailure(['var lines =<< trim END " comment', 'END'], 'E488:')
@@ -2642,51 +2663,68 @@ let g:someVar = 'X'
 " Test for heredoc with Vim expressions.
 " This messes up highlighting, keep it near the end.
 def Test_heredoc_expr()
+  var lines =<< trim CODE
+    var s = "local"
+    var a1 = "1"
+    var a2 = "2"
+    var a3 = "3"
+    var a4 = ""
     var code =<< trim eval END
       var a = `=5 + 10`
       var b = `=min([10, 6])` + `=max([4, 6])`
+      var c = "`=s`"
+      var d = x`=a1`x`=a2`x`=a3`x`=a4`
     END
-  assert_equal(['var a = 15', 'var b = 6 + 6'], code)
+    assert_equal(['var a = 15', 'var b = 6 + 6', 'var c = "local"', 'var d = x1x2x3x'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval trim END
+  lines =<< trim CODE
+    var code =<< eval trim END
       var s = "`=$SOME_ENV_VAR`"
     END
     assert_equal(['var s = "somemore"'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval END
+  lines =<< trim CODE
+    var code =<< eval END
       var s = "`=$SOME_ENV_VAR`"
-END
+    END
     assert_equal(['  var s = "somemore"'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval trim END
+  lines =<< trim CODE
+    var code =<< eval trim END
       let a = `abc`
       let b = `=g:someVar`
       let c = `
     END
     assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  var lines =<< trim LINES
+  lines =<< trim LINES
       var text =<< eval trim END
         let b = `=
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+  v9.CheckDefAndScriptFailure(lines, ['E1143: Empty expression: ""', 'E1083: Missing backtick'])
 
   lines =<< trim LINES
       var text =<< eval trim END
         let b = `=abc
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+  v9.CheckDefAndScriptFailure(lines, ['E1001: Variable not found: abc', 'E1083: Missing backtick'])
 
   lines =<< trim LINES
       var text =<< eval trim END
         let b = `=`
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E15:')
+  v9.CheckDefAndScriptFailure(lines, ['E1015: Name expected: `', 'E15: Invalid expression: "`"'])
 enddef
 
-
-
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker

src/version.c

@@ -746,6 +746,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    4804,
 /**/
     4803,
 /**/

src/vim9compile.c

@@ -595,6 +595,7 @@ find_imported_in_script(char_u *name, size_t len, int sid)
 
 /*
  * Find "name" in imported items of the current script.
+ * If "len" is 0 use any length that works.
  * If "load" is TRUE and the script was not loaded yet, load it now.
  */
     imported_T *
@@ -967,6 +968,83 @@ theend:
     return r == FAIL ? NULL : (char_u *)"";
 }
 
+/*
+ * Compile a heredoc string "str" (either containing a literal string or a mix
+ * of literal strings and Vim expressions of the form `=<expr>`).  This is used
+ * when compiling a heredoc assignment to a variable in a Vim9 def function.
+ * Vim9 instructions are generated to push strings, evaluate expressions,
+ * concatenate them and create a list of lines.  When "evalstr" is TRUE, Vim
+ * expressions in "str" are evaluated.
+ */
+    int
+compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx)
+{
+    char_u	*p;
+    char_u	*val;
+
+    if (cctx->ctx_skip == SKIP_YES)
+	return OK;
+
+    if (evalstr && (p = (char_u *)strstr((char *)str, "`=")) != NULL)
+    {
+	char_u	*start = str;
+
+	// Need to evaluate expressions of the form `=<expr>` in the string.
+	// Split the string into literal strings and Vim expressions and
+	// generate instructions to concatenate the literal strings and the
+	// result of evaluating the Vim expressions.
+	val = vim_strsave((char_u *)"");
+	generate_PUSHS(cctx, &val);
+
+	for (;;)
+	{
+	    if (p > start)
+	    {
+		// literal string before the expression
+		val = vim_strnsave(start, p - start);
+		generate_PUSHS(cctx, &val);
+		generate_instr_drop(cctx, ISN_CONCAT, 1);
+	    }
+	    p += 2;
+
+	    // evaluate the Vim expression and convert the result to string.
+	    if (compile_expr0(&p, cctx) == FAIL)
+		return FAIL;
+	    may_generate_2STRING(-1, TRUE, cctx);
+	    generate_instr_drop(cctx, ISN_CONCAT, 1);
+
+	    p = skipwhite(p);
+	    if (*p != '`')
+	    {
+		emsg(_(e_missing_backtick));
+		return FAIL;
+	    }
+	    start = p + 1;
+
+	    p = (char_u *)strstr((char *)start, "`=");
+	    if (p == NULL)
+	    {
+		// no more Vim expressions left to process
+		if (*skipwhite(start) != NUL)
+		{
+		    val = vim_strsave(start);
+		    generate_PUSHS(cctx, &val);
+		    generate_instr_drop(cctx, ISN_CONCAT, 1);
+		}
+		break;
+	    }
+	}
+    }
+    else
+    {
+	// literal string
+	val = vim_strsave(str);
+	generate_PUSHS(cctx, &val);
+    }
+
+    return OK;
+}
+
 /*
  * Return the length of an assignment operator, or zero if there isn't one.
  */
@@ -1946,25 +2024,14 @@ compile_assignment(char_u *arg, exarg_T *eap, cmdidx_T cmdidx, cctx_T *cctx)
     if (heredoc)
     {
 	list_T	   *l;
-	listitem_T *li;
 
 	// [let] varname =<< [trim] {end}
 	eap->getline = exarg_getline;
 	eap->cookie = cctx;
-	l = heredoc_get(eap, op + 3, FALSE);
+	l = heredoc_get(eap, op + 3, FALSE, TRUE);
 	if (l == NULL)
 	    return NULL;
 
-	if (cctx->ctx_skip != SKIP_YES)
-	{
-	    // Push each line and the create the list.
-	    FOR_ALL_LIST_ITEMS(l, li)
-	    {
-		generate_PUSHS(cctx, &li->li_tv.vval.v_string);
-		li->li_tv.vval.v_string = NULL;
-	    }
-	    generate_NEWLIST(cctx, l->lv_len, FALSE);
-	}
 	list_free(l);
 	p += STRLEN(p);
 	end = p;