forked from aniani/vim
e70587dbdb
Problem: Vim tests are slow and flaky at the same time due to reliance
on timeouts which are unreliable.
Solution: improve Vim test performance and reduce flakiness
(Yee Cheng Chin)
A lot of Vim tests currently rely on waiting a specific amount of time
before asserting a condition. This is bad because 1) it is slow, as the
timeout is hardcoded, 2) it's unreliable as a resource-starved runner
may overshoot the timeout. Also, there are a lot of builtin sleep
commands in commonly used utilities like VerifyScreenDump and WaitFor()
which leads to a lot of unnecessary idle time.
Fix these issues by doing the following:
1. Make utilities like VerifyScreenDump and WaitFor use the lowest wait
time possible (1 ms). This essentially turns it into a spin wait. On
fast machines, these will finish very quickly. For existing tests
that had an implicit reliance on the old timeouts (e.g.
VerifyScreenDump had a 50ms wait before), fix the tests to wait that
specific amount explicitly.
2. Fix tests that sleep or wait for long amounts of time to instead
explicitly use a callback mechanism to be notified when a child
terminal job has finished. This allows the test to only take as much
time as possible instead of having to hard code an unreliable
timeout.
With these fixes, tests should 1) completely quickly on fast machines,
and 2) on slow machines they will still run to completion albeit slowly.
Note that previoulsy both were not true. The hardcoded timeouts meant
that on fast machines the tests were mostly idling wasting time, whereas
on slow machines, the timeouts often were not generous enough to allow
them to run to completion.
closes: #16615
Signed-off-by: Yee Cheng Chin <ychin.git@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
237 lines
7.0 KiB
VimL
237 lines
7.0 KiB
VimL
" Some tests, that used to crash Vim
|
|
source check.vim
|
|
source screendump.vim
|
|
|
|
CheckScreendump
|
|
|
|
" Run the command in terminal and wait for it to complete via notification
|
|
func s:RunCommandAndWait(buf, cmd)
|
|
call term_sendkeys(a:buf, a:cmd .. "; printf '" .. TermNotifyParentCmd(v:false) .. "'\<cr>")
|
|
call WaitForChildNotification()
|
|
endfunc
|
|
|
|
func Test_crash1()
|
|
CheckNotBSD
|
|
CheckExecutable dash
|
|
|
|
" The following used to crash Vim
|
|
let opts = #{cmd: 'sh'}
|
|
let vim = GetVimProg()
|
|
|
|
let buf = RunVimInTerminal('sh', opts)
|
|
|
|
let file = 'crash/poc_huaf1'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 1: [OK]" > X_crash1_result.txt')
|
|
|
|
let file = 'crash/poc_huaf2'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 2: [OK]" >> X_crash1_result.txt')
|
|
|
|
let file = 'crash/poc_huaf3'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 3: [OK]" >> X_crash1_result.txt')
|
|
|
|
let file = 'crash/bt_quickfix_poc'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 4: [OK]" >> X_crash1_result.txt')
|
|
" clean up
|
|
call delete('Xerr')
|
|
|
|
let file = 'crash/poc_tagfunc.vim'
|
|
let args = printf(cmn_args, vim, file)
|
|
" using || because this poc causes vim to exit with exitstatus != 0
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' || echo "crash 5: [OK]" >> X_crash1_result.txt')
|
|
|
|
|
|
let file = 'crash/bt_quickfix1_poc'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 6: [OK]" >> X_crash1_result.txt')
|
|
" clean up
|
|
call delete('X')
|
|
|
|
let file = 'crash/vim_regsub_both_poc'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 7: [OK]" >> X_crash1_result.txt')
|
|
|
|
let file = 'crash/vim_msg_trunc_poc'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' || echo "crash 8: [OK]" >> X_crash1_result.txt')
|
|
|
|
let file = 'crash/crash_scrollbar'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 9: [OK]" >> X_crash1_result.txt')
|
|
|
|
let file = 'crash/editing_arg_idx_POC_1'
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' || echo "crash 10: [OK]" >> X_crash1_result.txt')
|
|
call delete('Xerr')
|
|
call delete('@')
|
|
|
|
" clean up
|
|
exe buf .. "bw!"
|
|
|
|
sp X_crash1_result.txt
|
|
|
|
let expected = [
|
|
\ 'crash 1: [OK]',
|
|
\ 'crash 2: [OK]',
|
|
\ 'crash 3: [OK]',
|
|
\ 'crash 4: [OK]',
|
|
\ 'crash 5: [OK]',
|
|
\ 'crash 6: [OK]',
|
|
\ 'crash 7: [OK]',
|
|
\ 'crash 8: [OK]',
|
|
\ 'crash 9: [OK]',
|
|
\ 'crash 10: [OK]',
|
|
\ ]
|
|
|
|
call assert_equal(expected, getline(1, '$'))
|
|
bw!
|
|
|
|
call delete('X_crash1_result.txt')
|
|
endfunc
|
|
|
|
func Test_crash1_2()
|
|
CheckNotBSD
|
|
CheckExecutable dash
|
|
|
|
" The following used to crash Vim
|
|
let opts = #{cmd: 'sh'}
|
|
let vim = GetVimProg()
|
|
let result = 'X_crash1_2_result.txt'
|
|
|
|
let buf = RunVimInTerminal('sh', opts)
|
|
|
|
let file = 'crash/poc1'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 1: [OK]" > '.. result)
|
|
|
|
let file = 'crash/poc_win_enter_ext'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 2: [OK]" >> '.. result)
|
|
|
|
let file = 'crash/poc_suggest_trie_walk'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' && echo "crash 3: [OK]" >> '.. result)
|
|
|
|
let file = 'crash/poc_did_set_langmap'
|
|
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' ; echo "crash 4: [OK]" >> '.. result)
|
|
|
|
let file = 'crash/reverse_text_overflow'
|
|
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args ..
|
|
\ ' ; echo "crash 5: [OK]" >> '.. result)
|
|
|
|
" clean up
|
|
exe buf .. "bw!"
|
|
exe "sp " .. result
|
|
let expected = [
|
|
\ 'crash 1: [OK]',
|
|
\ 'crash 2: [OK]',
|
|
\ 'crash 3: [OK]',
|
|
\ 'crash 4: [OK]',
|
|
\ 'crash 5: [OK]',
|
|
\ ]
|
|
|
|
call assert_equal(expected, getline(1, '$'))
|
|
bw!
|
|
call delete(result)
|
|
endfunc
|
|
|
|
" This test just runs various scripts, that caused issues before.
|
|
" We are not really asserting anything here, it's just important
|
|
" that ASAN does not detect any issues.
|
|
func Test_crash1_3()
|
|
let vim = GetVimProg()
|
|
let buf = RunVimInTerminal('sh', #{cmd: 'sh'})
|
|
|
|
let file = 'crash/poc_ex_substitute'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/poc_uaf_exec_instructions'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/poc_uaf_check_argument_types'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/double_free'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/dialog_changed_uaf'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/nullpointer'
|
|
let cmn_args = "%s -u NONE -i NONE -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/heap_overflow3'
|
|
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/heap_overflow_glob2regpat'
|
|
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
let file = 'crash/nullptr_regexp_nfa'
|
|
let cmn_args = "%s -u NONE -i NONE -n -X -m -n -e -s -S %s -c ':qa!'"
|
|
let args = printf(cmn_args, vim, file)
|
|
call s:RunCommandAndWait(buf, args)
|
|
|
|
" clean up
|
|
exe buf .. "bw!"
|
|
bw!
|
|
endfunc
|
|
|
|
func Test_crash2()
|
|
" The following used to crash Vim
|
|
let opts = #{wait_for_ruler: 0, rows: 20}
|
|
let args = ' -u NONE -i NONE -n -e -s -S '
|
|
let buf = RunVimInTerminal(args .. ' crash/vim_regsub_both', opts)
|
|
call VerifyScreenDump(buf, 'Test_crash_01', {})
|
|
exe buf .. "bw!"
|
|
endfunc
|
|
|
|
func TearDown()
|
|
" That file is created at Test_crash1_3() by dialog_changed_uaf
|
|
" but cleaning up in that test doesn't remove it. Let's try again at
|
|
" the end of this test script
|
|
call delete('Untitled')
|
|
endfunc
|
|
|
|
" vim: shiftwidth=2 sts=2 expandtab
|