0
0
mirror of https://github.com/vim/vim.git synced 2025-07-26 11:04:33 -04:00

patch 8.1.0529: flaky test sometimes fails in different ways

Problem:    Flaky test sometimes fails in different ways.
Solution:   When the second run gives a different error, try running the test
            again, up to five times.
This commit is contained in:
Bram Moolenaar 2018-11-16 16:52:16 +01:00
parent c4568ab37e
commit f77af0e613
2 changed files with 37 additions and 16 deletions

View File

@ -314,28 +314,47 @@ endif
for s:test in sort(s:tests)
" Silence, please!
set belloff=all
let prev_error = ''
let total_errors = []
let run_nr = 1
call RunTheTest(s:test)
" Repeat a flaky test. Give up when:
" - it fails again with the same message
" - it fails five times (with a different mesage)
if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
call add(s:messages, 'Found errors in ' . s:test . ':')
call extend(s:messages, v:errors)
call add(s:messages, 'Flaky test failed, running it again')
let first_run = v:errors
while 1
call add(s:messages, 'Found errors in ' . s:test . ':')
call extend(s:messages, v:errors)
" Flakiness is often caused by the system being very busy. Sleep a couple
" of seconds to have a higher chance of succeeding the second time.
sleep 2
call add(total_errors, 'Run ' . run_nr . ':')
call extend(total_errors, v:errors)
let v:errors = []
call RunTheTest(s:test)
if len(v:errors) > 0
let second_run = v:errors
let v:errors = ['First run:']
call extend(v:errors, first_run)
call add(v:errors, 'Second run:')
call extend(v:errors, second_run)
endif
if run_nr == 5 || prev_error == v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
endif
call add(s:messages, 'Flaky test failed, running it again')
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
sleep 2
let prev_error = v:errors[0]
let v:errors = []
let run_nr += 1
call RunTheTest(s:test)
if len(v:errors) == 0
" Test passed on rerun.
break
endif
endwhile
endif
call AfterTheTest()

View File

@ -792,6 +792,8 @@ static char *(features[]) =
static int included_patches[] =
{ /* Add new patch number below this line */
/**/
529,
/**/
528,
/**/