0
0
mirror of https://github.com/vim/vim.git synced 2025-07-26 11:04:33 -04:00

patch 8.1.0529: flaky test sometimes fails in different ways

Problem:    Flaky test sometimes fails in different ways.
Solution:   When the second run gives a different error, try running the test
            again, up to five times.
This commit is contained in:
Bram Moolenaar 2018-11-16 16:52:16 +01:00
parent c4568ab37e
commit f77af0e613
2 changed files with 37 additions and 16 deletions

View File

@ -314,28 +314,47 @@ endif
for s:test in sort(s:tests) for s:test in sort(s:tests)
" Silence, please! " Silence, please!
set belloff=all set belloff=all
let prev_error = ''
let total_errors = []
let run_nr = 1
call RunTheTest(s:test) call RunTheTest(s:test)
" Repeat a flaky test. Give up when:
" - it fails again with the same message
" - it fails five times (with a different mesage)
if len(v:errors) > 0 && index(s:flaky, s:test) >= 0 if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
while 1
call add(s:messages, 'Found errors in ' . s:test . ':') call add(s:messages, 'Found errors in ' . s:test . ':')
call extend(s:messages, v:errors) call extend(s:messages, v:errors)
call add(s:messages, 'Flaky test failed, running it again')
let first_run = v:errors
" Flakiness is often caused by the system being very busy. Sleep a couple call add(total_errors, 'Run ' . run_nr . ':')
" of seconds to have a higher chance of succeeding the second time. call extend(total_errors, v:errors)
if run_nr == 5 || prev_error == v:errors[0]
call add(total_errors, 'Flaky test failed too often, giving up')
let v:errors = total_errors
break
endif
call add(s:messages, 'Flaky test failed, running it again')
" Flakiness is often caused by the system being very busy. Sleep a
" couple of seconds to have a higher chance of succeeding the second
" time.
sleep 2 sleep 2
let prev_error = v:errors[0]
let v:errors = [] let v:errors = []
let run_nr += 1
call RunTheTest(s:test) call RunTheTest(s:test)
if len(v:errors) > 0
let second_run = v:errors if len(v:errors) == 0
let v:errors = ['First run:'] " Test passed on rerun.
call extend(v:errors, first_run) break
call add(v:errors, 'Second run:')
call extend(v:errors, second_run)
endif endif
endwhile
endif endif
call AfterTheTest() call AfterTheTest()

View File

@ -792,6 +792,8 @@ static char *(features[]) =
static int included_patches[] = static int included_patches[] =
{ /* Add new patch number below this line */ { /* Add new patch number below this line */
/**/
529,
/**/ /**/
528, 528,
/**/ /**/