aniani/vim
0
0
Fork 1
mirror of https://github.com/vim/vim.git synced 2025-09-23 03:43:49 -04:00
Code Activity

patch 8.2.2178: Python 3: non-utf8 character cannot be handled

Browse Source 
Problem:    Python 3: non-utf8 character cannot be handled.
Solution:   Change the string decode. (Björn Linse, closes #1053)
This commit is contained in:
Bram Moolenaar
2020-12-21 16:03:02 +01:00
parent ef2dff52de
commit 2e2f52a4a0
6 changed files with 34 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/if_py_both.h
View File

@@ -130,10 +130,11 @@ StringToChars(PyObject *obj, PyObject **todecref)
{ {
PyObject *bytes; PyObject *bytes;
if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL))) if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
ERRORS_ENCODE_ARG)))
return NULL; return NULL;
if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1 if (PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1
|| str == NULL) || str == NULL)
{ {
Py_DECREF(bytes); Py_DECREF(bytes);
@@ -4243,7 +4244,8 @@ StringToLine(PyObject *obj)
} }
else if (PyUnicode_Check(obj)) else if (PyUnicode_Check(obj))
{ {
if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL))) if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
ERRORS_ENCODE_ARG)))
return NULL; return NULL;
if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1 if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1
@@ -6290,11 +6292,11 @@ _ConvertFromPyObject(PyObject *obj, typval_T *tv, PyObject *lookup_dict)
PyObject *bytes; PyObject *bytes;
char_u *str; char_u *str;
bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL); bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, ERRORS_ENCODE_ARG);
if (bytes == NULL) if (bytes == NULL)
return -1; return -1;
if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1) if (PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1)
return -1; return -1;
if (str == NULL) if (str == NULL)
return -1; return -1;

4
src/if_python.c
View File

@@ -69,6 +69,10 @@
# undef PY_SSIZE_T_CLEAN # undef PY_SSIZE_T_CLEAN
#endif #endif
// these are NULL for Python 2
#define ERRORS_DECODE_ARG NULL
#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
#undef main // Defined in python.h - aargh #undef main // Defined in python.h - aargh
#undef HAVE_FCNTL_H // Clash with os_win32.h #undef HAVE_FCNTL_H // Clash with os_win32.h

11
src/if_python3.c
View File

@@ -81,12 +81,15 @@
// Python 3 does not support CObjects, always use Capsules // Python 3 does not support CObjects, always use Capsules
#define PY_USE_CAPSULE #define PY_USE_CAPSULE
#define ERRORS_DECODE_ARG CODEC_ERROR_HANDLER
#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
#define PyInt Py_ssize_t #define PyInt Py_ssize_t
#ifndef PyString_Check #ifndef PyString_Check
# define PyString_Check(obj) PyUnicode_Check(obj) # define PyString_Check(obj) PyUnicode_Check(obj)
#endif #endif
#define PyString_FromString(repr) \ #define PyString_FromString(repr) \
PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, NULL) PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, ERRORS_DECODE_ARG)
#define PyString_FromFormat PyUnicode_FromFormat #define PyString_FromFormat PyUnicode_FromFormat
#ifndef PyInt_Check #ifndef PyInt_Check
# define PyInt_Check(obj) PyLong_Check(obj) # define PyInt_Check(obj) PyLong_Check(obj)
@@ -1088,8 +1091,8 @@ DoPyCommand(const char *cmd, rangeinitializer init_range, runner run, void *arg)
// PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause // PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause
// SyntaxError (unicode error). // SyntaxError (unicode error).
cmdstr = PyUnicode_Decode(cmd, strlen(cmd), cmdstr = PyUnicode_Decode(cmd, strlen(cmd),
(char *)ENC_OPT, CODEC_ERROR_HANDLER); (char *)ENC_OPT, ERRORS_DECODE_ARG);
cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", CODEC_ERROR_HANDLER); cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", ERRORS_ENCODE_ARG);
Py_XDECREF(cmdstr); Py_XDECREF(cmdstr);
run(PyBytes_AsString(cmdbytes), arg, &pygilstate); run(PyBytes_AsString(cmdbytes), arg, &pygilstate);
@@ -1745,7 +1748,7 @@ LineToString(const char *str)
} }
*p = '\0'; *p = '\0';
result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, CODEC_ERROR_HANDLER); result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, ERRORS_DECODE_ARG);
vim_free(tmp); vim_free(tmp);
return result; return result;

7
src/testdir/test_python2.vim
View File

@@ -3775,4 +3775,11 @@ func Test_python_keyboard_interrupt()
close! close!
endfunc endfunc
func Test_python_non_utf8_string()
smap <Esc>@ <A-@>
python vim.command('redir => _tmp_smaps | smap | redir END')
python vim.eval('_tmp_smaps').splitlines()
sunmap <Esc>@
endfunc
" vim: shiftwidth=2 sts=2 expandtab " vim: shiftwidth=2 sts=2 expandtab

7
src/testdir/test_python3.vim
View File

@@ -4008,4 +4008,11 @@ func Test_python3_iter_ref()
call assert_equal(1, g:options_iter_ref_count_increase) call assert_equal(1, g:options_iter_ref_count_increase)
endfunc endfunc
func Test_python3_non_utf8_string()
smap <Esc>@ <A-@>
py3 vim.command('redir => _tmp_smaps | smap | redir END')
py3 vim.eval('_tmp_smaps').splitlines()
sunmap <Esc>@
endfunc
" vim: shiftwidth=2 sts=2 expandtab " vim: shiftwidth=2 sts=2 expandtab

2
src/version.c
View File

@@ -750,6 +750,8 @@ static char *(features[]) =
static int included_patches[] = static int included_patches[] =
{ /* Add new patch number below this line */ { /* Add new patch number below this line */
/**/
2178,
/**/ /**/
2177, 2177,
/**/ /**/