| Message ID | fbbb792fa299bfdfa818f6e0790fa545297dc1cf.1721949827.git.osandov@fb.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | libbtrfsutil: python: fix build on Python 3.13 | expand |
On Thu, Jul 25, 2024 at 7:28 PM Omar Sandoval <osandov@osandov.com> wrote: > > From: Omar Sandoval <osandov@fb.com> > > Python 3.13, currently in beta, removed the internal > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > it in the path_converter() function because it was based on internal > path_converter() function in CPython [1]. This is causing build failures > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > version that only uses public functions based on the one in drgn [4]. > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > 3: https://github.com/kdave/btrfs-progs/issues/838 > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > Reported-by: Neal Gompa <neal@gompa.dev> > Reported-by: Sam James <sam@gentoo.org> > Signed-off-by: Omar Sandoval <osandov@fb.com> > --- > libbtrfsutil/python/btrfsutilpy.h | 7 +-- > libbtrfsutil/python/module.c | 98 ++++++++++--------------------- > 2 files changed, 33 insertions(+), 72 deletions(-) > > diff --git a/libbtrfsutil/python/btrfsutilpy.h b/libbtrfsutil/python/btrfsutilpy.h > index ee70c23a..49702dcc 100644 > --- a/libbtrfsutil/python/btrfsutilpy.h > +++ b/libbtrfsutil/python/btrfsutilpy.h > @@ -40,16 +40,13 @@ extern PyTypeObject SubvolumeInfo_type; > extern PyTypeObject SubvolumeIterator_type; > extern PyTypeObject QgroupInherit_type; > > -/* > - * Helpers for path arguments based on posixmodule.c in CPython. > - */ > struct path_arg { > bool allow_fd; > - char *path; > int fd; > + char *path; > Py_ssize_t length; > PyObject *object; > - PyObject *cleanup; > + PyObject *bytes; > }; > int path_converter(PyObject *o, void *p); > void path_cleanup(struct path_arg *path); > diff --git a/libbtrfsutil/python/module.c b/libbtrfsutil/python/module.c > index 2657ee28..9b0b86b5 100644 > --- a/libbtrfsutil/python/module.c > +++ b/libbtrfsutil/python/module.c > @@ -44,85 +44,49 @@ static int fd_converter(PyObject *o, void *p) > int path_converter(PyObject *o, void *p) > { > struct path_arg *path = p; > - int is_index, is_bytes, is_unicode; > - PyObject *bytes = NULL; > - Py_ssize_t length = 0; > - char *tmp; > > if (o == NULL) { > path_cleanup(p); > return 1; > } > > - path->object = path->cleanup = NULL; > - Py_INCREF(o); > - > path->fd = -1; > + path->path = NULL; > + path->length = 0; > + path->bytes = NULL; > + if (path->allow_fd && PyIndex_Check(o)) { > + PyObject *fd_obj; > + int overflow; > + long fd; > > - is_index = path->allow_fd && PyIndex_Check(o); > - is_bytes = PyBytes_Check(o); > - is_unicode = PyUnicode_Check(o); > - > - if (!is_index && !is_bytes && !is_unicode) { > - _Py_IDENTIFIER(__fspath__); > - PyObject *func; > - > - func = _PyObject_LookupSpecial(o, &PyId___fspath__); > - if (func == NULL) > - goto err_format; > - Py_DECREF(o); > - o = PyObject_CallFunctionObjArgs(func, NULL); > - Py_DECREF(func); > - if (o == NULL) > + fd_obj = PyNumber_Index(o); > + if (!fd_obj) > return 0; > - is_bytes = PyBytes_Check(o); > - is_unicode = PyUnicode_Check(o); > - } > - > - if (is_unicode) { > - if (!PyUnicode_FSConverter(o, &bytes)) > - goto err; > - } else if (is_bytes) { > - bytes = o; > - Py_INCREF(bytes); > - } else if (is_index) { > - if (!fd_converter(o, &path->fd)) > - goto err; > - path->path = NULL; > - goto out; > + fd = PyLong_AsLongAndOverflow(fd_obj, &overflow); > + Py_DECREF(fd_obj); > + if (fd == -1 && PyErr_Occurred()) > + return 0; > + if (overflow > 0 || fd > INT_MAX) { > + PyErr_SetString(PyExc_OverflowError, > + "fd is greater than maximum"); > + return 0; > + } > + if (fd < 0) { > + PyErr_SetString(PyExc_ValueError, "fd is negative"); > + return 0; > + } > + path->fd = fd; > } else { > -err_format: > - PyErr_Format(PyExc_TypeError, "expected %s, not %s", > - path->allow_fd ? "string, bytes, os.PathLike, or integer" : > - "string, bytes, or os.PathLike", > - Py_TYPE(o)->tp_name); > - goto err; > + if (!PyUnicode_FSConverter(o, &path->bytes)) { > + path->object = path->bytes = NULL; > + return 0; > + } > + path->path = PyBytes_AS_STRING(path->bytes); > + path->length = PyBytes_GET_SIZE(path->bytes); > } > - > - length = PyBytes_GET_SIZE(bytes); > - tmp = PyBytes_AS_STRING(bytes); > - if ((size_t)length != strlen(tmp)) { > - PyErr_SetString(PyExc_TypeError, > - "path has embedded nul character"); > - goto err; > - } > - > - path->path = tmp; > - if (bytes == o) > - Py_DECREF(bytes); > - else > - path->cleanup = bytes; > - path->fd = -1; > - > -out: > - path->length = length; > + Py_INCREF(o); > path->object = o; > return Py_CLEANUP_SUPPORTED; > - > -err: > - Py_XDECREF(o); > - Py_XDECREF(bytes); > - return 0; > } > > PyObject *list_from_uint64_array(const uint64_t *arr, size_t n) > @@ -150,8 +114,8 @@ PyObject *list_from_uint64_array(const uint64_t *arr, size_t n) > > void path_cleanup(struct path_arg *path) > { > + Py_CLEAR(path->bytes); > Py_CLEAR(path->object); > - Py_CLEAR(path->cleanup); > } > > static PyMethodDef btrfsutil_methods[] = { > -- > 2.45.2 > The code changes look reasonable and the Python module builds and installs properly. The result also seems to work as intended. Reviewed-by: Neal Gompa <neal@gompa.dev> Verified that the build is fixed for Fedora Linux 41 with Python 3.13: https://koji.fedoraproject.org/koji/taskinfo?taskID=121058789 Confirmed that it still builds in Fedora Linux 40 with Python 3.12: https://koji.fedoraproject.org/koji/taskinfo?taskID=121058798 Tested-by: Neal Gompa <neal@gompa.dev> -- 真実はいつも一つ！/ Always, there's only one truth!
On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > From: Omar Sandoval <osandov@fb.com> > > Python 3.13, currently in beta, removed the internal > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > it in the path_converter() function because it was based on internal > path_converter() function in CPython [1]. This is causing build failures > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > version that only uses public functions based on the one in drgn [4]. > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > 3: https://github.com/kdave/btrfs-progs/issues/838 > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > Reported-by: Neal Gompa <neal@gompa.dev> > Reported-by: Sam James <sam@gentoo.org> > Signed-off-by: Omar Sandoval <osandov@fb.com> Thanks, this is more convoluted than I expected. Does this work on other python versions, like 3.8 and above? I'd have to check what's the lowest expected python version derived from the base line for distro support so 3.6 is just a guess.
On Thu, Jul 25, 2024 at 9:12 PM David Sterba <dsterba@suse.cz> wrote: > > On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > > From: Omar Sandoval <osandov@fb.com> > > > > Python 3.13, currently in beta, removed the internal > > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > > it in the path_converter() function because it was based on internal > > path_converter() function in CPython [1]. This is causing build failures > > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > > version that only uses public functions based on the one in drgn [4]. > > > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > > 3: https://github.com/kdave/btrfs-progs/issues/838 > > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > > > Reported-by: Neal Gompa <neal@gompa.dev> > > Reported-by: Sam James <sam@gentoo.org> > > Signed-off-by: Omar Sandoval <osandov@fb.com> > > Thanks, this is more convoluted than I expected. Does this work on other > python versions, like 3.8 and above? I'd have to check what's the lowest > expected python version derived from the base line for distro support so > 3.6 is just a guess. Well, I can't build it on AlmaLinux 8 with Python 3.6 because libgcrypt is too old, but it builds and works fine on CentOS Stream 9 (which has Python 3.9).
On Thu, Jul 25, 2024 at 09:29:44PM -0400, Neal Gompa wrote: > On Thu, Jul 25, 2024 at 9:12 PM David Sterba <dsterba@suse.cz> wrote: > > > > On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > > > From: Omar Sandoval <osandov@fb.com> > > > > > > Python 3.13, currently in beta, removed the internal > > > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > > > it in the path_converter() function because it was based on internal > > > path_converter() function in CPython [1]. This is causing build failures > > > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > > > version that only uses public functions based on the one in drgn [4]. > > > > > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > > > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > > > 3: https://github.com/kdave/btrfs-progs/issues/838 > > > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > > > > > Reported-by: Neal Gompa <neal@gompa.dev> > > > Reported-by: Sam James <sam@gentoo.org> > > > Signed-off-by: Omar Sandoval <osandov@fb.com> > > > > Thanks, this is more convoluted than I expected. Does this work on other > > python versions, like 3.8 and above? I'd have to check what's the lowest > > expected python version derived from the base line for distro support so > > 3.6 is just a guess. > > Well, I can't build it on AlmaLinux 8 with Python 3.6 because > libgcrypt is too old, but it builds and works fine on CentOS Stream 9 > (which has Python 3.9). Yup, this works for drgn on Python 3.6-3.13.
On Thu, Jul 25, 2024 at 06:37:01PM -0700, Omar Sandoval wrote: > On Thu, Jul 25, 2024 at 09:29:44PM -0400, Neal Gompa wrote: > > On Thu, Jul 25, 2024 at 9:12 PM David Sterba <dsterba@suse.cz> wrote: > > > > > > On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > > > > From: Omar Sandoval <osandov@fb.com> > > > > > > > > Python 3.13, currently in beta, removed the internal > > > > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > > > > it in the path_converter() function because it was based on internal > > > > path_converter() function in CPython [1]. This is causing build failures > > > > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > > > > version that only uses public functions based on the one in drgn [4]. > > > > > > > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > > > > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > > > > 3: https://github.com/kdave/btrfs-progs/issues/838 > > > > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > > > > > > > Reported-by: Neal Gompa <neal@gompa.dev> > > > > Reported-by: Sam James <sam@gentoo.org> > > > > Signed-off-by: Omar Sandoval <osandov@fb.com> > > > > > > Thanks, this is more convoluted than I expected. Does this work on other > > > python versions, like 3.8 and above? I'd have to check what's the lowest > > > expected python version derived from the base line for distro support so > > > 3.6 is just a guess. > > > > Well, I can't build it on AlmaLinux 8 with Python 3.6 because > > libgcrypt is too old, but it builds and works fine on CentOS Stream 9 > > (which has Python 3.9). > > Yup, this works for drgn on Python 3.6-3.13. Also confirmed by the CI test images, the lowest base is Centos 7 with 3.6.
On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > From: Omar Sandoval <osandov@fb.com> > > Python 3.13, currently in beta, removed the internal > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > it in the path_converter() function because it was based on internal > path_converter() function in CPython [1]. This is causing build failures > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > version that only uses public functions based on the one in drgn [4]. > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > 3: https://github.com/kdave/btrfs-progs/issues/838 > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > Reported-by: Neal Gompa <neal@gompa.dev> > Reported-by: Sam James <sam@gentoo.org> > Signed-off-by: Omar Sandoval <osandov@fb.com> Added to devel, thanks.
On Thu, Jul 25, 2024 at 04:28:35PM -0700, Omar Sandoval wrote: > From: Omar Sandoval <osandov@fb.com> > > Python 3.13, currently in beta, removed the internal > _PyObject_LookupSpecial function. The libbtrfsutil Python bindings use > it in the path_converter() function because it was based on internal > path_converter() function in CPython [1]. This is causing build failures > on Fedora Rawhide [2] and Gentoo [3]. Replace path_converter() with a > version that only uses public functions based on the one in drgn [4]. > > 1: https://github.com/python/cpython/blob/d9efa45d7457b0dfea467bb1c2d22c69056ffc73/Modules/posixmodule.c#L1253 > 2: https://bugzilla.redhat.com/show_bug.cgi?id=2245650 > 3: https://github.com/kdave/btrfs-progs/issues/838 > 4: https://github.com/osandov/drgn/blob/9ad29fd86499eb32847473e928b6540872d3d59a/libdrgn/python/util.c#L81 > > Reported-by: Neal Gompa <neal@gompa.dev> > Reported-by: Sam James <sam@gentoo.org> > Signed-off-by: Omar Sandoval <osandov@fb.com> > --- > libbtrfsutil/python/btrfsutilpy.h | 7 +-- > libbtrfsutil/python/module.c | 98 ++++++++++--------------------- > 2 files changed, 33 insertions(+), 72 deletions(-) > > diff --git a/libbtrfsutil/python/btrfsutilpy.h b/libbtrfsutil/python/btrfsutilpy.h > index ee70c23a..49702dcc 100644 > --- a/libbtrfsutil/python/btrfsutilpy.h > +++ b/libbtrfsutil/python/btrfsutilpy.h > @@ -40,16 +40,13 @@ extern PyTypeObject SubvolumeInfo_type; > extern PyTypeObject SubvolumeIterator_type; > extern PyTypeObject QgroupInherit_type; > > -/* > - * Helpers for path arguments based on posixmodule.c in CPython. > - */ > struct path_arg { > bool allow_fd; > - char *path; > int fd; > + char *path; > Py_ssize_t length; > PyObject *object; > - PyObject *cleanup; > + PyObject *bytes; > }; > int path_converter(PyObject *o, void *p); > void path_cleanup(struct path_arg *path); > diff --git a/libbtrfsutil/python/module.c b/libbtrfsutil/python/module.c > index 2657ee28..9b0b86b5 100644 > --- a/libbtrfsutil/python/module.c > +++ b/libbtrfsutil/python/module.c > @@ -44,85 +44,49 @@ static int fd_converter(PyObject *o, void *p) > int path_converter(PyObject *o, void *p) > { > struct path_arg *path = p; > - int is_index, is_bytes, is_unicode; > - PyObject *bytes = NULL; > - Py_ssize_t length = 0; > - char *tmp; > > if (o == NULL) { > path_cleanup(p); > return 1; > } > > - path->object = path->cleanup = NULL; > - Py_INCREF(o); > - > path->fd = -1; > + path->path = NULL; > + path->length = 0; > + path->bytes = NULL; > + if (path->allow_fd && PyIndex_Check(o)) { > + PyObject *fd_obj; > + int overflow; > + long fd; > > - is_index = path->allow_fd && PyIndex_Check(o); > - is_bytes = PyBytes_Check(o); > - is_unicode = PyUnicode_Check(o); > - > - if (!is_index && !is_bytes && !is_unicode) { > - _Py_IDENTIFIER(__fspath__); > - PyObject *func; > - > - func = _PyObject_LookupSpecial(o, &PyId___fspath__); > - if (func == NULL) > - goto err_format; > - Py_DECREF(o); > - o = PyObject_CallFunctionObjArgs(func, NULL); > - Py_DECREF(func); > - if (o == NULL) > + fd_obj = PyNumber_Index(o); > + if (!fd_obj) > return 0; > - is_bytes = PyBytes_Check(o); > - is_unicode = PyUnicode_Check(o); > - } > - > - if (is_unicode) { > - if (!PyUnicode_FSConverter(o, &bytes)) > - goto err; > - } else if (is_bytes) { > - bytes = o; > - Py_INCREF(bytes); > - } else if (is_index) { > - if (!fd_converter(o, &path->fd)) This removes the last use of fd_converter module.c:22:12: warning: ‘fd_converter’ defined but not used [-Wunused-function] 22 | static int fd_converter(PyObject *o, void *p) I'll ifdef it out as it will be probably needed in the future.
diff --git a/libbtrfsutil/python/btrfsutilpy.h b/libbtrfsutil/python/btrfsutilpy.h index ee70c23a..49702dcc 100644 --- a/libbtrfsutil/python/btrfsutilpy.h +++ b/libbtrfsutil/python/btrfsutilpy.h @@ -40,16 +40,13 @@ extern PyTypeObject SubvolumeInfo_type; extern PyTypeObject SubvolumeIterator_type; extern PyTypeObject QgroupInherit_type; -/* - * Helpers for path arguments based on posixmodule.c in CPython. - */ struct path_arg { bool allow_fd; - char *path; int fd; + char *path; Py_ssize_t length; PyObject *object; - PyObject *cleanup; + PyObject *bytes; }; int path_converter(PyObject *o, void *p); void path_cleanup(struct path_arg *path); diff --git a/libbtrfsutil/python/module.c b/libbtrfsutil/python/module.c index 2657ee28..9b0b86b5 100644 --- a/libbtrfsutil/python/module.c +++ b/libbtrfsutil/python/module.c @@ -44,85 +44,49 @@ static int fd_converter(PyObject *o, void *p) int path_converter(PyObject *o, void *p) { struct path_arg *path = p; - int is_index, is_bytes, is_unicode; - PyObject *bytes = NULL; - Py_ssize_t length = 0; - char *tmp; if (o == NULL) { path_cleanup(p); return 1; } - path->object = path->cleanup = NULL; - Py_INCREF(o); - path->fd = -1; + path->path = NULL; + path->length = 0; + path->bytes = NULL; + if (path->allow_fd && PyIndex_Check(o)) { + PyObject *fd_obj; + int overflow; + long fd; - is_index = path->allow_fd && PyIndex_Check(o); - is_bytes = PyBytes_Check(o); - is_unicode = PyUnicode_Check(o); - - if (!is_index && !is_bytes && !is_unicode) { - _Py_IDENTIFIER(__fspath__); - PyObject *func; - - func = _PyObject_LookupSpecial(o, &PyId___fspath__); - if (func == NULL) - goto err_format; - Py_DECREF(o); - o = PyObject_CallFunctionObjArgs(func, NULL); - Py_DECREF(func); - if (o == NULL) + fd_obj = PyNumber_Index(o); + if (!fd_obj) return 0; - is_bytes = PyBytes_Check(o); - is_unicode = PyUnicode_Check(o); - } - - if (is_unicode) { - if (!PyUnicode_FSConverter(o, &bytes)) - goto err; - } else if (is_bytes) { - bytes = o; - Py_INCREF(bytes); - } else if (is_index) { - if (!fd_converter(o, &path->fd)) - goto err; - path->path = NULL; - goto out; + fd = PyLong_AsLongAndOverflow(fd_obj, &overflow); + Py_DECREF(fd_obj); + if (fd == -1 && PyErr_Occurred()) + return 0; + if (overflow > 0 || fd > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "fd is greater than maximum"); + return 0; + } + if (fd < 0) { + PyErr_SetString(PyExc_ValueError, "fd is negative"); + return 0; + } + path->fd = fd; } else { -err_format: - PyErr_Format(PyExc_TypeError, "expected %s, not %s", - path->allow_fd ? "string, bytes, os.PathLike, or integer" : - "string, bytes, or os.PathLike", - Py_TYPE(o)->tp_name); - goto err; + if (!PyUnicode_FSConverter(o, &path->bytes)) { + path->object = path->bytes = NULL; + return 0; + } + path->path = PyBytes_AS_STRING(path->bytes); + path->length = PyBytes_GET_SIZE(path->bytes); } - - length = PyBytes_GET_SIZE(bytes); - tmp = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(tmp)) { - PyErr_SetString(PyExc_TypeError, - "path has embedded nul character"); - goto err; - } - - path->path = tmp; - if (bytes == o) - Py_DECREF(bytes); - else - path->cleanup = bytes; - path->fd = -1; - -out: - path->length = length; + Py_INCREF(o); path->object = o; return Py_CLEANUP_SUPPORTED; - -err: - Py_XDECREF(o); - Py_XDECREF(bytes); - return 0; } PyObject *list_from_uint64_array(const uint64_t *arr, size_t n) @@ -150,8 +114,8 @@ PyObject *list_from_uint64_array(const uint64_t *arr, size_t n) void path_cleanup(struct path_arg *path) { + Py_CLEAR(path->bytes); Py_CLEAR(path->object); - Py_CLEAR(path->cleanup); } static PyMethodDef btrfsutil_methods[] = {