From 91f27cb441cb6a6050830b87793ef29b4ce97b89 Mon Sep 17 00:00:00 2001 From: Iustin Pop Date: Sun, 28 Oct 2018 19:02:48 +0100 Subject: [PATCH] New upstream version 0.6.1 --- Makefile | 23 +++++++++- NEWS | 22 ++++++++++ PKG-INFO | 2 +- README.rst | 9 ++-- doc/conf.py | 4 +- doc/news.rst | 22 ++++++++++ pyxattr.egg-info/PKG-INFO | 2 +- setup.py | 4 +- test/test_xattr.py | 88 ++++++++++++++++++++++----------------- xattr.c | 56 +++++++++++++++---------- 10 files changed, 157 insertions(+), 75 deletions(-) diff --git a/Makefile b/Makefile index 905791d..141685f 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ ALLSPHINXOPTS = -d $(DOCTREES) $(SPHINXOPTS) $(DOCDIR) MODNAME = xattr.so RSTFILES = doc/index.rst doc/module.rst NEWS README.rst doc/conf.py +PYVERS = 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 +REPS = 5 all: doc test @@ -23,7 +25,7 @@ dist: fakeroot ./setup.py sdist test: - @for ver in 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5; do \ + @for ver in $(PYVERS); do \ for flavour in "" "-dbg"; do \ if type python$$ver$$flavour >/dev/null; then \ echo Testing with python$$ver$$flavour; \ @@ -36,6 +38,25 @@ test: pypy ./setup.py test -q; \ fi +benchmark: $(MODNAME) + @set -e; \ + TESTFILE=`mktemp`;\ + trap 'rm $$TESTFILE' EXIT; \ + for ver in $(PYVERS) ; do \ + if type python$$ver >/dev/null; then \ + echo Benchmarking with python$$ver; \ + python$$ver ./setup.py build -q; \ + echo " - set (with override)"; \ + python$$ver -m timeit -r $(REPS) -s 'import xattr' "xattr.set('$$TESTFILE', 'user.comment', 'hello')"; \ + echo " - list"; \ + python$$ver -m timeit -r $(REPS) -s 'import xattr' "xattr.list('$$TESTFILE')"; \ + echo " - get"; \ + python$$ver -m timeit -r $(REPS) -s 'import xattr' "xattr.get('$$TESTFILE', 'user.comment')"; \ + echo " - set + remove"; \ + python$$ver -m timeit -r $(REPS) -s 'import xattr' "xattr.set('$$TESTFILE', 'user.comment', 'hello'); xattr.remove('$$TESTFILE', 'user.comment')"; \ + fi; \ + done; + coverage: $(MAKE) clean $(MAKE) test CFLAGS="-coverage" diff --git a/NEWS b/NEWS index f9002c7..6bee508 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,28 @@ News ==== +Version 0.6.1 +------------- + +*released Tue, 24 Jul 2018* + +Minor bugfix, performance and compatibility release. + +* Minor compatibility fix: on Linux, drop the use of the `attr` library, + and instead switch to the glibc header `sys/xattr.h`, which is + provided for a really long time (since glibc 2.3). The formerly used + header `attr/xattr.h` has been removed from the `attr` library in + version 2.4.48. Fix provided by Lars Wendler, many thanks! +* Release the GIL when performing I/O. Patch proposed by xwhuang, many + thanks. I tested this a long while back it seemed to impact + performance on local filesystems, but upon further inspection, the + downsides are minor (between 0 and 5%, in many cases negligible). For + remote or slow filesystems, this should allow much increased + parallelism. +* Fix symlink set operation on MacOS X; bugfix provided by adamlin, much + appreciated! This also uncovered testing problems related to symlinks, + which are now fixed (the bug would be caught by the updated tests). + Version 0.6.0 ------------- diff --git a/PKG-INFO b/PKG-INFO index 0e2792c..525c3b3 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: pyxattr -Version: 0.6.0 +Version: 0.6.1 Summary: Filesystem extended attributes for python Home-page: http://pyxattr.k1024.org/ Author: Iustin Pop diff --git a/README.rst b/README.rst index 2844a61..033d597 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ This is the pyxattr module, a Python extension module which gives access to the extended attributes for filesystem objects available in some operating systems. -Downloads: go to http://pyxattr.k1024.org/downloads/. Latest -version is 0.6.0. The source repository is either at +Downloads: go to https://pyxattr.k1024.org/downloads/. Latest +version is 0.6.1. The source repository is either at http://git.k1024.org/pyxattr.git or at https://github.com/iustin/pyxattr. @@ -17,9 +17,8 @@ pyxattr has been written and tested on Linux, kernel v2.4 or later, with XFS filesystems; ext2/ext3 should work also. If any other platform implements the same behavior, pyxattr could be used. -You need to have the attr library (including development headers; most -distributions should have this, under various names) and setuptools -installed in order to build and install the module. +You need to have the setuptools tool installed in order to build and +install the module. License ------- diff --git a/doc/conf.py b/doc/conf.py index e6c994a..a2ac56a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,9 +48,9 @@ copyright = u'2002, 2003, 2006, 2008, 2012, 2013, 2014, 2015, Iustin Pop' # built documents. # # The short X.Y version. -version = '0.6.0' +version = '0.6.1' # The full version, including alpha/beta/rc tags. -release = '0.6.0' +release = '0.6.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/news.rst b/doc/news.rst index f9002c7..6bee508 100644 --- a/doc/news.rst +++ b/doc/news.rst @@ -1,6 +1,28 @@ News ==== +Version 0.6.1 +------------- + +*released Tue, 24 Jul 2018* + +Minor bugfix, performance and compatibility release. + +* Minor compatibility fix: on Linux, drop the use of the `attr` library, + and instead switch to the glibc header `sys/xattr.h`, which is + provided for a really long time (since glibc 2.3). The formerly used + header `attr/xattr.h` has been removed from the `attr` library in + version 2.4.48. Fix provided by Lars Wendler, many thanks! +* Release the GIL when performing I/O. Patch proposed by xwhuang, many + thanks. I tested this a long while back it seemed to impact + performance on local filesystems, but upon further inspection, the + downsides are minor (between 0 and 5%, in many cases negligible). For + remote or slow filesystems, this should allow much increased + parallelism. +* Fix symlink set operation on MacOS X; bugfix provided by adamlin, much + appreciated! This also uncovered testing problems related to symlinks, + which are now fixed (the bug would be caught by the updated tests). + Version 0.6.0 ------------- diff --git a/pyxattr.egg-info/PKG-INFO b/pyxattr.egg-info/PKG-INFO index 0e2792c..525c3b3 100644 --- a/pyxattr.egg-info/PKG-INFO +++ b/pyxattr.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: pyxattr -Version: 0.6.0 +Version: 0.6.1 Summary: Filesystem extended attributes for python Home-page: http://pyxattr.k1024.org/ Author: Iustin Pop diff --git a/setup.py b/setup.py index 54259db..4cd944b 100755 --- a/setup.py +++ b/setup.py @@ -10,12 +10,10 @@ except ImportError: long_desc = """This is a C extension module for Python which implements extended attributes manipulation. It is a wrapper on top of the attr C library - see attr(5).""" -version = "0.6.0" +version = "0.6.1" author = "Iustin Pop" author_email = "iustin@k1024.org" libraries = [] -if platform.system() == 'Linux': - libraries.append("attr") macros = [ ("_XATTR_VERSION", '"%s"' % version), ("_XATTR_AUTHOR", '"%s"' % author), diff --git a/test/test_xattr.py b/test/test_xattr.py index 084b831..7b14eec 100644 --- a/test/test_xattr.py +++ b/test/test_xattr.py @@ -10,6 +10,8 @@ import errno import xattr from xattr import NS_USER, XATTR_CREATE, XATTR_REPLACE +NAMESPACE = os.environ.get("NAMESPACE", NS_USER) + if sys.hexversion >= 0x03000000: PY3K = True EMPTY_NS = bytes() @@ -29,7 +31,7 @@ else: class xattrTest(unittest.TestCase): USER_NN = "test" - USER_ATTR = NS_USER.decode() + "." + USER_NN + USER_ATTR = NAMESPACE.decode() + "." + USER_NN USER_VAL = "abc" EMPTY_VAL = "" LARGE_VAL = "x" * 2048 @@ -108,7 +110,7 @@ class xattrTest(unittest.TestCase): self.checkList(xattr.listxattr(item, symlink), []) self.assertRaises(EnvironmentError, xattr.setxattr, item, self.USER_ATTR, self.USER_VAL, - XATTR_REPLACE) + XATTR_REPLACE, symlink) try: xattr.setxattr(item, self.USER_ATTR, self.USER_VAL, 0, symlink) except IOError: @@ -121,31 +123,35 @@ class xattrTest(unittest.TestCase): return raise self.assertRaises(EnvironmentError, xattr.setxattr, item, - self.USER_ATTR, self.USER_VAL, XATTR_CREATE) + self.USER_ATTR, self.USER_VAL, XATTR_CREATE, symlink) self.checkList(xattr.listxattr(item, symlink), [self.USER_ATTR]) self.assertEqual(xattr.getxattr(item, self.USER_ATTR, symlink), self.USER_VAL) self.checkTuples(xattr.get_all(item, nofollow=symlink), [(self.USER_ATTR, self.USER_VAL)]) - xattr.removexattr(item, self.USER_ATTR) + xattr.removexattr(item, self.USER_ATTR, symlink) self.checkList(xattr.listxattr(item, symlink), []) self.checkTuples(xattr.get_all(item, nofollow=symlink), []) self.assertRaises(EnvironmentError, xattr.removexattr, - item, self.USER_ATTR) + item, self.USER_ATTR, symlink) def _checkListSetGet(self, item, symlink=False, use_ns=False): """check list, set, get operations against an item""" self.checkList(xattr.list(item, symlink), []) self.assertRaises(EnvironmentError, xattr.set, item, - self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE) + self.USER_ATTR, self.USER_VAL, + flags=XATTR_REPLACE, + nofollow=symlink) self.assertRaises(EnvironmentError, xattr.set, item, - self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE, - namespace=NS_USER) + self.USER_NN, self.USER_VAL, + flags=XATTR_REPLACE, + namespace=NAMESPACE, + nofollow=symlink) try: if use_ns: xattr.set(item, self.USER_NN, self.USER_VAL, - namespace=NS_USER, + namespace=NAMESPACE, nofollow=symlink) else: xattr.set(item, self.USER_ATTR, self.USER_VAL, @@ -160,36 +166,40 @@ class xattrTest(unittest.TestCase): return raise self.assertRaises(EnvironmentError, xattr.set, item, - self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE) + self.USER_ATTR, self.USER_VAL, + flags=XATTR_CREATE, + nofollow=symlink) self.assertRaises(EnvironmentError, xattr.set, item, self.USER_NN, self.USER_VAL, - flags=XATTR_CREATE, namespace=NS_USER) + flags=XATTR_CREATE, + namespace=NAMESPACE, + nofollow=symlink) self.checkList(xattr.list(item, nofollow=symlink), [self.USER_ATTR]) self.checkList(xattr.list(item, nofollow=symlink, - namespace=EMPTY_NS), - [self.USER_ATTR]) - self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink), + namespace=EMPTY_NS), + [self.USER_ATTR]) + self.assertEqual(xattr.list(item, namespace=NAMESPACE, nofollow=symlink), [self.USER_NN]) self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink), self.USER_VAL) self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink, - namespace=NS_USER), self.USER_VAL) + namespace=NAMESPACE), self.USER_VAL) self.checkTuples(xattr.get_all(item, nofollow=symlink), [(self.USER_ATTR, self.USER_VAL)]) self.assertEqual(xattr.get_all(item, nofollow=symlink, - namespace=NS_USER), + namespace=NAMESPACE), [(self.USER_NN, self.USER_VAL)]) if use_ns: - xattr.remove(item, self.USER_NN, namespace=NS_USER) + xattr.remove(item, self.USER_NN, namespace=NAMESPACE, nofollow=symlink) else: - xattr.remove(item, self.USER_ATTR) - self.checkList(xattr.list(item, symlink), []) + xattr.remove(item, self.USER_ATTR, nofollow=symlink) + self.checkList(xattr.list(item, nofollow=symlink), []) self.checkTuples(xattr.get_all(item, nofollow=symlink), []) self.assertRaises(EnvironmentError, xattr.remove, item, self.USER_ATTR, nofollow=symlink) self.assertRaises(EnvironmentError, xattr.remove, item, - self.USER_NN, namespace=NS_USER, nofollow=symlink) + self.USER_NN, namespace=NAMESPACE, nofollow=symlink) def testNoXattrDeprecated(self): """test no attributes (deprecated functions)""" @@ -214,27 +224,27 @@ class xattrTest(unittest.TestCase): """test no attributes""" fh, fname = self._getfile() self.checkList(xattr.list(fname), []) - self.assertEqual(xattr.list(fname, namespace=NS_USER), []) + self.assertEqual(xattr.list(fname, namespace=NAMESPACE), []) self.checkTuples(xattr.get_all(fname), []) - self.assertEqual(xattr.get_all(fname, namespace=NS_USER), []) + self.assertEqual(xattr.get_all(fname, namespace=NAMESPACE), []) self.assertRaises(EnvironmentError, xattr.get, fname, - self.USER_NN, namespace=NS_USER) + self.USER_NN, namespace=NAMESPACE) dname = self._getdir() self.checkList(xattr.list(dname), []) - self.assertEqual(xattr.list(dname, namespace=NS_USER), []) + self.assertEqual(xattr.list(dname, namespace=NAMESPACE), []) self.checkTuples(xattr.get_all(dname), []) - self.assertEqual(xattr.get_all(dname, namespace=NS_USER), []) + self.assertEqual(xattr.get_all(dname, namespace=NAMESPACE), []) self.assertRaises(EnvironmentError, xattr.get, dname, - self.USER_NN, namespace=NS_USER) + self.USER_NN, namespace=NAMESPACE) _, sname = self._getsymlink() self.checkList(xattr.list(sname, nofollow=True), []) self.assertEqual(xattr.list(sname, nofollow=True, - namespace=NS_USER), []) + namespace=NAMESPACE), []) self.checkTuples(xattr.get_all(sname, nofollow=True), []) self.assertEqual(xattr.get_all(sname, nofollow=True, - namespace=NS_USER), []) + namespace=NAMESPACE), []) self.assertRaises(EnvironmentError, xattr.get, sname, - self.USER_NN, namespace=NS_USER, nofollow=True) + self.USER_NN, namespace=NAMESPACE, nofollow=True) def testFileByNameDeprecated(self): """test set and retrieve one attribute by file name (deprecated)""" @@ -297,17 +307,17 @@ class xattrTest(unittest.TestCase): self.checkList(xattr.list(fname), []) xattr.set(fname, self.USER_ATTR, self.USER_VAL) self.checkList(xattr.list(fh), [self.USER_ATTR]) - self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN]) + self.assertEqual(xattr.list(fh, namespace=NAMESPACE), [self.USER_NN]) self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL) - self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER), + self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NAMESPACE), self.USER_VAL) self.checkTuples(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)]) - self.assertEqual(xattr.get_all(fo, namespace=NS_USER), + self.assertEqual(xattr.get_all(fo, namespace=NAMESPACE), [(self.USER_NN, self.USER_VAL)]) self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, self.USER_VAL)]) - self.assertEqual(xattr.get_all(fname, namespace=NS_USER), + self.assertEqual(xattr.get_all(fname, namespace=NAMESPACE), [(self.USER_NN, self.USER_VAL)]) fo.close() @@ -371,12 +381,12 @@ class xattrTest(unittest.TestCase): BINVAL = BINVAL.encode() xattr.set(fname, self.USER_ATTR, BINVAL) self.checkList(xattr.list(fname), [self.USER_ATTR]) - self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN]) + self.assertEqual(xattr.list(fname, namespace=NAMESPACE), [self.USER_NN]) self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL) self.assertEqual(xattr.get(fname, self.USER_NN, - namespace=NS_USER), BINVAL) + namespace=NAMESPACE), BINVAL) self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)]) - self.assertEqual(xattr.get_all(fname, namespace=NS_USER), + self.assertEqual(xattr.get_all(fname, namespace=NAMESPACE), [(self.USER_NN, BINVAL)]) xattr.remove(fname, self.USER_ATTR) @@ -402,15 +412,15 @@ class xattrTest(unittest.TestCase): for i in range(self.MANYOPS_COUNT): self.checkList(xattr.list(fh), VL) self.checkList(xattr.list(fh, namespace=EMPTY_NS), VL) - self.assertEqual(xattr.list(fh, namespace=NS_USER), VN) + self.assertEqual(xattr.list(fh, namespace=NAMESPACE), VN) for i in range(self.MANYOPS_COUNT): self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL) - self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER), + self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NAMESPACE), self.USER_VAL) for i in range(self.MANYOPS_COUNT): self.checkTuples(xattr.get_all(fh), [(self.USER_ATTR, self.USER_VAL)]) - self.assertEqual(xattr.get_all(fh, namespace=NS_USER), + self.assertEqual(xattr.get_all(fh, namespace=NAMESPACE), [(self.USER_NN, self.USER_VAL)]) def testNoneNamespace(self): diff --git a/xattr.c b/xattr.c index 111cec1..0087b7e 100644 --- a/xattr.c +++ b/xattr.c @@ -23,10 +23,8 @@ #define PY_SSIZE_T_CLEAN #include -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) #include -#elif defined(__linux__) -#include #endif #include @@ -235,7 +233,7 @@ static inline int _setxattr(const char *path, const char *name, const void *valu return setxattr(path, name, value, size, 0, flags); } static inline int _lsetxattr(const char *path, const char *name, const void *value, size_t size, int flags) { - return setxattr(path, name, value, size, 0, flags & XATTR_NOFOLLOW); + return setxattr(path, name, value, size, 0, flags | XATTR_NOFOLLOW); } static inline int _fsetxattr(int filedes, const char *name, const void *value, size_t size, int flags) { return fsetxattr(filedes, name, value, size, 0, flags); @@ -275,41 +273,58 @@ typedef ssize_t (*buf_getter)(target_t *tgt, const char *name, static ssize_t _list_obj(target_t *tgt, const char *unused, void *list, size_t size) { + ssize_t ret; + + Py_BEGIN_ALLOW_THREADS; if(tgt->type == T_FD) - return _flistxattr(tgt->fd, list, size); + ret = _flistxattr(tgt->fd, list, size); else if (tgt->type == T_LINK) - return _llistxattr(tgt->name, list, size); + ret = _llistxattr(tgt->name, list, size); else - return _listxattr(tgt->name, list, size); + ret = _listxattr(tgt->name, list, size); + Py_END_ALLOW_THREADS; + return ret; } static ssize_t _get_obj(target_t *tgt, const char *name, void *value, size_t size) { + ssize_t ret; + Py_BEGIN_ALLOW_THREADS; if(tgt->type == T_FD) - return _fgetxattr(tgt->fd, name, value, size); + ret = _fgetxattr(tgt->fd, name, value, size); else if (tgt->type == T_LINK) - return _lgetxattr(tgt->name, name, value, size); + ret = _lgetxattr(tgt->name, name, value, size); else - return _getxattr(tgt->name, name, value, size); + ret = _getxattr(tgt->name, name, value, size); + Py_END_ALLOW_THREADS; + return ret; } static int _set_obj(target_t *tgt, const char *name, const void *value, size_t size, int flags) { + int ret; + Py_BEGIN_ALLOW_THREADS; if(tgt->type == T_FD) - return _fsetxattr(tgt->fd, name, value, size, flags); + ret = _fsetxattr(tgt->fd, name, value, size, flags); else if (tgt->type == T_LINK) - return _lsetxattr(tgt->name, name, value, size, flags); + ret = _lsetxattr(tgt->name, name, value, size, flags); else - return _setxattr(tgt->name, name, value, size, flags); + ret = _setxattr(tgt->name, name, value, size, flags); + Py_END_ALLOW_THREADS; + return ret; } static int _remove_obj(target_t *tgt, const char *name) { + int ret; + Py_BEGIN_ALLOW_THREADS; if(tgt->type == T_FD) - return _fremovexattr(tgt->fd, name); + ret = _fremovexattr(tgt->fd, name); else if (tgt->type == T_LINK) - return _lremovexattr(tgt->name, name); + ret = _lremovexattr(tgt->name, name); else - return _removexattr(tgt->name, name); + ret = _removexattr(tgt->name, name); + Py_END_ALLOW_THREADS; + return ret; } /* Perform a get/list operation with appropriate buffer size, @@ -642,11 +657,7 @@ get_all(PyObject *self, PyObject *args, PyObject *keywds) /* Now retrieve the attribute value */ nval = _generic_get(_get_obj, &tgt, s, &buf_val, &nalloc, &io_errno); if (nval == -1) { - if ( -#ifdef ENODATA - io_errno == ENODATA || -#endif - io_errno == ENOATTR) { + if (io_errno == ENODATA) { PyErr_Clear(); continue; } else { @@ -1173,8 +1184,7 @@ static char __xattr_doc__[] = \ " a :exc:`EnvironmentError`; under\n" " Linux, the following ``errno`` values are used:\n" "\n" - " - ``ENOATTR`` and ``ENODATA`` mean that the attribute name is\n" - " invalid\n" + " - ``ENODATA`` means that the attribute name is\n invalid\n" " - ``ENOTSUP`` and ``EOPNOTSUPP`` mean that the filesystem does not\n" " support extended attributes, or that the namespace is invalid\n" " - ``E2BIG`` mean that the attribute value is too big\n" -- 2.39.2