diff --git a/lib/oelite/cmd/__init__.py b/lib/oelite/cmd/__init__.py index 57090676..ae994c8d 100644 --- a/lib/oelite/cmd/__init__.py +++ b/lib/oelite/cmd/__init__.py @@ -1 +1 @@ -manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer" ] +manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer", "test" ] diff --git a/lib/oelite/cmd/test.py b/lib/oelite/cmd/test.py new file mode 100644 index 00000000..1f64d3c9 --- /dev/null +++ b/lib/oelite/cmd/test.py @@ -0,0 +1,221 @@ +import errno +import fcntl +import hashlib +import os +import select +import shutil +import signal +import sys +import tempfile +import time +import unittest + +import oelite.util +import oelite.signal +import oelite.compat + +description = "Run tests of internal utility functions" +def add_parser_options(parser): + parser.add_option("-s", "--show", + action="store_true", default=False, + help="Show list of tests") + +class OEliteTest(unittest.TestCase): + def setUp(self): + self.wd = tempfile.mkdtemp() + os.chdir(self.wd) + + def tearDown(self): + os.chdir("/") + shutil.rmtree(self.wd) + + def test_makedirs(self): + """Test the semantics of oelite.util.makedirs""" + + makedirs = oelite.util.makedirs + touch = oelite.util.touch + self.assertIsNone(makedirs("x")) + self.assertIsNone(makedirs("x")) + self.assertIsNone(makedirs("y/")) + self.assertIsNone(makedirs("y/")) + self.assertIsNone(makedirs("x/y/z")) + # One can create multiple leaf directories in one go; mkdir -p + # behaves the same way. + self.assertIsNone(makedirs("z/.././z//w//../v")) + self.assertTrue(os.path.isdir("z/w")) + self.assertTrue(os.path.isdir("z/v")) + + self.assertIsNone(touch("x/a")) + with self.assertRaises(OSError) as cm: + makedirs("x/a") + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + with self.assertRaises(OSError) as cm: + makedirs("x/a/z") + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + + self.assertIsNone(os.symlink("a", "x/b")) + with self.assertRaises(OSError) as cm: + makedirs("x/b") + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + with self.assertRaises(OSError) as cm: + makedirs("x/b/z") + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + + self.assertIsNone(os.symlink("../y", "x/c")) + self.assertIsNone(makedirs("x/c")) + self.assertIsNone(makedirs("x/c/")) + + self.assertIsNone(os.symlink("nowhere", "broken")) + with self.assertRaises(OSError) as cm: + makedirs("broken") + self.assertEqual(cm.exception.errno, errno.ENOENT) + + self.assertIsNone(os.symlink("loop1", "loop2")) + self.assertIsNone(os.symlink("loop2", "loop1")) + with self.assertRaises(OSError) as cm: + makedirs("loop1") + self.assertEqual(cm.exception.errno, errno.ELOOP) + + def test_cloexec(self): + open_cloexec = oelite.compat.open_cloexec + dup_cloexec = oelite.compat.dup_cloexec + + def has_cloexec(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + return (flags & fcntl.FD_CLOEXEC) != 0 + + fd = open_cloexec("/dev/null", os.O_RDONLY) + self.assertGreaterEqual(fd, 0) + self.assertTrue(has_cloexec(fd)) + + fd2 = os.dup(fd) + self.assertGreaterEqual(fd2, 0) + self.assertFalse(has_cloexec(fd2)) + self.assertIsNone(os.close(fd2)) + + fd2 = dup_cloexec(fd) + self.assertGreaterEqual(fd2, 0) + self.assertTrue(has_cloexec(fd2)) + self.assertIsNone(os.close(fd2)) + + self.assertIsNone(os.close(fd)) + + def test_hash_file(self): + testv = [(0, "d41d8cd98f00b204e9800998ecf8427e", "da39a3ee5e6b4b0d3255bfef95601890afd80709"), + (1, "0cc175b9c0f1b6a831c399e269772661", "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"), + (1000, "cabe45dcc9ae5b66ba86600cca6b8ba8", "291e9a6c66994949b57ba5e650361e98fc36b1ba"), + (1000000, "7707d6ae4e027c70eea2a935c2296f21", "34aa973cd4c4daa4f61eeb2bdbad27316534016f")] + hash_file = oelite.util.hash_file + + for size, md5, sha1 in testv: + # open and say "aaaa...." :-) + with tempfile.NamedTemporaryFile() as tmp: + self.assertIsNone(tmp.write("a"*size)) + self.assertIsNone(tmp.flush()) + self.assertEqual(os.path.getsize(tmp.name), size) + + h = hash_file(hashlib.md5(), tmp.name).hexdigest() + self.assertEqual(h, md5) + + h = hash_file(hashlib.sha1(), tmp.name).hexdigest() + self.assertEqual(h, sha1) + +class MakedirsRaceTest(OEliteTest): + def child(self): + signal.alarm(2) # just in case of infinite recursion bugs + try: + # wait for go + select.select([self.r], [], [], 1) + oelite.util.makedirs(self.path) + # no exception? all right + res = "OK" + except OSError as e: + # errno.errorcode(errno.ENOENT) == "ENOENT" etc. + res = errno.errorcode.get(e.errno) or str(e.errno) + except Exception as e: + res = "??" + finally: + # Short pipe writes are guaranteed atomic + os.write(self.w, res+"\n") + os._exit(0) + + def setUp(self): + super(MakedirsRaceTest, self).setUp() + self.path = "x/" * 10 + self.r, self.w = os.pipe() + self.children = [] + for i in range(8): + pid = os.fork() + if pid == 0: + self.child() + self.children.append(pid) + + def runTest(self): + """Test concurrent calls of oelite.util.makedirs""" + + os.write(self.w, "go go go\n") + time.sleep(0.01) + os.close(self.w) + with os.fdopen(self.r) as f: + v = [v.strip() for v in f] + d = {x: v.count(x) for x in v if x != "go go go"} + # On failure this won't give a very user-friendly error + # message, but it should contain information about the errors + # encountered. + self.assertEqual(d, {"OK": len(self.children)}) + self.assertTrue(os.path.isdir(self.path)) + + def tearDown(self): + for pid in self.children: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + super(MakedirsRaceTest, self).tearDown() + +class SigPipeTest(OEliteTest): + def run_sub(self, preexec_fn): + from subprocess import PIPE, Popen + + sub = Popen(["yes"], stdout=PIPE, stderr=PIPE, + preexec_fn = preexec_fn) + # Force a broken pipe. + sub.stdout.close() + err = sub.stderr.read() + ret = sub.wait() + return (ret, err) + + @unittest.skipIf(sys.version_info >= (3, 2), "Python is new enough") + def test_no_restore(self): + """Check that subprocesses inherit the SIG_IGNORE disposition for SIGPIPE.""" + (ret, err) = self.run_sub(None) + # This should terminate with a write error; we assume that + # 'yes' is so well-behaved that it both exits with a non-zero + # exit code as well as prints an error message containing + # strerror(errno). + self.assertGreater(ret, 0) + self.assertIn(os.strerror(errno.EPIPE), err) + + def test_restore(self): + """Check that oelite.signal.restore_defaults resets the SIGPIPE disposition.""" + (ret, err) = self.run_sub(oelite.signal.restore_defaults) + # This should terminate due to SIGPIPE, and not get a chance + # to write to stderr. + self.assertEqual(ret, -signal.SIGPIPE) + self.assertEqual(err, "") + +def run(options, args, config): + suite = unittest.TestSuite() + suite.addTest(MakedirsRaceTest()) + suite.addTest(OEliteTest('test_makedirs')) + suite.addTest(SigPipeTest('test_no_restore')) + suite.addTest(SigPipeTest('test_restore')) + suite.addTest(OEliteTest('test_cloexec')) + suite.addTest(OEliteTest('test_hash_file')) + + if options.show: + for t in suite: + print str(t), "--", t.shortDescription() + return 0 + runner = unittest.TextTestRunner(verbosity=3) + runner.run(suite) + + return 0 diff --git a/lib/oelite/compat.py b/lib/oelite/compat.py index 8451f4b2..f384aaa3 100755 --- a/lib/oelite/compat.py +++ b/lib/oelite/compat.py @@ -57,25 +57,3 @@ def open_cloexec_fallback(filename, flag, mode=0777): except KeyError: dup_cloexec = dup_cloexec_fallback - -def has_cloexec(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - return (flags & fcntl.FD_CLOEXEC) != 0 - -def test_open_cloexec(): - fd = open_cloexec("/dev/null", os.O_RDONLY) - assert(has_cloexec(fd)) - os.close(fd) - fd = open_cloexec("/dev/null", os.O_WRONLY) - assert(has_cloexec(fd)) - os.close(fd) - -def test_dup_cloexec(): - fd = dup_cloexec(sys.stdin.fileno()) - assert(has_cloexec(fd)) - os.close(fd) - - -if __name__ == "__main__": - test_open_cloexec() - test_dup_cloexec() diff --git a/lib/oelite/fetch/fetch.py b/lib/oelite/fetch/fetch.py index 0242326a..97f69600 100644 --- a/lib/oelite/fetch/fetch.py +++ b/lib/oelite/fetch/fetch.py @@ -228,29 +228,6 @@ def signature(self): print "%s: no checksum known for %s"%(self.recipe, url) return "" - def cache(self): - if not "cache" in dir(self.fetcher): - return True - return self.fetcher.cache() - - def write_checksum(self, filepath): - md5path = filepath + ".md5" - checksum = hashlib.md5() - with open(filepath) as f: - checksum.update(f.read()) - with open(md5path, "w") as f: - f.write(checksum.digest()) - - def verify_checksum(self, filepath): - md5path = filepath + ".md5" - if not os.path.exists(md5path): - return None - checksum = hashlib.md5() - with open(filepath) as f: - checksum.update(f.read()) - with open(md5path) as f: - return f.readline().strip() == checksum.digest() - def fetch(self): if not "fetch" in dir(self.fetcher): return True @@ -328,13 +305,11 @@ def mirror(self, d, mirror): dst = os.path.join(mirror, src[len(self.ingredients)+1:]) if os.path.exists(dst): m = hashlib.md5() - with open(src, "r") as srcfile: - m.update(srcfile.read()) - src_md5 = m.hexdigest() + oelite.util.hash_file(m, src) + src_md5 = m.hexdigest() m = hashlib.md5() - with open(dst, "r") as dstfile: - m.update(dstfile.read()) - dst_md5 = m.hexdigest() + oelite.util.hash_file(m, dst) + dst_md5 = m.hexdigest() if src_md5 != dst_md5: print "Mirror inconsistency:", dst print "%s != %s"%(src_md5, dst_md5) diff --git a/lib/oelite/fetch/local.py b/lib/oelite/fetch/local.py index a43f2367..789fc871 100644 --- a/lib/oelite/fetch/local.py +++ b/lib/oelite/fetch/local.py @@ -1,5 +1,6 @@ import oelite.fetch import oelite.path +import oelite.util import os import hashlib @@ -35,6 +36,6 @@ def signature(self): if os.path.isdir(self.localpath): raise oelite.fetch.NoSignature(self.uri, "can't compute directory signature") m = hashlib.sha1() - m.update(open(self.localpath, "r").read()) + oelite.util.hash_file(m, self.localpath) self._signature = m.digest() return self._signature diff --git a/lib/oelite/fetch/url.py b/lib/oelite/fetch/url.py index c3d69d02..75670d25 100644 --- a/lib/oelite/fetch/url.py +++ b/lib/oelite/fetch/url.py @@ -83,7 +83,7 @@ def grabbedsignature(self): def localsignature(self): m = hashlib.sha1() - m.update(open(self.localpath, "r").read()) + oelite.util.hash_file(m, self.localpath) return m.hexdigest() def get_proxies(self, d): @@ -119,8 +119,7 @@ def grab(url, filename, timeout=120, retry=5, proxies=None, passive_ftp=True): d = os.path.dirname(filename) f = os.path.basename(filename) - if not os.path.exists(d): - os.makedirs(d) + oelite.util.makedirs(d) # Use mkstemp to create and open a guaranteed unique file. We use # the file descriptor as wget's stdout. We must download to the diff --git a/lib/oelite/signal.py b/lib/oelite/signal.py index a7fd30cf..582aea44 100644 --- a/lib/oelite/signal.py +++ b/lib/oelite/signal.py @@ -30,34 +30,3 @@ def restore_defaults(): signal.signal(signal.SIGXFZ, signal.SIG_DFL) if hasattr(signal, "SIGXFSZ"): signal.signal(signal.SIGXFSZ, signal.SIG_DFL) - -def test_restore(): - import os - import subprocess - import errno - - PIPE=subprocess.PIPE - sub = subprocess.Popen(["yes"], stdout=PIPE, stderr=PIPE) - # Force a broken pipe. - sub.stdout.close() - err = sub.stderr.read() - # This should terminate with a write error; we assume that 'yes' - # is so well-behaved that it both exits with a non-zero exit code - # as well as prints an error message containing strerror(errno). - ret = sub.wait() - assert(ret > 0) - assert(os.strerror(errno.EPIPE) in err) - - sub = subprocess.Popen(["yes"], stdout=PIPE, stderr=PIPE, preexec_fn = restore_defaults) - # Force a broken pipe. - sub.stdout.close() - err = sub.stderr.read() - # This should terminate due to SIGPIPE, and not get a chance to write to stderr. - ret = sub.wait() - assert(ret == -signal.SIGPIPE) - assert(err == "") - -if __name__ == "__main__": - # To run: - # meta/core/lib$ python -m oelite.signal - test_restore() diff --git a/lib/oelite/util.py b/lib/oelite/util.py index 016c0e32..34eda1a2 100644 --- a/lib/oelite/util.py +++ b/lib/oelite/util.py @@ -4,6 +4,7 @@ import sys import subprocess import time +import errno from oelite.compat import dup_cloexec import oelite.signal @@ -208,10 +209,29 @@ def unique_list(seq): def makedirs(path, mode=0777): - if os.path.exists(path): - return - os.makedirs(path, mode) - return + # Create the directory 'path' and all necessary intermediate + # directories. The complexity stems from trying to ensure no + # exception is raised even if two processes concurrently try to do + # this operation. + while True: + try: + return os.mkdir(path, mode) + except OSError as e: + if e.errno == errno.EEXIST: + # Ensure that 'path' is a directory or a symlink to + # one. This will raise ENOTDIR here instead of later + # when the caller tries to create a file inside + # 'path'. + os.stat(path + "/.") + return None + elif e.errno == errno.ENOENT and path != "": + # makedirs("") must raise ENOENT, hence the second + # condition. We force u+rwx on all parent + # directories, anything else would be silly. + makedirs(os.path.dirname(path), mode | 0o700) + continue + else: + raise def touch(path, makedirs=False, truncate=False): @@ -245,3 +265,17 @@ def timing_info(msg, start): msg += pretty_time(delta) info(msg) return + +def chunk_iter(fn, chunk_size = 32768): + with open(fn) as f: + while True: + chunk = f.read(chunk_size) + if chunk: + yield chunk + else: + return + +def hash_file(hasher, fn): + for chunk in chunk_iter(fn): + hasher.update(chunk) + return hasher