add "oe test" subcommand

lib/oelite/cmd/__init__.py

@@ -1 +1 @@
-manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer" ]
+manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer", "test" ]

lib/oelite/cmd/test.py

@@ -0,0 +1,221 @@
+import errno
+import fcntl
+import hashlib
+import os
+import select
+import shutil
+import signal
+import sys
+import tempfile
+import time
+import unittest
+
+import oelite.util
+import oelite.signal
+import oelite.compat
+
+description = "Run tests of internal utility functions"
+def add_parser_options(parser):
+    parser.add_option("-s", "--show",
+                      action="store_true", default=False,
+                      help="Show list of tests")
+
+class OEliteTest(unittest.TestCase):
+    def setUp(self):
+        self.wd = tempfile.mkdtemp()
+        os.chdir(self.wd)
+
+    def tearDown(self):
+        os.chdir("/")
+        shutil.rmtree(self.wd)
+
+    def test_makedirs(self):
+        """Test the semantics of oelite.util.makedirs"""
+
+        makedirs = oelite.util.makedirs
+        touch = oelite.util.touch
+        self.assertIsNone(makedirs("x"))
+        self.assertIsNone(makedirs("x"))
+        self.assertIsNone(makedirs("y/"))
+        self.assertIsNone(makedirs("y/"))
+        self.assertIsNone(makedirs("x/y/z"))
+        # One can create multiple leaf directories in one go; mkdir -p
+        # behaves the same way.
+        self.assertIsNone(makedirs("z/.././z//w//../v"))
+        self.assertTrue(os.path.isdir("z/w"))
+        self.assertTrue(os.path.isdir("z/v"))
+
+        self.assertIsNone(touch("x/a"))
+        with self.assertRaises(OSError) as cm:
+            makedirs("x/a")
+        self.assertEqual(cm.exception.errno, errno.ENOTDIR)
+        with self.assertRaises(OSError) as cm:
+            makedirs("x/a/z")
+        self.assertEqual(cm.exception.errno, errno.ENOTDIR)
+
+        self.assertIsNone(os.symlink("a", "x/b"))
+        with self.assertRaises(OSError) as cm:
+            makedirs("x/b")
+        self.assertEqual(cm.exception.errno, errno.ENOTDIR)
+        with self.assertRaises(OSError) as cm:
+            makedirs("x/b/z")
+        self.assertEqual(cm.exception.errno, errno.ENOTDIR)
+
+        self.assertIsNone(os.symlink("../y", "x/c"))
+        self.assertIsNone(makedirs("x/c"))
+        self.assertIsNone(makedirs("x/c/"))
+
+        self.assertIsNone(os.symlink("nowhere", "broken"))
+        with self.assertRaises(OSError) as cm:
+            makedirs("broken")
+        self.assertEqual(cm.exception.errno, errno.ENOENT)
+
+        self.assertIsNone(os.symlink("loop1", "loop2"))
+        self.assertIsNone(os.symlink("loop2", "loop1"))
+        with self.assertRaises(OSError) as cm:
+            makedirs("loop1")
+        self.assertEqual(cm.exception.errno, errno.ELOOP)
+
+    def test_cloexec(self):
+        open_cloexec = oelite.compat.open_cloexec
+        dup_cloexec = oelite.compat.dup_cloexec
+
+        def has_cloexec(fd):
+            flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+            return (flags & fcntl.FD_CLOEXEC) != 0
+
+        fd = open_cloexec("/dev/null", os.O_RDONLY)
+        self.assertGreaterEqual(fd, 0)
+        self.assertTrue(has_cloexec(fd))
+
+        fd2 = os.dup(fd)
+        self.assertGreaterEqual(fd2, 0)
+        self.assertFalse(has_cloexec(fd2))
+        self.assertIsNone(os.close(fd2))
+
+        fd2 = dup_cloexec(fd)
+        self.assertGreaterEqual(fd2, 0)
+        self.assertTrue(has_cloexec(fd2))
+        self.assertIsNone(os.close(fd2))
+
+        self.assertIsNone(os.close(fd))
+
+    def test_hash_file(self):
+        testv = [(0, "d41d8cd98f00b204e9800998ecf8427e", "da39a3ee5e6b4b0d3255bfef95601890afd80709"),
+                 (1, "0cc175b9c0f1b6a831c399e269772661", "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"),
+                 (1000, "cabe45dcc9ae5b66ba86600cca6b8ba8", "291e9a6c66994949b57ba5e650361e98fc36b1ba"),
+                 (1000000, "7707d6ae4e027c70eea2a935c2296f21", "34aa973cd4c4daa4f61eeb2bdbad27316534016f")]
+        hash_file = oelite.util.hash_file
+
+        for size, md5, sha1 in testv:
+            # open and say "aaaa...." :-)
+            with tempfile.NamedTemporaryFile() as tmp:
+                self.assertIsNone(tmp.write("a"*size))
+                self.assertIsNone(tmp.flush())
+                self.assertEqual(os.path.getsize(tmp.name), size)
+
+                h = hash_file(hashlib.md5(), tmp.name).hexdigest()
+                self.assertEqual(h, md5)
+
+                h = hash_file(hashlib.sha1(), tmp.name).hexdigest()
+                self.assertEqual(h, sha1)
+
+class MakedirsRaceTest(OEliteTest):
+    def child(self):
+        signal.alarm(2) # just in case of infinite recursion bugs
+        try:
+            # wait for go
+            select.select([self.r], [], [], 1)
+            oelite.util.makedirs(self.path)
+            # no exception? all right
+            res = "OK"
+        except OSError as e:
+            # errno.errorcode(errno.ENOENT) == "ENOENT" etc.
+            res = errno.errorcode.get(e.errno) or str(e.errno)
+        except Exception as e:
+            res = "??"
+        finally:
+            # Short pipe writes are guaranteed atomic
+            os.write(self.w, res+"\n")
+            os._exit(0)
+
+    def setUp(self):
+        super(MakedirsRaceTest, self).setUp()
+        self.path = "x/" * 10
+        self.r, self.w = os.pipe()
+        self.children = []
+        for i in range(8):
+            pid = os.fork()
+            if pid == 0:
+                self.child()
+            self.children.append(pid)
+
+    def runTest(self):
+        """Test concurrent calls of oelite.util.makedirs"""
+
+        os.write(self.w, "go go go\n")
+        time.sleep(0.01)
+        os.close(self.w)
+        with os.fdopen(self.r) as f:
+            v = [v.strip() for v in f]
+        d = {x: v.count(x) for x in v if x != "go go go"}
+        # On failure this won't give a very user-friendly error
+        # message, but it should contain information about the errors
+        # encountered.
+        self.assertEqual(d, {"OK": len(self.children)})
+        self.assertTrue(os.path.isdir(self.path))
+
+    def tearDown(self):
+        for pid in self.children:
+            os.kill(pid, signal.SIGKILL)
+            os.waitpid(pid, 0)
+        super(MakedirsRaceTest, self).tearDown()
+
+class SigPipeTest(OEliteTest):
+    def run_sub(self, preexec_fn):
+        from subprocess import PIPE, Popen
+
+        sub = Popen(["yes"], stdout=PIPE, stderr=PIPE,
+                    preexec_fn = preexec_fn)
+        # Force a broken pipe.
+        sub.stdout.close()
+        err = sub.stderr.read()
+        ret = sub.wait()
+        return (ret, err)
+
+    @unittest.skipIf(sys.version_info >= (3, 2), "Python is new enough")
+    def test_no_restore(self):
+        """Check that subprocesses inherit the SIG_IGNORE disposition for SIGPIPE."""
+        (ret, err) = self.run_sub(None)
+        # This should terminate with a write error; we assume that
+        # 'yes' is so well-behaved that it both exits with a non-zero
+        # exit code as well as prints an error message containing
+        # strerror(errno).
+        self.assertGreater(ret, 0)
+        self.assertIn(os.strerror(errno.EPIPE), err)
+
+    def test_restore(self):
+        """Check that oelite.signal.restore_defaults resets the SIGPIPE disposition."""
+        (ret, err) = self.run_sub(oelite.signal.restore_defaults)
+        # This should terminate due to SIGPIPE, and not get a chance
+        # to write to stderr.
+        self.assertEqual(ret, -signal.SIGPIPE)
+        self.assertEqual(err, "")
+
+def run(options, args, config):
+    suite = unittest.TestSuite()
+    suite.addTest(MakedirsRaceTest())
+    suite.addTest(OEliteTest('test_makedirs'))
+    suite.addTest(SigPipeTest('test_no_restore'))
+    suite.addTest(SigPipeTest('test_restore'))
+    suite.addTest(OEliteTest('test_cloexec'))
+    suite.addTest(OEliteTest('test_hash_file'))
+
+    if options.show:
+        for t in suite:
+            print str(t), "--", t.shortDescription()
+        return 0
+    runner = unittest.TextTestRunner(verbosity=3)
+    runner.run(suite)
+
+    return 0

lib/oelite/compat.py

@@ -57,25 +57,3 @@ def open_cloexec_fallback(filename, flag, mode=0777):
     except KeyError:
         dup_cloexec = dup_cloexec_fallback
 
-
-def has_cloexec(fd):
-    flags = fcntl.fcntl(fd, fcntl.F_GETFD)
-    return (flags & fcntl.FD_CLOEXEC) != 0
-
-def test_open_cloexec():
-    fd = open_cloexec("/dev/null", os.O_RDONLY)
-    assert(has_cloexec(fd))
-    os.close(fd)
-    fd = open_cloexec("/dev/null", os.O_WRONLY)
-    assert(has_cloexec(fd))
-    os.close(fd)
-
-def test_dup_cloexec():
-    fd = dup_cloexec(sys.stdin.fileno())
-    assert(has_cloexec(fd))
-    os.close(fd)
-
-
-if __name__ == "__main__":
-    test_open_cloexec()
-    test_dup_cloexec()

lib/oelite/fetch/fetch.py

@@ -228,29 +228,6 @@ def signature(self):
             print "%s: no checksum known for %s"%(self.recipe, url)
             return ""
 
-    def cache(self):
-        if not "cache" in dir(self.fetcher):
-            return True
-        return self.fetcher.cache()
-
-    def write_checksum(self, filepath):
-        md5path = filepath + ".md5"
-        checksum = hashlib.md5()
-        with open(filepath) as f:
-            checksum.update(f.read())
-        with open(md5path, "w") as f:
-            f.write(checksum.digest())
-
-    def verify_checksum(self, filepath):
-        md5path = filepath + ".md5"
-        if not os.path.exists(md5path):
-            return None
-        checksum = hashlib.md5()
-        with open(filepath) as f:
-            checksum.update(f.read())
-        with open(md5path) as f:
-            return f.readline().strip() == checksum.digest()
-
     def fetch(self):
         if not "fetch" in dir(self.fetcher):
             return True
@@ -328,13 +305,11 @@ def mirror(self, d, mirror):
         dst = os.path.join(mirror, src[len(self.ingredients)+1:])
         if os.path.exists(dst):
             m = hashlib.md5()
-            with open(src, "r") as srcfile:
-                m.update(srcfile.read())
-                src_md5 = m.hexdigest()
+            oelite.util.hash_file(m, src)
+            src_md5 = m.hexdigest()
             m = hashlib.md5()
-            with open(dst, "r") as dstfile:
-                m.update(dstfile.read())
-                dst_md5 = m.hexdigest()
+            oelite.util.hash_file(m, dst)
+            dst_md5 = m.hexdigest()
             if src_md5 != dst_md5:
                 print "Mirror inconsistency:", dst
                 print "%s != %s"%(src_md5, dst_md5)

lib/oelite/fetch/local.py

@@ -1,5 +1,6 @@
 import oelite.fetch
 import oelite.path
+import oelite.util
 import os
 import hashlib
 
@@ -35,6 +36,6 @@ def signature(self):
         if os.path.isdir(self.localpath):
             raise oelite.fetch.NoSignature(self.uri, "can't compute directory signature")
         m = hashlib.sha1()
-        m.update(open(self.localpath, "r").read())
+        oelite.util.hash_file(m, self.localpath)
         self._signature = m.digest()
         return self._signature

lib/oelite/fetch/url.py

@@ -83,7 +83,7 @@ def grabbedsignature(self):
 
     def localsignature(self):
         m = hashlib.sha1()
-        m.update(open(self.localpath, "r").read())
+        oelite.util.hash_file(m, self.localpath)
         return m.hexdigest()
 
     def get_proxies(self, d):
@@ -119,8 +119,7 @@ def grab(url, filename, timeout=120, retry=5, proxies=None, passive_ftp=True):
 
     d = os.path.dirname(filename)
     f = os.path.basename(filename)
-    if not os.path.exists(d):
-        os.makedirs(d)
+    oelite.util.makedirs(d)
 
     # Use mkstemp to create and open a guaranteed unique file. We use
     # the file descriptor as wget's stdout. We must download to the

lib/oelite/signal.py

@@ -30,34 +30,3 @@ def restore_defaults():
         signal.signal(signal.SIGXFZ, signal.SIG_DFL)
     if hasattr(signal, "SIGXFSZ"):
         signal.signal(signal.SIGXFSZ, signal.SIG_DFL)
-
-def test_restore():
-    import os
-    import subprocess
-    import errno
-
-    PIPE=subprocess.PIPE
-    sub = subprocess.Popen(["yes"], stdout=PIPE, stderr=PIPE)
-    # Force a broken pipe.
-    sub.stdout.close()
-    err = sub.stderr.read()
-    # This should terminate with a write error; we assume that 'yes'
-    # is so well-behaved that it both exits with a non-zero exit code
-    # as well as prints an error message containing strerror(errno).
-    ret = sub.wait()
-    assert(ret > 0)
-    assert(os.strerror(errno.EPIPE) in err)
-
-    sub = subprocess.Popen(["yes"], stdout=PIPE, stderr=PIPE, preexec_fn = restore_defaults)
-    # Force a broken pipe.
-    sub.stdout.close()
-    err = sub.stderr.read()
-    # This should terminate due to SIGPIPE, and not get a chance to write to stderr.
-    ret = sub.wait()
-    assert(ret == -signal.SIGPIPE)
-    assert(err == "")
-
-if __name__ == "__main__":
-    # To run:
-    # meta/core/lib$ python -m oelite.signal
-    test_restore()