diff --git a/docker-stress b/docker-stress index cfcc3db..126d41a 100755 --- a/docker-stress +++ b/docker-stress @@ -138,7 +138,7 @@ def main(): except KeyboardInterrupt: pass finally: - print "Terminating workers and cleaning up..." + print "Terminating workers and cleaning up...(in background)" kill_workers() destroy_containers(client, nametag) diff --git a/jobs.json b/jobs.json index 9d82629..6a01ce5 100644 --- a/jobs.json +++ b/jobs.json @@ -1,13 +1,10 @@ [ {"image": "ubuntu:12.04", "command": ["sh", "-c", "apt-get update; while :; do sleep 1; done"]}, {"image": "ubuntu:12.04", "command": ["sh", "-c", "apt-get install nmap -qq --force-yes && while :; do ncat -e /bin/cat -k -l 4711; done"], "ports": [4711]}, - {"image": "linux/postgres", "command": [], "ports": [5432]}, {"image": "tutum/wordpress", "command": [], "ports": [80, 3306]}, {"image": "dockerfile/nginx", "command": [], "ports": [80]}, {"image": "skxskx/memcached", "command": [], "ports": [11211]}, {"image": "dockerfile/redis", "command": [], "ports": [6379]}, - {"image": "johannesh/bind10", "command": [], "ports": [53]}, - {"image": "dockerfiles/django-uwsgi-nginx", "command": [], "ports": [80]}, {"image": "jacksoncage/varnish", "command": [], "ports": [80]}, - {"image": "sameersbn/redmine", "command": [], "ports": [80]} + {"image": "isnog00d/rubydummyapp", "command": ["sh", "-c", "ruby /dummyapp/hw.rb"], "ports": [860]} ] diff --git a/spotify/docker_stress/docker_client.py b/spotify/docker_stress/docker_client.py index f32f3ae..134bc21 100644 --- a/spotify/docker_stress/docker_client.py +++ b/spotify/docker_stress/docker_client.py @@ -97,11 +97,20 @@ def destroy(self, container_id): log.debug('destroy %s', container_id) self.cli_check('rm', container_id) - def list_containers(self, needle=''): + def list_containers(self, needle='', _all=False): if not needle: return self.cli_check('ps', '-q').splitlines() else: - lines = self.cli_check('ps').splitlines()[1:] + if _all: + lines = self.cli_check('ps', '-a').splitlines()[1:] + else: + lines = self.cli_check('ps').splitlines()[1:] matches = [word for line in lines for word in line.split() if needle in word] log.debug('list_containers: needle=%s, matches=%s', needle, matches) return matches + + def list_all_exited(self, needle): + lines = self.cli_check('ps', '-a').splitlines()[1:] + lines = [line for line in lines if 'Exited' in line] + matches = [word for line in lines for word in line.split() if needle in word] + return matches diff --git a/spotify/docker_stress/docker_util.py b/spotify/docker_stress/docker_util.py index c04c304..1d65005 100644 --- a/spotify/docker_stress/docker_util.py +++ b/spotify/docker_stress/docker_util.py @@ -1,6 +1,7 @@ +import os import logging from socket import create_connection -from time import sleep +from time import time, sleep from urlparse import urlparse from docker_client import DockerClientError @@ -49,22 +50,34 @@ def connectable(client, container_id, hostname, ports): def destroy_containers(client, nametag): - while True: + timeout = time() + 60 * 60 + try: + if os.fork(): + return + except OSError, e: + log.debug('failed to fork: %s', e) + return + + while time() < timeout: sleep(1) - containers = client.list_containers(needle=nametag) + containers = client.list_containers(needle=nametag, _all=True) if not containers: break + containers = client.list_containers(needle=nametag) for container in containers: try: client.kill(container) except Exception, e: log.debug('kill failed: %s', e) + containers = client.list_all_exited(needle=nametag) for container in containers: try: client.destroy(container) except Exception, e: log.debug('destroy failed: %s', e) - + sleep(99) + if time() > timeout: + log.debug('Reached timeout for container destroy') def endpoint_address(endpoint): if '://' not in endpoint: