From ba723639b5d4a6549ace2392e1f697c82337965e Mon Sep 17 00:00:00 2001 From: Mauro Date: Sat, 19 Oct 2019 21:21:30 -0300 Subject: [PATCH 01/17] init --- Dockerfile | 18 ++++++++++++++++++ docker-compose.yml | 32 ++++++++++++++++++++++++++++++++ docker.env | 13 +++++++++++++ requirements.txt | 1 + shorturl.py | 33 +++++++++++++++++++++++++++++++++ test_shorturl.py | 20 ++++++++++++++++++++ 6 files changed, 117 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker.env create mode 100644 requirements.txt create mode 100644 shorturl.py create mode 100644 test_shorturl.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..42b34f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.6-alpine + +ENV PYTHONUNBUFFERED 1 +ENV LANG C.UTF-8 +ENV HOME /app + +COPY ./requirements.txt /app/requirements.txt + +RUN apk add --no-cache \ + --virtual build-deps git python3-dev build-base \ + && pip install -U pip gunicorn \ + && pip install -r /app/requirements.txt \ + && rm -fr /app/.cache \ + && apk --purge del build-deps + +COPY . /app + +WORKDIR /app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..53b6f64 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3.1' +services: + db: + image: postgres:9.6-alpine + restart: unless-stopped + container_name: buoydev-db + env_file: + - ./docker.env + ports: + - "5432:5432" + networks: + - buoy_apps + web: + build: . + image: buoydev-py + container_name: buoydev-web + restart: unless-stopped + command: ./entrypoint.sh + env_file: + - ./docker.env + volumes: + - .:/app/ + ports: + - "5000:5000" + depends_on: + - db + networks: + - buoy_apps + +networks: + buoy_apps: + external: true diff --git a/docker.env b/docker.env new file mode 100644 index 0000000..f9bb3fc --- /dev/null +++ b/docker.env @@ -0,0 +1,13 @@ +POSTGRES_USER=buoy +POSTGRES_PASSWORD=buoy +POSTGRES_DB=buoy + +DB_HOST=db + +HOST=0.0.0.0 +PORT=5000 +WORKERS=3 + +FLASK_APP=shorturl.py +FLASK_ENV=development +FLASK_DEBUG=0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1828ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Flask==1.1.0 diff --git a/shorturl.py b/shorturl.py new file mode 100644 index 0000000..7de7be8 --- /dev/null +++ b/shorturl.py @@ -0,0 +1,33 @@ +from flask import ( + Flask, + jsonify, + redirect +) + + +app = Flask(__name__) + + +@app.route('/', methods=['GET']) +def index(): + """Home""" + return jsonify(endpoints={}), 200 + + +@app.route('/urls', methods=['POST']) +def url_post(): + """Save URLs""" + return jsonify(code="abc123"), 201 + + +@app.route('//stats', methods=['GET']) +def url_stats(code: str): + """Shows stats for a given code""" + return jsonify(data={}), 200 + + + +@app.route('/', methods=['GET']) +def url_get(code: str): + """Redirects to a URL given a code else 404""" + return redirect("http://ddg.gg/", code=302) diff --git a/test_shorturl.py b/test_shorturl.py new file mode 100644 index 0000000..a096389 --- /dev/null +++ b/test_shorturl.py @@ -0,0 +1,20 @@ +import os +import unittest +from shorturl import app + +class ShorturlTests(unittest.TestCase): + def setUp(self): + app.config["TESTING"] = True + app.config["DEBUG"] = True + self.app = app.test_client() + self.assertEqual(app.debug, True) + + def tearDown(self): + pass + + def test_index(self): + response = self.app.get('/', follow_redirects = True) + self.assertEqual(response.status_code, 200) + +if __name__ == "__main__": + unittest.main() From 6f818ec0a4e7ce33049e76d5851a1ed1ac7bd429 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sat, 19 Oct 2019 21:52:01 -0300 Subject: [PATCH 02/17] adding tests for each endpoint --- test_shorturl.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test_shorturl.py b/test_shorturl.py index a096389..dfa8abb 100644 --- a/test_shorturl.py +++ b/test_shorturl.py @@ -13,8 +13,48 @@ def tearDown(self): pass def test_index(self): - response = self.app.get('/', follow_redirects = True) + response = self.app.get('/', + follow_redirects=True) self.assertEqual(response.status_code, 200) + def test_url_get_ok(self): + response = self.app.get('/abc123', + follow_redirects=False) + self.assertEqual(response.status_code, 302) + + def test_url_get_err(self): + response = self.app.get('/abc123', + follow_redirects=False) + self.assertEqual(response.status_code, 302) + + def test_url_stats(self): + response = self.app.get('/abc123/stats', + follow_redirects=True) + self.assertEqual(response.status_code, 200) + + def test_url_post_ok(self): + response = self.app.post('/urls', + data={ + "url": "http://ddg.gg/", + "code": "ddg123" + }) + self.assertEqual(response.status_code, 201) + + def test_url_post_err(self): + response = self.app.post('/urls', + data={ + "url": "http://ddg.gg/", + "code": "wololo" + }) + self.assertEqual(response.status_code, 201) + + def test_url_post_nocode(self): + response = self.app.post('/urls', + data={ + "url": "http://sdf.org/", + }) + self.assertEqual(response.status_code, 201) + + if __name__ == "__main__": unittest.main() From 4ed08df1d5dfc5f8bfcac49f980048d3134a7693 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sat, 19 Oct 2019 23:02:50 -0300 Subject: [PATCH 03/17] ADD: entrypoint, models, utils, gitignore. UPDATE: click commands --- .dockerignore | 6 +++++ .gitignore | 6 +++++ entrypoint.sh | 6 +++++ models.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ utils.py | 29 ++++++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100755 entrypoint.sh create mode 100644 models.py create mode 100644 utils.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..dd0b593 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +__pycache__/ +.cache/ +*.pyc +*.db +.ash_* +.#* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd0b593 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +.cache/ +*.pyc +*.db +.ash_* +.#* diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..3032b22 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +flask create-tables && \ + gunicorn --bind $HOST:$PORT \ + --workers $WORKERS \ + shorturl:app diff --git a/models.py b/models.py new file mode 100644 index 0000000..2dc6309 --- /dev/null +++ b/models.py @@ -0,0 +1,68 @@ +import utils +from peewee import * +from os import environ as env + +if env.get('FLASK_DEBUG', 0): + db = SqliteDatabase('shorturl.db') +else: + db = PostgresqlDatabase( + env.get('POSTGRES_DB', 'postgres'), + user=env.get('POSTGRES_USER', 'postgres'), + password=env.get('POSTGRES_PASSWORD', 'postgres'), + host=env.get('DB_HOST', 'localhost'), + ) + + +class BaseModel(Model): + class Meta: + database = db + + +class URL(BaseModel): + url = CharField(max_length=35) + code = CharField(max_length=6) + + class Meta: + table_name = 'urls' + + @property + def serialize(self): + data = { + 'id': self.id, + 'url': str(self.url).strip(), + 'code': str(self.code).strip(), + } + + return data + + def __repr__(self): + code = self.code + url = self.url + return f"{code} ({url})" + + +class Stats(BaseModel): + url = ForeignKeyField(URL, backref='url') + created_at = DateTimeField(default=utils.now) + last_usage = DateTimeField(null=True) + usage_count = IntegerField() + + class Meta: + table_name = 'stats' + + @property + def serialize(self): + data = { + 'id': self.id, + 'url': str(self.url).strip(), + 'created_at': str(self.created_at).strip(), + 'last_usage': str(self.last_usage).strip(), + 'usage_count': str(self.usage_count).strip(), + } + + return data + + def __repr__(self): + url = self.url + count = self.usage_count + return f"{url} ({count})" diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..ac4428b --- /dev/null +++ b/utils.py @@ -0,0 +1,29 @@ +import datetime +from models import ( + db, + URL, + Stats +) + +def now() -> datetime.datetime: + """what time is now""" + return datetime.datetime.now() + + +def create_tables() -> None: + """Creates tables""" + db.create_tables([URL, Stats]) + + +def drop_tables() -> None: + """Drops tables""" + map(lambda m: m.drop_table(), [URL, Stats]) + + +def insert_url(url:str, code:str) -> int: + """Inserts a new URL""" + q = URL.insert(url=url, code=code) + url_id = q.execute() + Stats.insert(url=url_id, usage_count=0).execute() + + return url_id From 16b1320bbf7657549e5adada04e0ff4c9c57fff3 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sat, 19 Oct 2019 23:03:58 -0300 Subject: [PATCH 04/17] UPDATE: requirement peewee. Minor changes in tests/shorturl --- docker.env | 2 +- requirements.txt | 1 + shorturl.py | 41 +++++++++++++++++++++++++++++++++++++++-- test_shorturl.py | 19 +++++++++++++++++-- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/docker.env b/docker.env index f9bb3fc..814fadd 100644 --- a/docker.env +++ b/docker.env @@ -10,4 +10,4 @@ WORKERS=3 FLASK_APP=shorturl.py FLASK_ENV=development -FLASK_DEBUG=0 +FLASK_DEBUG=1 diff --git a/requirements.txt b/requirements.txt index 1828ac4..446aca6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ Flask==1.1.0 +peewee diff --git a/shorturl.py b/shorturl.py index 7de7be8..b7c628d 100644 --- a/shorturl.py +++ b/shorturl.py @@ -1,3 +1,9 @@ +import utils +from models import ( + db, + URL, + Stats +) from flask import ( Flask, jsonify, @@ -8,10 +14,42 @@ app = Flask(__name__) +## +# Flask commands +# drop-tables +# create-tables +# + +@app.cli.command() +def drop_tables(): + """Use PeeWee to drop tables.""" + utils.drop_tables() + + +@app.cli.command() +def create_tables(): + """Use PeeWee to create tables.""" + utils.create_tables() + + +## +# Flask endpoints +# index +# url_get +# url_stats +# url_post +# + @app.route('/', methods=['GET']) def index(): """Home""" - return jsonify(endpoints={}), 200 + endpoints = { + "/urls": "POST -> url:str:required, code:str:optional", + "/": "GET -> code:str:required", + "//stats": "GET -> code:str:required" + } + + return jsonify(endpoints=endpoints) @app.route('/urls', methods=['POST']) @@ -26,7 +64,6 @@ def url_stats(code: str): return jsonify(data={}), 200 - @app.route('/', methods=['GET']) def url_get(code: str): """Redirects to a URL given a code else 404""" diff --git a/test_shorturl.py b/test_shorturl.py index dfa8abb..a5fcdc5 100644 --- a/test_shorturl.py +++ b/test_shorturl.py @@ -1,16 +1,31 @@ -import os +import utils import unittest from shorturl import app + class ShorturlTests(unittest.TestCase): def setUp(self): app.config["TESTING"] = True app.config["DEBUG"] = True self.app = app.test_client() + utils.create_tables() + + self.test_data = { + "url":"https://sdf.org/", + "code": "sdforg", + "id": 0 + } + self.test_data["id"] = utils.insert_url( + self.test_data["url"], + self.test_data["code"] + ) + + print (self.test_data) + self.assertEqual(app.debug, True) def tearDown(self): - pass + utils.drop_tables() def test_index(self): response = self.app.get('/', From eedeabcb690903f2b0670588631e0cd785d8dd31 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 00:07:03 -0300 Subject: [PATCH 05/17] UPDATE: models. fix datetime, utils. adding helpers --- models.py | 6 +++--- utils.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/models.py b/models.py index 2dc6309..1b83ed1 100644 --- a/models.py +++ b/models.py @@ -1,9 +1,9 @@ -import utils +import datetime from peewee import * from os import environ as env if env.get('FLASK_DEBUG', 0): - db = SqliteDatabase('shorturl.db') + db = SqliteDatabase(':memory:') else: db = PostgresqlDatabase( env.get('POSTGRES_DB', 'postgres'), @@ -43,7 +43,7 @@ def __repr__(self): class Stats(BaseModel): url = ForeignKeyField(URL, backref='url') - created_at = DateTimeField(default=utils.now) + created_at = DateTimeField(default=datetime.datetime.now) last_usage = DateTimeField(null=True) usage_count = IntegerField() diff --git a/utils.py b/utils.py index ac4428b..f5dccf9 100644 --- a/utils.py +++ b/utils.py @@ -1,15 +1,27 @@ +import re +import secrets +import string import datetime +from urllib.parse import urlparse from models import ( db, URL, Stats ) + def now() -> datetime.datetime: """what time is now""" return datetime.datetime.now() +def to_iso8601(dt: str) -> datetime.datetime: + """Convert string to iso-8601""" + return (datetime.datetime + .strptime(dt, "%Y-%m-%d %H:%M:%S.%f") + .isoformat()) + + def create_tables() -> None: """Creates tables""" db.create_tables([URL, Stats]) @@ -27,3 +39,38 @@ def insert_url(url:str, code:str) -> int: Stats.insert(url=url_id, usage_count=0).execute() return url_id + + +def gen_code() -> str: + """Generates a code""" + alphabet = string.ascii_letters + string.digits + code = ''.join(secrets.choice(alphabet) for i in range(6)) + _, exists = code_exists(code) + if exists: + return gen_code() + else: + return code + + +def code_exists(code) -> tuple: + """Checks if a code exists in DB""" + q = (URL + .select(URL.id, URL.code) + .where(URL.code==code) + .limit(1)) + return q, q.count() > 0 + + +def is_valid_url(url: str) -> bool: + """is this URL well formatted? + (aka: does it looks like an url?)""" + result = urlparse(url) + return all([result.scheme, result.netloc, result.path]) + + +def is_valid_code(code: str) -> bool: + """check if a code complies with <=6 alphanumeric chr""" + return ( + len(re.sub(r'[a-zA-Z0-9]', '', code)) == 0 and + len(code) == 6 + ) From 6ebff6dcfbdba4981a856576d7c774816f5806bc Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 00:07:24 -0300 Subject: [PATCH 06/17] ADD: testing utilities --- test_utils.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test_utils.py diff --git a/test_utils.py b/test_utils.py new file mode 100644 index 0000000..cdfc1bb --- /dev/null +++ b/test_utils.py @@ -0,0 +1,65 @@ +import utils +import datetime +import unittest + + +class UtilsTests(unittest.TestCase): + def setUp(self): + utils.create_tables() + self.data = { + "url": "https://sdf.org/", + "code": "sdforg", + "id": 0 + } + self.data["id"] = utils.insert_url( + self.data["url"], + self.data["code"], + ) + + def tearDown(self): + utils.drop_tables() + + def test_is_valid_url(self): + self.assertEqual( + utils.is_valid_url(self.data["url"]), + True + ) + + def test_is_valid_code(self): + self.assertEqual( + utils.is_valid_code(self.data["code"]), + True + ) + + def test_gen_code(self): + code = utils.gen_code() + self.assertEqual(len(code), 6) + self.assertEqual(utils.is_valid_code(code), True) + + def test_insert_url(self): + url_id = utils.insert_url( + self.data["url"], + utils.gen_code() + ) + self.assertEqual(isinstance(url_id, int), True) + + def test_code_exists_true(self): + _, exists = utils.code_exists(self.data["code"]) + self.assertEqual(exists, True) + + def test_code_exists_false(self): + _, exists = utils.code_exists("abc123") + self.assertEqual(exists, False) + + def test_date_iso8601(self): + now = utils.now() + now_iso8601 = now.isoformat() + + self.assertEqual( + utils.to_iso8601(str(now)), + now_iso8601 + ) + + +if __name__ == "__main__": + unittest.main() From 55fc0ec438ebc51dde75fd78b78542ef2416132a Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 10:40:56 -0300 Subject: [PATCH 07/17] UPDATE: models. code=unique & utils/tests, get and bump stats --- models.py | 2 +- test_utils.py | 28 +++++++++++++++++++--------- utils.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/models.py b/models.py index 1b83ed1..3a77974 100644 --- a/models.py +++ b/models.py @@ -20,7 +20,7 @@ class Meta: class URL(BaseModel): url = CharField(max_length=35) - code = CharField(max_length=6) + code = CharField(max_length=6, unique=True) class Meta: table_name = 'urls' diff --git a/test_utils.py b/test_utils.py index cdfc1bb..f1ecd46 100644 --- a/test_utils.py +++ b/test_utils.py @@ -1,5 +1,4 @@ import utils -import datetime import unittest @@ -7,14 +6,9 @@ class UtilsTests(unittest.TestCase): def setUp(self): utils.create_tables() self.data = { - "url": "https://sdf.org/", - "code": "sdforg", - "id": 0 + "url": "https://ddg.gg/", + "code": utils.gen_code(), } - self.data["id"] = utils.insert_url( - self.data["url"], - self.data["code"], - ) def tearDown(self): utils.drop_tables() @@ -39,11 +33,27 @@ def test_gen_code(self): def test_insert_url(self): url_id = utils.insert_url( self.data["url"], - utils.gen_code() + self.data["code"] ) self.assertEqual(isinstance(url_id, int), True) + def test_stats(self): + _id = utils.insert_url( + self.data["url"], + self.data["code"] + ) + for i in range(1,5): + utils.bump_stats(_id) + created_at, last_usage, usage_count = utils.get_stats( + self.data["code"] + ) + self.assertEqual(usage_count, i) + def test_code_exists_true(self): + utils.insert_url( + self.data["url"], + self.data["code"] + ) _, exists = utils.code_exists(self.data["code"]) self.assertEqual(exists, True) diff --git a/utils.py b/utils.py index f5dccf9..0680d09 100644 --- a/utils.py +++ b/utils.py @@ -41,6 +41,31 @@ def insert_url(url:str, code:str) -> int: return url_id +def bump_stats(url_id: int) -> tuple: + """Updates Stat records""" + q = (Stats + .update({Stats.usage_count: Stats.usage_count+1}) + .where(Stats.url_id==url_id) + .execute()) + + return q + + +def get_stats(code: str) -> tuple: + """Returns Stats data""" + q = (Stats + .select( + Stats.created_at, + Stats.last_usage, + Stats.usage_count + ) + .join_from(Stats, URL) + .where(URL.code==code) + .execute()) + + return q.cursor.fetchone() + + def gen_code() -> str: """Generates a code""" alphabet = string.ascii_letters + string.digits From c9e9dcbf88fe9560e8e858b46d3ef2c8fe7f5832 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 10:54:46 -0300 Subject: [PATCH 08/17] UPDATE: utils tests --- test_utils.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test_utils.py b/test_utils.py index f1ecd46..21efebe 100644 --- a/test_utils.py +++ b/test_utils.py @@ -13,22 +13,22 @@ def setUp(self): def tearDown(self): utils.drop_tables() - def test_is_valid_url(self): - self.assertEqual( - utils.is_valid_url(self.data["url"]), - True - ) + def test_is_valid_url_ok(self): + self.assertTrue(utils.is_valid_url(self.data["url"])) - def test_is_valid_code(self): - self.assertEqual( - utils.is_valid_code(self.data["code"]), - True - ) + def test_is_valid_url_eerr(self): + self.assertFalse(utils.is_valid_url("this will fail")) + + def test_is_valid_code_ok(self): + self.assertTrue(utils.is_valid_code(self.data["code"])) + + def test_is_valid_code_err(self): + self.assertFalse(utils.is_valid_code("this will fail")) def test_gen_code(self): code = utils.gen_code() self.assertEqual(len(code), 6) - self.assertEqual(utils.is_valid_code(code), True) + self.assertTrue(utils.is_valid_code(code)) def test_insert_url(self): url_id = utils.insert_url( @@ -55,11 +55,11 @@ def test_code_exists_true(self): self.data["code"] ) _, exists = utils.code_exists(self.data["code"]) - self.assertEqual(exists, True) + self.assertTrue(exists) def test_code_exists_false(self): _, exists = utils.code_exists("abc123") - self.assertEqual(exists, False) + self.assertFalse(exists) def test_date_iso8601(self): now = utils.now() From eca8af61a3498d786769e633084d6ab2f13be75a Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 11:33:09 -0300 Subject: [PATCH 09/17] UPDATE: tests for url_post --- shorturl.py | 28 ++++++++++++-- test_shorturl.py | 95 ++++++++++++++++++++++++++++-------------------- 2 files changed, 81 insertions(+), 42 deletions(-) diff --git a/shorturl.py b/shorturl.py index b7c628d..32f5646 100644 --- a/shorturl.py +++ b/shorturl.py @@ -7,7 +7,8 @@ from flask import ( Flask, jsonify, - redirect + request, + redirect, ) @@ -54,8 +55,29 @@ def index(): @app.route('/urls', methods=['POST']) def url_post(): - """Save URLs""" - return jsonify(code="abc123"), 201 + """Saves URLs if ok, else wont (?)""" + data = request.json.copy() + if "url" not in list(data.keys()): + return jsonify(error="URL not provided"), 400 + + else: + if "code" not in list(data.keys()): + code = utils.gen_code() + else: + code = data.get('code') + if not utils.is_valid_code(code): + return jsonify(error="Code not valid"), 409 + + _, exists = utils.code_exists(code) + if exists: + return jsonify(error="Code in use"), 422 + else: + if utils.is_valid_url(data.get('url')): + utils.insert_url(data.get('url'), code) + else: + return jsonify(error="URL not valid"), 409 + + return jsonify(code=code), 201 @app.route('//stats', methods=['GET']) diff --git a/test_shorturl.py b/test_shorturl.py index a5fcdc5..0fa5cb0 100644 --- a/test_shorturl.py +++ b/test_shorturl.py @@ -1,3 +1,4 @@ +import json import utils import unittest from shorturl import app @@ -9,19 +10,10 @@ def setUp(self): app.config["DEBUG"] = True self.app = app.test_client() utils.create_tables() - - self.test_data = { - "url":"https://sdf.org/", - "code": "sdforg", - "id": 0 + self.data = { + "url": "https://ddg.gg/", + "code": utils.gen_code(), } - self.test_data["id"] = utils.insert_url( - self.test_data["url"], - self.test_data["code"] - ) - - print (self.test_data) - self.assertEqual(app.debug, True) def tearDown(self): @@ -32,43 +24,68 @@ def test_index(self): follow_redirects=True) self.assertEqual(response.status_code, 200) - def test_url_get_ok(self): - response = self.app.get('/abc123', - follow_redirects=False) - self.assertEqual(response.status_code, 302) + # def test_url_get_ok(self): + # response = self.app.get('/abc123', + # follow_redirects=False) + # self.assertEqual(response.status_code, 302) - def test_url_get_err(self): - response = self.app.get('/abc123', - follow_redirects=False) - self.assertEqual(response.status_code, 302) + # def test_url_get_err(self): + # response = self.app.get('/abc123', + # follow_redirects=False) + # self.assertEqual(response.status_code, 302) - def test_url_stats(self): - response = self.app.get('/abc123/stats', - follow_redirects=True) - self.assertEqual(response.status_code, 200) + # def test_url_stats(self): + # response = self.app.get('/abc123/stats', + # follow_redirects=True) + # self.assertEqual(response.status_code, 200) - def test_url_post_ok(self): + def test_url_post_201(self): response = self.app.post('/urls', - data={ - "url": "http://ddg.gg/", - "code": "ddg123" - }) + data=json.dumps(self.data), + content_type='application/json') self.assertEqual(response.status_code, 201) + self.assertEqual(response.json["code"], self.data["code"]) - def test_url_post_err(self): + def test_url_post_201_nocode(self): response = self.app.post('/urls', - data={ - "url": "http://ddg.gg/", - "code": "wololo" - }) + data=json.dumps({ + "url": self.data["url"] + }), + content_type='application/json') self.assertEqual(response.status_code, 201) + self.assertTrue(len(response.json["code"]) == 6) - def test_url_post_nocode(self): + def test_url_post_400(self): response = self.app.post('/urls', - data={ - "url": "http://sdf.org/", - }) - self.assertEqual(response.status_code, 201) + data=json.dumps({}), + content_type='application/json') + self.assertEqual(response.status_code, 400) + + def test_url_post_409_code(self): + response = self.app.post('/urls', + data=json.dumps({ + "url": self.data["url"], + "code": "this will fail" + }), + content_type='application/json') + self.assertEqual(response.status_code, 409) + + def test_url_post_409_url(self): + response = self.app.post('/urls', + data=json.dumps({ + "url": "http://lalala", + }), + content_type='application/json') + self.assertEqual(response.status_code, 409) + + def test_url_post_422(self): + self.app.post('/urls', + data=json.dumps(self.data), + content_type='application/json') + response = self.app.post('/urls', + data=json.dumps(self.data), + content_type='application/json') + self.assertEqual(response.status_code, 422) if __name__ == "__main__": From 0e20869fb0f06864b5eb35d3f866c35a05bd0880 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 13:42:09 -0300 Subject: [PATCH 10/17] UPDATE: url_get + url_stats & test cases --- shorturl.py | 28 +++++++++++++++++++-- test_shorturl.py | 63 ++++++++++++++++++++++++++++++++++++------------ utils.py | 5 +++- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/shorturl.py b/shorturl.py index 32f5646..13920b6 100644 --- a/shorturl.py +++ b/shorturl.py @@ -83,10 +83,34 @@ def url_post(): @app.route('//stats', methods=['GET']) def url_stats(code: str): """Shows stats for a given code""" - return jsonify(data={}), 200 + _, exists = utils.code_exists(code) + if not exists: + return jsonify(error="Code Not Found"), 404 + else: + created_at, last_usage, usage_count = utils.get_stats( + code + ) + + result = { + 'created_at': utils.to_iso8601(created_at), + 'usage_count': usage_count + } + if last_usage: + result['last_usage'] = utils.to_iso8601(last_usage) + + return jsonify(result), 200 + @app.route('/', methods=['GET']) def url_get(code: str): """Redirects to a URL given a code else 404""" - return redirect("http://ddg.gg/", code=302) + q, exists = utils.code_exists(code) + if exists: + url_id, url = q.execute().cursor.fetchone() + utils.bump_stats(url_id) + return redirect(url, code=302) + + else: + return jsonify(error="Code Not Found"), 404 + diff --git a/test_shorturl.py b/test_shorturl.py index 0fa5cb0..9067220 100644 --- a/test_shorturl.py +++ b/test_shorturl.py @@ -24,21 +24,6 @@ def test_index(self): follow_redirects=True) self.assertEqual(response.status_code, 200) - # def test_url_get_ok(self): - # response = self.app.get('/abc123', - # follow_redirects=False) - # self.assertEqual(response.status_code, 302) - - # def test_url_get_err(self): - # response = self.app.get('/abc123', - # follow_redirects=False) - # self.assertEqual(response.status_code, 302) - - # def test_url_stats(self): - # response = self.app.get('/abc123/stats', - # follow_redirects=True) - # self.assertEqual(response.status_code, 200) - def test_url_post_201(self): response = self.app.post('/urls', data=json.dumps(self.data), @@ -88,5 +73,53 @@ def test_url_post_422(self): self.assertEqual(response.status_code, 422) + def test_url_post_422(self): + self.app.post('/urls', + data=json.dumps(self.data), + content_type='application/json') + response = self.app.post('/urls', + data=json.dumps(self.data), + content_type='application/json') + self.assertEqual(response.status_code, 422) + + def test_url_get_302(self): + post = self.app.post('/urls', + data=json.dumps({ + "url": self.data["url"] + }), + content_type='application/json') + code = post.json["code"] + response = self.app.get(f"/{code}", + follow_redirects=False) + self.assertEqual(response.status_code, 302) + + def test_url_get_404(self): + code = "err404" + response = self.app.get(f"/{code}", + follow_redirects=False) + self.assertEqual(response.status_code, 404) + + + def test_url_stats_200(self): + post = self.app.post('/urls', + data=json.dumps({ + "url": self.data["url"] + }), + content_type='application/json') + code = post.json["code"] + usage_expected = 5 + for i in range(usage_expected): + self.app.get(f"/{code}", + follow_redirects=False) + + response = self.app.get(f"/{code}/stats") + self.assertEqual(response.json["usage_count"], + usage_expected) + + def test_url_stats_404(self): + response = self.app.get("/err404/stats") + self.assertEqual(response.status_code, 404) + + if __name__ == "__main__": unittest.main() diff --git a/utils.py b/utils.py index 0680d09..71f2ce9 100644 --- a/utils.py +++ b/utils.py @@ -44,7 +44,10 @@ def insert_url(url:str, code:str) -> int: def bump_stats(url_id: int) -> tuple: """Updates Stat records""" q = (Stats - .update({Stats.usage_count: Stats.usage_count+1}) + .update({ + Stats.usage_count: Stats.usage_count+1, + Stats.last_usage: now() + }) .where(Stats.url_id==url_id) .execute()) From e4f8c9a8c3f15e24956ac90019fda9d9d64fc147 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 20:23:41 -0300 Subject: [PATCH 11/17] UPDATE: added minor checks in test_url_stats --- test_shorturl.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/test_shorturl.py b/test_shorturl.py index 9067220..7b1227a 100644 --- a/test_shorturl.py +++ b/test_shorturl.py @@ -1,9 +1,22 @@ import json import utils +import datetime import unittest from shorturl import app +def cmp_dt(dt_a: str, dt_b: str) -> bool: + dt_a = datetime.datetime.strftime( + datetime.datetime.strptime( + dt_a, '%Y-%m-%dT%H:%M:%S.%f' + ), '%Y-%m-%dT%H:%M:%S' + ) + dt_b = datetime.datetime.strftime( + dt_b, '%Y-%m-%dT%H:%M:%S' + ) + return dt_a == dt_b + + class ShorturlTests(unittest.TestCase): def setUp(self): app.config["TESTING"] = True @@ -63,16 +76,6 @@ def test_url_post_409_url(self): content_type='application/json') self.assertEqual(response.status_code, 409) - def test_url_post_422(self): - self.app.post('/urls', - data=json.dumps(self.data), - content_type='application/json') - response = self.app.post('/urls', - data=json.dumps(self.data), - content_type='application/json') - self.assertEqual(response.status_code, 422) - - def test_url_post_422(self): self.app.post('/urls', data=json.dumps(self.data), @@ -99,8 +102,8 @@ def test_url_get_404(self): follow_redirects=False) self.assertEqual(response.status_code, 404) - def test_url_stats_200(self): + now = utils.now() post = self.app.post('/urls', data=json.dumps({ "url": self.data["url"] @@ -111,10 +114,17 @@ def test_url_stats_200(self): for i in range(usage_expected): self.app.get(f"/{code}", follow_redirects=False) + now_usage = utils.now() response = self.app.get(f"/{code}/stats") + self.assertEqual(response.json["usage_count"], usage_expected) + self.assertTrue(cmp_dt(response.json["created_at"], + now)) + self.assertTrue(cmp_dt(response.json["last_usage"], + now_usage)) + def test_url_stats_404(self): response = self.app.get("/err404/stats") From 4aba0366c40a3f9cf7dda7bb22bc09b66edf3504 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 21:00:09 -0300 Subject: [PATCH 12/17] UPDATE: fix small issue with utils.to_iso8601 --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 71f2ce9..71192e8 100644 --- a/utils.py +++ b/utils.py @@ -18,7 +18,7 @@ def now() -> datetime.datetime: def to_iso8601(dt: str) -> datetime.datetime: """Convert string to iso-8601""" return (datetime.datetime - .strptime(dt, "%Y-%m-%d %H:%M:%S.%f") + .strptime(str(dt), "%Y-%m-%d %H:%M:%S.%f") .isoformat()) From f35ad2c97b60193817295fef7a8fa386c249b2a7 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 21:00:53 -0300 Subject: [PATCH 13/17] UPDATE: add dependency on psycopg2 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 446aca6..677f627 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ Flask==1.1.0 +psycopg2==2.7.3.2 peewee From b0a8d6512980dbc146ccda22108d3312f908b00c Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 21:02:17 -0300 Subject: [PATCH 14/17] UPDATE: build dep on psql. bump docker image to py3.7 + non-root user/group --- Dockerfile | 16 +++++++++++++--- docker-compose.yml | 6 +++++- docker.env | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 42b34f6..66095d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,27 @@ -FROM python:3.6-alpine +FROM python:3.7-alpine + +ARG UID=1000 +ARG GID=1000 ENV PYTHONUNBUFFERED 1 ENV LANG C.UTF-8 +ENV USER shorturl ENV HOME /app COPY ./requirements.txt /app/requirements.txt RUN apk add --no-cache \ - --virtual build-deps git python3-dev build-base \ + --virtual build-deps \ + musl-dev postgresql-dev \ + build-base python3-dev zlib-dev git \ + && apk add --no-cache \ + postgresql-libs libpq \ && pip install -U pip gunicorn \ && pip install -r /app/requirements.txt \ && rm -fr /app/.cache \ - && apk --purge del build-deps + && apk --purge del build-deps \ + && addgroup -S $USER -g $GID \ + && adduser -S -G $USER -u $UID -h $HOME $USER COPY . /app diff --git a/docker-compose.yml b/docker-compose.yml index 53b6f64..cdf74e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,11 @@ services: networks: - buoy_apps web: - build: . + build: + context: . + args: + UID: ${UID:-1000} + GID: ${GID:-1000} image: buoydev-py container_name: buoydev-web restart: unless-stopped diff --git a/docker.env b/docker.env index 814fadd..f9bb3fc 100644 --- a/docker.env +++ b/docker.env @@ -10,4 +10,4 @@ WORKERS=3 FLASK_APP=shorturl.py FLASK_ENV=development -FLASK_DEBUG=1 +FLASK_DEBUG=0 From 32902f06919de3386d9b09ed01b9756514ca4b49 Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 21:35:35 -0300 Subject: [PATCH 15/17] ADD: install guide + docker-compose for testing --- .gitignore | 1 + INSTALL.md | 45 +++++++++++++++++++++++++++++++++++++++++ docker-compose.test.yml | 17 ++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 INSTALL.md create mode 100644 docker-compose.test.yml diff --git a/.gitignore b/.gitignore index dd0b593..799808e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ .cache/ +env/ *.pyc *.db .ash_* diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..77bcc7a --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,45 @@ +# shorturl how to + +## Install + +### docker + +``` +$ docker-compose build +``` + +### virtualenv + +``` +$ virtualenv -p python3 --system-site-packages env +$ . ./env/bin/activate +(env) $ pip install -r requirements.txt +``` + +### Run + +### docker + +``` +$ docker-compose up +``` + +### virtualenv + +``` +(env) $ ./entrypoint.sh +``` + +### Test + +### docker + +``` +$ docker-compose -f docker-compose.test.yml up +``` + +### virtualenv + +``` +(env) $ python -m unittest discover +``` diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..ca74eab --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,17 @@ +version: '3.1' +services: + web: + build: + context: . + args: + UID: ${UID:-1000} + GID: ${GID:-1000} + image: buoydev-py + container_name: buoydev-test + command: python -m unittest discover + environment: + FLASK_APP: shorturl.py + FLASK_ENV: development + FLASK_DEBUG: 1 + volumes: + - .:/app/ From 3a1389301cf9756dc471e9c12126e0146bb9efdd Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 22:28:00 -0300 Subject: [PATCH 16/17] UPDATE: choose db based on DEBUG --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index 3a77974..67208b1 100644 --- a/models.py +++ b/models.py @@ -2,7 +2,7 @@ from peewee import * from os import environ as env -if env.get('FLASK_DEBUG', 0): +if env.get('FLASK_DEBUG') == 1: db = SqliteDatabase(':memory:') else: db = PostgresqlDatabase( From bf018902294918c3c2553c559716422d56b02d4e Mon Sep 17 00:00:00 2001 From: Mauro Date: Sun, 20 Oct 2019 23:52:33 -0300 Subject: [PATCH 17/17] UPDATE: Using a db_proxy to switch test from 'prod' db --- models.py | 27 +++++++++++++++------------ shorturl.py | 5 ----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/models.py b/models.py index 67208b1..a833d1a 100644 --- a/models.py +++ b/models.py @@ -2,21 +2,11 @@ from peewee import * from os import environ as env -if env.get('FLASK_DEBUG') == 1: - db = SqliteDatabase(':memory:') -else: - db = PostgresqlDatabase( - env.get('POSTGRES_DB', 'postgres'), - user=env.get('POSTGRES_USER', 'postgres'), - password=env.get('POSTGRES_PASSWORD', 'postgres'), - host=env.get('DB_HOST', 'localhost'), - ) - +db_proxy = Proxy() class BaseModel(Model): class Meta: - database = db - + database = db_proxy class URL(BaseModel): url = CharField(max_length=35) @@ -66,3 +56,16 @@ def __repr__(self): url = self.url count = self.usage_count return f"{url} ({count})" + + + +db = SqliteDatabase(':memory:') +if int(env.get('FLASK_DEBUG')) == 0: + db = PostgresqlDatabase( + env.get('POSTGRES_DB', 'postgres'), + user=env.get('POSTGRES_USER', 'postgres'), + password=env.get('POSTGRES_PASSWORD', 'postgres'), + host=env.get('DB_HOST', 'localhost'), + ) + +db_proxy.initialize(db) diff --git a/shorturl.py b/shorturl.py index 13920b6..061d7a1 100644 --- a/shorturl.py +++ b/shorturl.py @@ -1,9 +1,4 @@ import utils -from models import ( - db, - URL, - Stats -) from flask import ( Flask, jsonify,