From 216924ddd70d2a1e8b6a48399d156030732b8954 Mon Sep 17 00:00:00 2001 From: Nicolas Valcarcel Date: Wed, 10 Jul 2019 15:29:19 -0500 Subject: [PATCH 1/4] swotch to sqlalchemy --- README.md | 7 +- marketplace/__init__.py | 12 ++- marketplace/db.py | 37 --------- marketplace/listings.py | 28 +++---- marketplace/models.py | 8 ++ marketplace/templates/listings/index.html | 4 +- migrations/README | 1 + migrations/alembic.ini | 45 +++++++++++ migrations/env.py | 96 +++++++++++++++++++++++ migrations/script.py.mako | 24 ++++++ migrations/versions/d018d799acf7_.py | 33 ++++++++ requirements.txt | 2 + 12 files changed, 237 insertions(+), 60 deletions(-) delete mode 100644 marketplace/db.py create mode 100644 marketplace/models.py create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/d018d799acf7_.py diff --git a/README.md b/README.md index 7f036d4..0210613 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,11 @@ In order to fix the SQL injetion once and for all, we should rely on prepared st Now both our unit test and bandit are happy! +### Fix part 3 +An even better approach is to use an ORM, in this case we set up SQLAlchemy, by using the standard methods the ORM will do the sanitization so we don't need to worry about it. + +**Note**: Most ORMs in some special use cases can still allow SQL Injections to happen, if you are using non-standard methods, review the ORMs security guidelines and test your application. + ## Description Welcome to the Secure coding with python course. In this repository you will find a series of branches for each step of the development of a sample marketplace application. In such a development, we will be making security mistakes and introducing vulnerabilities, we will add tests for them and finally fixing them. @@ -143,7 +148,7 @@ The branches will have the following naming scheme for easier navigation: {Chapt For this course we will be using Python3, Flask and PostgreSQL. -**Proceed to [next section](https://github.com/nxvl/secure-coding-with-python/tree/2.3-sql-injection/fix3)** +**Proceed to [next section](https://github.com/nxvl/secure-coding-with-python/tree/3.1-weak-password-storage/code)** ## Index ### 1. Vulnerable Components diff --git a/marketplace/__init__.py b/marketplace/__init__.py index a23df9c..3b1896e 100644 --- a/marketplace/__init__.py +++ b/marketplace/__init__.py @@ -1,12 +1,19 @@ import os from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate + + +db = SQLAlchemy() +migrate = Migrate() def create_app(test_config=None): app = Flask(__name__, instance_relative_config=True) app.config.from_mapping( SECRET_KEY='dev', - DATABASE='marketplace', + SQLALCHEMY_DATABASE_URI='postgresql:///marketplace', + SQLALCHEMY_TRACK_MODIFICATIONS=False, ) try: @@ -14,8 +21,9 @@ def create_app(test_config=None): except OSError: pass - from . import db + from . import models db.init_app(app) + migrate.init_app(app, db) from . import listings app.register_blueprint(listings.bp) diff --git a/marketplace/db.py b/marketplace/db.py deleted file mode 100644 index b60ee14..0000000 --- a/marketplace/db.py +++ /dev/null @@ -1,37 +0,0 @@ -import psycopg2 - -import click -from flask import current_app, g -from flask.cli import with_appcontext - -def get_db(): - if 'db' not in g: - g.db = psycopg2.connect(dbname=current_app.config['DATABASE']) - - return g.db - -def close_db(e=None): - db = g.pop('db', None) - - if db is not None: - db.close() - -def init_db(): - db = get_db() - cur = db.cursor() - - with current_app.open_resource('schema.sql') as f: - cur.execute(f.read().decode('utf8')) - db.commit() - - -@click.command('init-db') -@with_appcontext -def init_db_command(): - """Clear the existing data and create new tables.""" - init_db() - click.echo('Initialized the database.') - -def init_app(app): - app.teardown_appcontext(close_db) - app.cli.add_command(init_db_command) \ No newline at end of file diff --git a/marketplace/listings.py b/marketplace/listings.py index 7e92417..d354e23 100644 --- a/marketplace/listings.py +++ b/marketplace/listings.py @@ -1,32 +1,24 @@ import sys - + from flask import Blueprint, request, redirect, render_template, url_for - -from marketplace.db import get_db - + +from . import db +from .models import Listing + bp = Blueprint('listings', __name__, url_prefix='/listings') - + @bp.route('/') def index(): - cur = get_db().cursor() - cur.execute( - 'SELECT id, title, description' - ' FROM listings' - ) - listings = cur.fetchall() + listings = Listing.query.all() return render_template('listings/index.html', listings=listings) @bp.route('/create', methods=('GET', 'POST')) def register(): if request.method == 'POST': - title = request.form['title'] - description = request.form['description'] - db = get_db() - cur = db.cursor() + listing = Listing(title=request.form['title'], description=request.form['description']) + db.session.add(listing) + db.session.commit() - sql = "INSERT INTO listings (title, description) VALUES (%s, %s)" - cur.execute(sql, (title, description)) - db.commit() return redirect(url_for('listings.index')) return render_template('listings/create.html') diff --git a/marketplace/models.py b/marketplace/models.py new file mode 100644 index 0000000..17f3ef6 --- /dev/null +++ b/marketplace/models.py @@ -0,0 +1,8 @@ +from . import db + +class Listing(db.Model): + __tablename__ = 'listings' + + id = db.Column(db.Integer, primary_key=True) + title = db.Column(db.String(120)) + description = db.Column(db.String(500)) \ No newline at end of file diff --git a/marketplace/templates/listings/index.html b/marketplace/templates/listings/index.html index 8da3176..46602bf 100644 --- a/marketplace/templates/listings/index.html +++ b/marketplace/templates/listings/index.html @@ -9,10 +9,10 @@

{% block title %}Listings{% endblock %}

-

{{ listing[1] }}

+

{{ listing.title }}

-

{{ listing[2] }}

+

{{ listing.description }}

{% if not loop.last %}
diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..79b8174 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,96 @@ +from __future__ import with_statement + +import logging +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option( + 'sqlalchemy.url', current_app.config.get( + 'SQLALCHEMY_DATABASE_URI').replace('%', '%%')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/d018d799acf7_.py b/migrations/versions/d018d799acf7_.py new file mode 100644 index 0000000..1157feb --- /dev/null +++ b/migrations/versions/d018d799acf7_.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: d018d799acf7 +Revises: +Create Date: 2019-07-10 15:21:38.057975 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd018d799acf7' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('listings', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=120), nullable=True), + sa.Column('description', sa.String(length=500), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('listings') + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index f147c86..b34d6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ Flask==1.0.3 psycopg2==2.8.3 +Flask-Migrate==2.5.2 +Flask-SQLAlchemy==2.4.0 \ No newline at end of file From 723153021294a147999e58042f1c8abd7f52988d Mon Sep 17 00:00:00 2001 From: Nicolas Valcarcel Date: Fri, 19 Jul 2019 11:37:13 -0500 Subject: [PATCH 2/4] update tests --- tests/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 391301a..f4bb15c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest from marketplace import create_app -from marketplace.db import get_db, init_db +from marketplace import db sys.path.append(os.path.join(os.path.dirname(__file__), 'helpers')) @@ -14,9 +14,6 @@ def app(): 'DATABASE': 'marketplace_test', }) - with app.app_context(): - init_db() - yield app @pytest.fixture From e0d19dd0d31d2671649e38eaf9605a270c10e0ab Mon Sep 17 00:00:00 2001 From: Nicolas Valcarcel Date: Fri, 19 Jul 2019 11:42:23 -0500 Subject: [PATCH 3/4] use Text for description --- marketplace/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marketplace/models.py b/marketplace/models.py index 17f3ef6..f234b22 100644 --- a/marketplace/models.py +++ b/marketplace/models.py @@ -5,4 +5,4 @@ class Listing(db.Model): id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(120)) - description = db.Column(db.String(500)) \ No newline at end of file + description = db.Column(db.Text()) \ No newline at end of file From 3ff4b7e25b966ef82e6bc6fbfbaae12608f4cb68 Mon Sep 17 00:00:00 2001 From: Nicolas Valcarcel Date: Sat, 3 Aug 2019 15:47:07 -0500 Subject: [PATCH 4/4] remove last step text --- README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/README.md b/README.md index 18d6216..05a039e 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,6 @@ # Secure Coding with Python. ## Chapter 2: SQL Injection -### Fix part 2 -In order to fix the SQL injetion once and for all, we should rely on prepared statements, and let the DB engine do the param sanitization, like this: -```python - sql = "INSERT INTO listings (title, description) VALUES (%s, %s)" - cur.execute(sql, (title, description)) -``` - -Now both our unit test and bandit are happy! - ### Fix part 3 An even better approach is to use an ORM, in this case we set up SQLAlchemy, by using the standard methods the ORM will do the sanitization so we don't need to worry about it. @@ -22,11 +13,7 @@ The branches will have the following naming scheme for easier navigation: {Chapt For this course we will be using Python3, Flask and PostgreSQL. -<<<<<<< HEAD **Proceed to [next section](https://github.com/nxvl/secure-coding-with-python/tree/3.1-weak-password-storage/code)** -======= -**Proceed to [next section](https://github.com/nxvl/secure-coding-with-python/tree/2.3-sql-injection/fix)** ->>>>>>> 63a94db15f2a764a554613dfe42e6423fbe40bb3 ## Index ### 1. Vulnerable Components