Fixtures and Mocking in Python

Fixtures and Mocking in Python

How do you test Moon-landing?

• without actually flying to the moon?

How do you test a system ...

• that sends a verification email?
• that relies on random values?
• in parallel without interference?
• together with a 3rd party?
• when a 3rd party it uses fails?
• with a timeout

Plan

• Introducton
• Presentation about Fixtures and Mocking
• Hands-on exercises
• Retrospective
• Job searching help

About me

• Gabor Szabo
• Help tech teams move faster with more stability and more predictability.
• Automation
• DevOps
• Code Maven Workshops
• Code Maven Workshops on Meetup

Goal

You will come out from the workshop knowing

• What are Fixtures?

• What is Mocking and Monkey Patching?

• When to use them?

• What are the dangers?

• Experiment with mocking in various situations.

Fixtures

• Fixtures - the environment in which a test runs (the outside world)

• Directory layout - files

• Database with or without data etc.

Fixtuers in Pytest

• A more generic term
• Helper tools to run and analyze your test code

Traditional xUnit fixtures

def setup_module():
    print("setup_module")

def teardown_module():
    print("teardown_module")


def setup_function():
    print("  setup_function")

def teardown_function():
    print("  teardown_function")


def test_one():
    print("    test_one")
    assert True
    print("    test_one after")

def test_two():
    print("    test_two")
    assert False
    print("    test_two after")

def test_three():
    print("    test_three")
    assert True
    print("    test_three after")

$ pytest test_fixture.py -s

setup_module

  setup_function
    test_one
    test_one after
  teardown_function

  setup_function
    test_two
  teardown_function

  setup_function
    test_three
    test_three after
  teardown_function

teardown_module

Dependency Injection

• Use introspection to find out what a method needs
• Pass in the right arguments

def do_something(name, age):
   pass

Temporary directory - tmpdir

• tmpdir

import json

def read(filename):
    with open(filename) as fh:
        return json.load(fh)

def save(filename, data):
    with open(filename, 'w') as fh:
        return json.dump(data, fh)

import app
import os

def test_json(tmpdir):
    tdir = str(tmpdir)
    print(tdir)

    data = {
        'name'  : 'Foo Bar',
        'email' : 'foo@bar.com',
    }

    filename = os.path.join(tdir, 'temp.json')
    app.save(filename, data)
    again = app.read(filename)
    assert data == again

• Directory location OSX: /private/var/folders/ry/z60xxmw0000gn/T/pytest-of-gabor/pytest-14/test_read0
• Linux: /tmp/pytest-of-gabor/pytest-9/test_json0
• Directory cleanup

Capture STDOUT and STDERR - capsys

• capsys

import sys

def greet(to_out, to_err=None):
    print(to_out)
    if to_err:
        print(to_err, file=sys.stderr)

import app

def test_myoutput(capsys):
    app.greet("hello", "world")
    out, err = capsys.readouterr()
    assert out == "hello\n"
    assert err == "world\n"

    app.greet("next")
    out, err = capsys.readouterr()
    assert out == "next\n"
    assert err == ""

Home-made fixture

import pytest

@pytest.fixture()
def config():
    return {
       'name'  : 'Foo Bar',
       'email' : 'foo@bar.com',
    }

def test_some_data(config):
    assert True
    print(config)

Home-made fixture - conftest

def test_some_data(config):
    assert True
    print(config)

import pytest

@pytest.fixture()
def config():
    return {
       'name'  : 'Foo Bar',
       'email' : 'foo@bar.com',
    }

Home-made fixture with tempdir

import yaml

def test_some_data(config):
    assert True
    print(config)

    with open(config) as fh:
        conf = yaml.load(fh, Loader=yaml.FullLoader)
    print(conf)

import pytest
import os
import yaml

@pytest.fixture()
def config(tmpdir):
    print(tmpdir.__class__)  # LocalPath
    tdir = str(tmpdir)
    print(tdir)

    config_data = {
       'name'  : 'Foo Bar',
       'email' : 'foo@bar.com',
    }
    config_file = os.path.join(tdir, 'test_db.yaml')
    with open(config_file, 'w') as yaml_file:
        yaml.dump(config_data, yaml_file, default_flow_style=False)

    return config_file

$ pytest -qs
{'name': 'Foo Bar', 'email': 'foo@bar.com'}

Home-made fixture with yield

import pytest

@pytest.fixture()
def configuration():
    print("Before")

    yield { 'name' : 'Foo Bar' }

    print("After")

def test_app(configuration):
    print("In test")
    print(configuration)
    assert True

$ pytest -sq
Before
In test
{'name': 'Foo Bar'}
.After

1 passed in 0.02 seconds

Fixture Autouse

import pytest

@pytest.fixture(autouse = True)
def configuration():
    print("Before")

def test_app():
    print("In test")
    assert True

$ pytest -sq
Before
In test
.

1 passed in 0.02 seconds

Fixture Autouse with yield

import pytest

@pytest.fixture(autouse = True)
def configuration():
    print("Before")

    yield

    print("After")

def test_app():
    print("In test")
    assert True

$ pytest -sq
Before
In test
.After

1 passed in 0.02 seconds

Fixture for MongoDB

import pytest
import os, time

from app.common import get_mongo

@pytest.fixture(autouse = True)
def configuration():
    dbname = 'test_app_' + str(int(time.time()))
    os.environ['APP_DB_NAME'] = dbname

    yield

    get_mongo().drop_database(dbname)

Test Doubles

• Mocks

• Spies

• Stubs

• Fakes

• Dummies

• Gerard Meszaros - xUnit Test Patterns

• Test Double explained by Martin Fowler

Test Doubles explained

Dummy objects are passed around but never actually used.

Fakes - Working implementation, but much more simple than the orignial.

• An in-memory list of username/password pairs that provide the authentication.
• A database interface where data stored in memory only, maybe in a dictionary.

Mocks - Mocks are objects that register calls they receive, but do not execute the real system behind.

Stubs - Stub is an object that holds predefined data and uses it to answer calls during tests.

• A list of "random values".
• Responses given to prompt.

Spies usually record some information based on how they were called and then call the real method. (or not)

Verify behavior or state?

What is Mocking and Monkey Patching?

• Replace some internal part of a module or class for the sake of testing.
• Mocking
• Monkey Patching

Situations

• TDD

• Write application agains API that is not ready yet or not controlled by you.

• Replace a complex object with a simpler one.

• Isolate parts of the system to test them on their own.

• Speed up tests (e.g. eliminate remote calls, eliminate database calls).

• Simulate cases that are hard to replicate. (What if the other system fails?)

• Unit tests.

Unit testing vs. Integration testing

Experiment with mocking in various situations

• Mocking external calls.
• Mocking method calls.
• Mocking a whole class.
• Mocking time.
• Mocking IO

Examples are simple

• Don't worry, real life code is much more complex!

Hard coded path

In many application we can find hard-coded pathes. In order to test them we will need to create that exact path which is not always easy. This also means we cannot run two tests at the same time with different content in those files. (The actual "application" is just adding numbers together.)

import json

data_file = "/corporate/fixed/path/data.json"

def get_sum():
    with open(data_file) as fh:
        data = json.load(fh)
        # ...
        result = data['x'] + data['y']
        return result

import app

def test_sum():
    res = app.get_sum()
    assert True

pytest test_data_1.py

    def get_sum():
>       with open(data_file) as fh:
E       FileNotFoundError: [Errno 2] No such file or directory: '/corporate/fixed/path/data.json'

Manually Patching attribute

We can replace the attribute during the test run and create a json file locally to be used. At least two problems with this:

• Readers might glance over the assignment and might be baffled
• The change is permanent in the whole test script so one test impacts the other.

import app

def test_sum():
    app.data_file = 'test_1.json'    # manually overwrite

    res = app.get_sum()
    assert True
    assert res == 42

def test_again():
    print(app.data_file)             # it is still test_1.json

{
    "x": 19,
    "y": 23
}

Monkey Patching attribute

We can use the monkeypatch fixture to do the same.

• It stands out more as it is a fxture and you can search for the name
• It only applies to the current test function. So they are now independent again.

import app

def test_sum(monkeypatch):
    monkeypatch.setattr(app, 'data_file', 'test_1.json')

    res = app.get_sum()
    assert True
    assert res == 42


def test_again():
    print(app.data_file)             # it is now back to the original value

Monkey Patching functions

def run(x):
    return 2 * x

import aut

def test_a():
    assert aut.run(7) == 14

def test_b(monkeypatch):
    monkeypatch.setattr('aut.run', lambda z: z)

    assert aut.run(5) == 5

def test_c():
    assert aut.run(10) == 20

Monkey Patching dictionary items

data = {
    'name' : 'foo',
    'age'  : 42
}

import aut

def test_a():
    assert aut.data == {
        'name' : 'foo',
        'age'  : 42
    }

def test_b(monkeypatch):
    monkeypatch.setitem(aut.data, 'name', 'bar')

    assert aut.data == {
        'name' : 'bar',
        'age'  : 42
    }

def test_c():
    assert aut.data == {
        'name' : 'foo',
        'age'  : 42
    }

Mocking a whole class

import json

class Thing(object):
    def data_file():
        return "/corporate/fixed/path/data.json"

    def get_sum(self):
        data_file = self.data_file()
        with open(data_file) as fh:
            data = json.load(fh)
            # ...
            result = data['x'] + data['y']
            return result

{
    "x": 19,
    "y": 23
}

import app

def test_sum():
    app.Thing.data_file = lambda self: 'data.json'
    t = app.Thing()
    res = t.get_sum()
    assert True
    assert res == 42

Mocking input/output

def calc():
    a = input("Type in a: ")
    b = input("Type in b: ")

    print("The result is:", add(int(a), int(b)))

def add(x, y):
    return x+y

if __name__ == '__main__':
    calc()

The test:

import app

def test_app():
    assert app.add(2, 3) == 5

Mocking input/output

import app

def test_calc():
    input_values = ['19', '23']
    output = []

    def mock_input(s):
       output.append(s)
       return input_values.pop(0)
    app.input = mock_input
    app.print = lambda *s : output.append(s)

    app.calc()

    assert output == [
        'Type in a: ',
        'Type in b: ',
        ('The result is:', 42),
    ] 

Mocking random numbers

• Mock the methods of the random module

Exercises

Download zip file or clone repository using

git clone https://github.com/szabgab/slides.git

the files are in the directory.

slides/python-mocking/

Work in pairs

• Navigator - Driver
• Driver - Observer

Exercise: test login expiration

import time

TIMEOUT = 60*60*24*7

class MySystem():
    def __init__(self):
        self.logged_in = 0

    def login(self, name, password):
        resp = self.verify_user(name, password)
        if resp:
            self.logged_in = True
            self.seen()

        return resp

    def seen(self):
        self.last_seen = time.time()

    def is_logged_in(self):
        return self.logged_in and self.last_seen + TIMEOUT > time.time()

    def verify_user(self, name, password):
        if name == 'foo' and password == 'secret':
            return True
        return False

from app import MySystem

def test_app():
     s = MySystem()
     assert not s.is_logged_in()

     assert not s.login('bar', 'secret')
     assert not s.is_logged_in()

     assert s.login('foo', 'secret')
     assert s.is_logged_in()

     # how to test the timeout?

Solution: test login expiration

from app import MySystem
import time

def test_app(monkeypatch):
     s = MySystem()
     assert not s.is_logged_in()

     assert not s.login('bar', 'secret')
     assert not s.is_logged_in()

     assert s.login('foo', 'secret')
     assert s.is_logged_in()

     now = time.time() + 60*60*24*7 + 1
     monkeypatch.setattr('time.time', lambda : now)

     assert not s.is_logged_in()

Exercise: Record e-mail sending

Implement a registration for a Flask (or other) web application: Accept e-mail as input send e-mail with a code to the given address use that code to verify e-mail address. Without actually sending e-mails.

from flask import Flask, request
import random

app = Flask(__name__)
db = {}

@app.route('/', methods=['GET'])
def home():
    return '''
          <form method="POST" action="/register">
          <input name="email">
          <input type="submit">
          </form>
          '''

@app.route('/register', methods=['POST'])
def register():
    email = request.form.get('email')
    code = str(random.random())
    if db_save(email, code):
        html = '<a href="/verify/{email}/{code}">here</a>'.format(email=email, code = code)
        sendmail({'to': email, 'subject': 'Registration', 'html': html })
        return 'OK'
    else:
        return 'FAILED'

@app.route('/verify/<email>/<code>', methods=['GET'])
def verify(email, code):
    if db_verify(email, code):
        sendmail({'to': email, 'subject': 'Welcome!', 'html': '' })
        return 'OK'
    else:
        return 'FAILED'

def sendmail(data):
    pass

def db_save(email, code):
   if email in db:
       return False
   db[email] = code
   return True

def db_verify(email, code):
    return email in db and db[email] == code

Solution: Record e-mail sending

import app
import re

def test_app(monkeypatch):
    aut = app.app.test_client()

    rv = aut.get('/')
    assert rv.status == '200 OK'
    assert '<form' in str(rv.data)
    assert not 'Welcome back!' in str(rv.data)

    email = 'foo@bar.com'

    messages = []
    monkeypatch.setattr('app.sendmail', lambda params: messages.append(params) )

    rv = aut.post('/register', data=dict(email = email ))
    assert rv.status == '200 OK'
    assert 'OK' in str(rv.data)
    print(messages)
    # [{'to': 'foo@bar.com', 'subject': 'Registration', 'html': '<a href="/verify/foo@bar.com/0.81280014">here</a>'}]

    rv = aut.get('/verify/{email}/{code}'.format(email = email, code = 'other' ))
    assert rv.status == '200 OK'
    assert 'FAILED' in str(rv.data)

    match = re.search(r'/(\d\.\d+)"', messages[0]['html'])
    if match:
        code = match.group(1)
    print(code)

    messages = []
    rv = aut.get('/verify/{email}/{code}'.format(email = email, code = code ))
    assert rv.status == '200 OK'
    assert 'OK' in str(rv.data)

    assert messages == [{'to': email, 'subject': 'Welcome!', 'html': ''}]

Exercise: Fixture database

Set up a database (can be sqlite, mysql, postgresql, mongodb, etc.) for each test run.

Exercise: One Dimentsional space-fight

• space-fight directory.

• Write a test that check the 'x' button works.

• Write a test that check system can properly report 'less than'.

• Write a test that check system can properly report 'greater than'.

• Write a test that check system can properly report 'found'.

• You might need to mock input/output/random.

import random

def play():
    debug = False
    move = False
    while True:
        print("\nWelcome to another Number Guessing game")
        hidden = random.randrange(1, 201)
        while True:
            if debug:
                print("Debug: ", hidden)
    
            if move:
                mv = random.randrange(-2, 3)
                hidden = hidden + mv
    
            user_input = input("Please enter your guess [x|s|d|m|n]: ")
            print(user_input)
    
            if user_input == 'x':
                print("Sad to see you leave early")
                return
    
            if user_input == 's':
                print("The hidden value is ", hidden)
                continue
    
            if user_input == 'd':
                debug = not debug
                continue
    
            if user_input == 'm':
                move = not move
                continue
    
            if user_input == 'n':
                print("Giving up, eh?")
                break
    
            guess = int(user_input)
            if guess == hidden:
                print("Hit!")
                break
    
            if guess < hidden:
                print("Your guess is too low")
            else:
                print("Your guess is too high")
    

if __name__ == '__main__':
    play()

Exercise: web client

• crawler directory.
• Test this application without hitting any web site.
• Test what happens if the URL returns 404
• What if it is a 500 error?
• What if the host not found?

import sys
import requests
import re

def count(url, word):
    r = requests.get(url)
    # r.status_code
    res = re.findall(word, r.text, re.IGNORECASE)
    return(len(res))

if __name__ == '__main__':
    if len(sys.argv) != 3:
        exit("{} URL string".format(sys.argv[0]))
    print(count(sys.argv[1], sys.argv[2]))

Exercise: Open WeatherMap client

• get API key
• It takes 10 minutes to activate the key, so do it now.
• Once you observerd that the code works, test it without internet access.

config.ini

[openweathermap]
api=93712604

import configparser
import requests
import sys

def get_api_key():
    config = configparser.ConfigParser()
    config.read('config.ini')
    return config['openweathermap']['api']

def get_weather(api_key, location):
    url = "https://api.openweathermap.org/data/2.5/weather?q={}&units=metric&appid={}".format(location, api_key)
    r = requests.get(url)
    return r.json()

def main():
    if len(sys.argv) != 2:
        exit("Usage: {} LOCATION".format(sys.argv[0]))
    location = sys.argv[1]

    api_key = get_api_key()
    weather = get_weather(api_key, location)

    print(weather)
    print(weather['main']['temp'])


if __name__ == '__main__':
    main()

Exercise: Mocking A Bank

import db

class Bank():
    def __init__(self):
        self.db = db.DB()

    def setup(self):
        self.db.create()

    def transfer(self, src, dst, amount):
        src_current = self.db.get(src)
        dst_current = self.db.get(dst)
        if src_current and src_current >= amount:
            self.db.update(src, src_current-amount)
            self.db.update(dst, dst_current+amount)
        else:
            raise Exception("Not enough money")

    def status(self, name):
        return self.db.get(name)
  
    def deposit(self, name, amount):
        current = self.db.get(name)
        if current == None:
            self.db.insert(name, amount)
        else:
            self.db.update(name, current+amount)

import sqlite3

db_filename = 'bank.db'


class DB():
    def __init__(self):
        self.db_filename = db_filename
        self.conn = sqlite3.connect(self.db_filename)

    def create(self):
        c = self.conn.cursor()
        c.execute('''CREATE TABLE account
                 (name text, ballance real)''')

    def get(self, name):
        c = self.conn.cursor()
        c.execute('SELECT ballance FROM account WHERE name=?', (name,))
        current = c.fetchone()
        if current == None:
            return current
        else:
            return current[0]

    def insert(self, name, amount):
        c = self.conn.cursor()
        c.execute('INSERT INTO account (name, ballance) VALUES (?, ?)', (name, amount))

    def update(self, name, amount):
        c = self.conn.cursor()
        c.execute('UPDATE account SET ballance = ? WHERE name = ?', (amount, name))
  

Testing the whole application

• Implement the tests without the need for a database.

import os
import pytest

import db
from bank import Bank


def test_app(tmpdir):
    db.db_filename = os.path.join( tmpdir, 'test.db' )
    app = Bank()
    app.setup()
    assert app.status('foo') == None
    assert app.status('bar') == None

    app.deposit('foo', 100)
    assert app.status('foo') == 100

    app.deposit('foo', 10)
    assert app.status('foo') == 110

    app.deposit('bar', 130)
    assert app.status('foo') == 110
    assert app.status('bar') == 130

    app.transfer('foo', 'bar', 19)
    assert app.status('foo') == 91
    assert app.status('bar') == 149

    with pytest.raises(Exception) as exinfo:
        app.transfer('foo', 'bar', 200)
    assert exinfo.type == Exception
    assert str(exinfo.value) == 'Not enough money' 

Resources

• MonkeyPatching
• Python slides
• Python testing with pytest
• Test Doubles - Fakes, Mocks and Stubs.
• Martin Fowler: Test Double

Retrospective

• What went well?
• What needs improvement?

Job searching help

• LinkedIn
• Open source projects

Solutions - game

import os
import sys
root = os.path.dirname( os.path.dirname( os.path.abspath(__file__) ))
sys.path.insert(0, os.path.join(root, 'space-fight'))

import game

def test_immediate_exit():
    input_values = ['x']
    output = []

    def mock_input(s):
       output.append(s)
       return input_values.pop(0)
    game.input = mock_input
    game.print = lambda s : output.append(s)

    game.play()

    assert output == [
        '\nWelcome to another Number Guessing game',
        'Please enter your guess [x|s|d|m|n]: ',
        'x',
        'Sad to see you leave early',
    ] 

import os
import sys
root = os.path.dirname( os.path.dirname( os.path.abspath(__file__) ))
sys.path.insert(0, os.path.join(root, 'space-fight'))

import game
import random

def test_immediate_exit():
    input_values = ['30', '50', '42', 'x']
    output = []

    def mock_input(s):
       output.append(s)
       return input_values.pop(0)
    game.input = mock_input
    game.print = lambda s : output.append(s)
    random.randrange = lambda a, b : 42

    game.play()

    assert output == [
        '\nWelcome to another Number Guessing game',
        'Please enter your guess [x|s|d|m|n]: ',
        '30',
        'Your guess is too low',
        'Please enter your guess [x|s|d|m|n]: ',
        '50',
        'Your guess is too high',
        'Please enter your guess [x|s|d|m|n]: ',
        '42',
        'Hit!',
        '\nWelcome to another Number Guessing game',
        'Please enter your guess [x|s|d|m|n]: ',
        'x',
        'Sad to see you leave early',
    ] 

Solutions - Mocking the database access

import os
import sys
root = os.path.dirname( os.path.dirname( os.path.abspath(__file__) ))
sys.path.insert(0, os.path.join(root, 'exdb'))
import pytest


import db
from bank import Bank


class MockDB(object):
    def create(self):
        self.db = {}

    def get(self, name):
        return self.db.get(name)

    def insert(self, name, amount):
        self.db[name] = amount

    def update(self, name, amount):
        self.db[name] = amount


db.DB = MockDB

def test_app(tmpdir):
    app = Bank()
    app.setup()
    assert app.status('foo') == None
    assert app.status('bar') == None

    app.deposit('foo', 100)
    assert app.status('foo') == 100

    app.deposit('foo', 10)
    assert app.status('foo') == 110

    app.deposit('bar', 130)
    assert app.status('foo') == 110
    assert app.status('bar') == 130

    app.transfer('foo', 'bar', 19)
    assert app.status('foo') == 91
    assert app.status('bar') == 149

    with pytest.raises(Exception) as exinfo:
        app.transfer('foo', 'bar', 200)
    assert exinfo.type == Exception
    assert str(exinfo.value) == 'Not enough money'