Unittesting, doctesting and userinterface testing

This is a script file to run all your tests in one go. Could be used for continuous integration testing. It is a augmented version of original work by jonromero and matclab.

testRunner.py

#!/usr/bin/python
"""
Execute with:
>   python web2py.py -S appname -M -R testRunner.py

o Runs all tests that exist in the tests directories of 'appname'.
o Also runs all doctests and userinterface tests
o Adds each test to the unittestsuite

Inside the unittest file the class should have a name based on the 
file's path, like this:

FilenameDirectory -> DefaultTasksModel

for example:
applications/[appname]/tests/controllers/default.py
is
class DefaultController(unittest.TestCase)

BEWARE that the name is NOT in plural (controllers->Controller)

depends on python modules unittest and selenium.

For documentation see slice 67 and 142 at http://www.web2pyslices.com

Original by 
02/03/2009
Jon Vlachoyiannis
jon@emotionull.com

Changes and additions:
o appname 
o enable running on windows by using os.path
o moved db_test in from models 
o added UI tests using Selenium

version 0.91
2011/08/08
Nico de Groot
ndegroot0@gmail.com
"""

from gluon import current
import unittest
import glob
import sys
import doctest
import os
import traceback
from copy import copy


# create a test database by copying the original db
test_db = DAL('sqlite://testing.sqlite')
for tablename in db.tables:  # Copy tables!
    table_copy = [copy(f) for f in db[tablename]]
    test_db.define_table(tablename, *table_copy)

db=test_db
current.app.db=db

def showfeedback():
    exc_type, exc_value, exc_traceback = sys.exc_info()
    print '-'*60
    for line in traceback.format_exception(exc_type, exc_value,exc_traceback):
        print line[:-1]
    print '-'*60


def custom_execfile(test_file):
    try:
        sys.path.append(os.path.split(test_file)[0]) # to support imports form current folder in the testfiles
        g=copy(globals())
        execfile(test_file, g) 
    except (WindowsError,ValueError,SystemExit):
        pass # we know about the rotating logger error...
             # and SystemExit is not useful to detect
    except:
        showfeedback()
    return g

suite = unittest.TestSuite()

appname= current.request.application

# find unittests
path=os.path.join('applications',appname,'tests','*','*.py')
test_files = glob.glob(path)
test_files =[x for x in test_files if not os.path.split(x)[0].endswith('userinterface')]

print len(test_files)," unittest files found."

# find doctests in controller
path=os.path.join('applications',appname,'controllers','*.py')
doc_test_files = glob.glob(path)
print len(doc_test_files)," controller files with possible doctests found."

# find selenium unittests
path=os.path.join('applications',appname,'tests','userinterface','case_*.py')
selenium_test_files = glob.glob(path)
print len(selenium_test_files)," userinterface testcases found."


if not test_files and not doc_test_files and not selenium_test_files:
    print "No unit, doctest or userinterface test files found for application: " + appname

# Bring in all doc tests and submit them
print "Run doctests" if doc_test_files else "No doctests"
for f in  doc_test_files:
    g=custom_execfile(f)
    suite.addTest(doctest.DocFileSuite(f, globs=g,
            module_relative=False))


# Bring in all unit tests and their controllers/models/whatever
print "Run unittests" if test_files else "No unittests"
for test_file in test_files:
    g=custom_execfile(test_file)
    # Create the appropriate class name based on filename and path
    components = os.path.split(test_file)
    filename = str.capitalize(components[-1][:-3]) # without .py
    directory = str.capitalize(os.path.split(components[-2])[-1])

    # Load the to-be-tested file
    to_be_tested_file = os.path.join("applications",
                                      appname,
                                      directory.lower() ,
                                      filename.lower() + ".py")

    #send class name (attribute of g) to the suite
    suite.addTest(unittest.makeSuite(g[filename+directory[:-1]] )) # lose the s

# Bring in all selenium tests and submit them to the suite
print "Run external UI tests (Selenium)" if selenium_test_files else "No external UI tests"
for test_file in  selenium_test_files:

    g = custom_execfile(test_file)
    # reconstruct the class name from filename
    components = os.path.split(test_file)
    filename = str.capitalize(components[-1][:-3]) # without .py
    classname = str.capitalize(filename.split("_")[1]) 

    #pass the classname to Suite
    suite.addTest(unittest.makeSuite(g[classname])) 

# lets get rolling
unittest.TextTestRunner(verbosity=2).run(suite)

helper

def form_postvars(tablename, fields, request, action="create", 
        record_id=None):
    """
    Creates the appropriate request vars for forms
    """

    vars = {}
    for field_name in fields:
        vars[field_name] = fields[field_name]
    if action == "create":
        vars["_formname"] = tablename + "_" + action
    elif action == "update":
        vars["_formname"] = tablename + "_" + str(record_id)
        vars["id"] = record_id
    elif action == "delete":
        vars["_formname"] = tablename + "_" + str(record_id)
        vars["id"] = record_id
        vars["delete_this_record"] = True
    elif action:
        vars["_formname"] = action

    request['vars'].update(vars)
    request['post_vars'].update(vars)

howto and conventions

1. Put this script in the web2py root folder, with the name *testRunner.p*y
2. Put the helper script in the gluon contrib folder, name it test_helpers.py
3. Install unittest and selenium using pip or easy_install.
4. Create a tests folder in your application folder and go inside this folder
5. Create a folder controllers and/or models for your unittests
6. Create a folder userinterface for your ui tests

See the web2py book for examples of Doctests, see below for an example of a unittest and a userinterface test

unit test

For now see the origin slice for information.

[appname]/tests/controller/default.py

#!/usr/bin/python
#found when running python web2py.py -S welcome -M -R testRunner.py 

import unittest
import cPickle as pickle
from gluon import * 
from gluon.contrib.test_helpers import form_postvars
from gluon import current

class DefaultController(unittest.TestCase):
    def __init__(self, p):
        global auth, session, request
        unittest.TestCase.__init__(self, p)
        self.session = pickle.dumps(current.session)
        current.request.application = 'welcome'
        current.request.controller = 'default'
        self.request = pickle.dumps(current.request)

    def setUp(self):
        global response, session, request, auth
        current.session = pickle.loads(self.session)
        current.request = pickle.loads(self.request)
        auth = Auth(globals(), db)
        auth.define_tables()

    def _testRedirect(self, callable, url="/index"):
        try:
            resp = callable() # auth.register() creates & submits registration form
            self.fail("%s should raise an exception\n%s" % (
                callable.__name__,
                resp.errors))
        except HTTP, e:
            self.assertTrue(e.headers['Location'] == url,
                    "Wrong redirection url for unauthenticated user on %s() : %s (%s)" %
                    (callable.__name__, e.headers['Location'],url))
        else:
            self.fail("%s should raise an HTTP exception\n%s" % (
                callable.__name__,
                e))

    def emptyUserDB(self):
        db(db.auth_user.id>0).delete()
        db.commit()

    def testRegisterSuccess(self):
        self.emptyUserDB()
        # Register a user in the db
        current.request.function='register'
        resp = auth.register() # get the form
        current.request = form_postvars("auth_user", {
            "email": "essai@gmail.com",
            "first_name": "e_first",
            "last_name": "e_last",
            "password" : "blob",
            "password_two": "blob",
            "_formkey": resp.formkey,
            },
            current.request, action="register",
            record_id=None, )     
        self._testRedirect(auth.register,'/welcome/default/index')
        self.assertTrue(auth.is_logged_in())
        self.assertEquals(auth.user.first_name, 'e_first')

userinterface test

You can read my blog for a introduction to Selenium. The file below is not very general, it is is tied to my application named dibsa. It tests an registration form called Aanmelding. This testcase imports a file named util(.py) from the same folder.

case_aanmelding.py

""" Uses Selenium2/webdriver to test the UI of DIBSA
    Version 0.9: 
    contain tests for aanmelding/registration page
"""
import unittest, time, re
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
from util import Browser


class Aanmelding(unittest.TestCase):
    def setUp(self):
        """test using Chrome"""
        self.verificationErrors = []
        self.browser = Browser("chrome") #webdriver.Chrome()
        self.action_chains = ActionChains(self.browser)

    def set_chrome(self):
        self.browser = Browser("chrome") #webdriver.Chrome()
        self.action_chains = ActionChains(self.browser)
    def set_firefox(self):
        self.browser = Browser("firefox") #webdriver.Firefox()
        self.action_chains = ActionChains(self.b)

    def test_aanmelding_c_is_p(self):
 #       self.set_chrome()
        self._aanmelding_c_is_p()

    def _aanmelding_c_is_p(self, variant=1):
        """ user is participant, from basket:
        1: autopay
        2: invoice requested"""
        b=self.browser
        b.get("http://127.0.0.1:8000/dibsa/aanmelding/index?sdnr=2")
        assert "Dibsa" in b.title
        b.find_element_by_name("company").send_keys("TST")
        b.find_element_by_name("firstname").send_keys("N.C.")
        b.find_element_by_name("prefix").send_keys("de")
        b.find_element_by_name("lastname").send_keys("Groot")
        el = b.find_element_by_id("person_gender")
        #el.click() # selecteer
        el.send_keys("M")
        el = b.find_element_by_id("pref0")
        assert el.text=='Geen voorkeur, klik op knopje ->ws eenws tweews drie'
        b.find_element_by_name("street").send_keys("Heidelberglaan 2")
        b.find_element_by_name("postalcode").send_keys("3584 CS")
        b.find_element_by_name("city").send_keys("Utrecht")
        b.find_element_by_name("email").send_keys("n.c.degroot(A)uvt.nl")
        b.find_element_by_name("memo").send_keys("Testcase")
        b.find_element_by_xpath("//input[@value='Verder']").click()
        assert "toon_overzicht" in b.current_url

    def tst_aanmelding_c_is_not_p(self):
        b=self.browser
        b.get("http://127.0.0.1:8000/dibsa/aanmelding/index?sdnr=2")
        b.find_element_by_name("company").send_keys("TST")
        b.find_element_by_name("firstname").send_keys("N.C.")
        b.find_element_by_name("prefix").send_keys("de")
        b.find_element_by_name("lastname").send_keys("Groot")
        el = b.find_element_by_id("person_gender")
        el.click() # selecteer
        el.send_keys("M")
        b.find_element_by_name("isParticipant").click()

        b.find_element_by_name("street").send_keys("Heidelberglaan 2")
        b.find_element_by_name("postalcode").send_keys("3584 CS")
        b.find_element_by_name("city").send_keys("Utrecht")
        b.find_element_by_name("email").send_keys("n.c.degroot@uvt.nl")
        b.find_element_by_name("memo").send_keys("Testcase")
        b.find_element_by_xpath("//input[@value='Verder']").click()
        assert "toon_overzicht" in b.current_url


    def tearDown(self):
        self.browser.close()
        self.assertEqual([], self.verificationErrors)

if __name__ == "__main__":
    unittest.main()
print("code sample");

util.py

""" model en util classes         
"""
# global packages
import os
import time
import logging
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains


class Browser(object):
    def __init__(self,name="chrome"):
        if name=="chrome":
            self.browser=webdriver.Chrome()
        elif name == "firefox":
            self.browser=webdriver.Firefox()
        else:
            raise Exception("NYI")
        self.action_chains = ActionChains(self.browser)

    @property
    def title(self):
        return self.browser.title

    @property
    def current_url(self):
        return self.browser.current_url

    @property
    def url(self):
        return self.browser.current_url

    def set_chrome(self):
        self.browser = webdriver.Chrome()
        self.action_chains = ActionChains(self.browser)

    def set_firefox(self):
        self.browser = webdriver.Firefox()
        self.action_chains = ActionChains(self.b)

    def find_element_by_name(self,name):
        try:
            el=self.browser.find_element_by_name(name)
            return el
        except:
            msg ="Failed to find name %s"%name
            raise Exception(msg)

    def find_element_by_id(self,id):
        try:
            el=self.browser.find_element_by_id(id)
            return el
        except:
            msg = "Failed to find id %s"%id
            raise Exception(msg)

    def find_element_by_xpath(self,xpath):
        try:
            el=self.browser.find_element_by_xpath(xpath)
            return el
        except:
            msg = "Failed to find xpath %s"%xpath
            raise Exception(msg)

    def get(self,url):
        return self.browser.get(url)

    def close(self):
        time.sleep(4)
        return self.browser.close()

# 'safe' globals
logging=logging
base="http://127.0.0.1:8000"
app="dibsa"

class Person(dict):
    def __init__(self, \
                 company="FKT",\
                 firstname="N.C.",\
                 prefix="de", \
                 lastname="Groot", \
                 street="Heidelberglaan 2",\
                 postalcode="3584 CS",\
                 city="Utrecht",\
                 email="n.c.degroot(AT)uvt.nl",\
                 memo="Testcase",\
                 account="123"):
        self["company"]=company
        self["firstname"]=firstname
        self["prefix"]=prefix
        self["lastname"]=lastname
        self["street"]=street
        self["postalcode"]=postalcode
        self["city"]=city
        self["email"]=email
        self["memo"]=memo
        self["account"]=account

class Participant(dict):
    def __init__(self, \
                 #company="UvT",\
                 firstname="Henk",\
                 prefix="de", \
                 lastname="Vries", \
                 #street="Heidelberglaan 2",\
                 #postalcode="3584 CS",\
                 #city="Utrecht",\
                 #email="n.c.degroot(AT)uvt.nl",\
                 #memo="Testcase",\
                 #account="123"\
                 ):
#        self["company"]=company
        self["firstname"]=firstname
        self["prefix"]=prefix
        self["lastname"]=lastname

class Pref(dict):
    def __init__(self,pref0,pref1,pref2):
        self["pref0"]=pref0
        self["pref1"]=pref1
        self["pref2"]=pref2


def sendMail(cnaam):
    address=""
    maintainer="n.c.degroot(AT)uvt.nl"
    print "send mail to %s"%maintainer
    text='mailto:%s?subject=Studiedag%20testrapport'%(address)+\
         '&body=Resultaten' +\
         "zojuist heb ik de test '%s' gedraaid. "%cnaam
    os.startfile(text)