diff --git a/.travis.yml b/.travis.yml index cdc85ab..cccc1d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: - sudo pip install selenium sauceclient - export DISPLAY=:99.0 - test/start_xvfb.sh - - pushd test/browser_tests + - pushd / - python -m SimpleHTTPServer 8000 & - popd - sleep 5 diff --git a/test/README.md b/test/README.md index d81facd..eeac300 100644 --- a/test/README.md +++ b/test/README.md @@ -15,10 +15,16 @@ - `./test.py test_local_browser.test_basic_text` - Or `./test.py test_basic_text - Environment variables: - - set `P2H_TEST_SAVE_TMP=1` to keep the temporary files in `/tmp/pdf2htmlEX_test` - - set `P2H_TEST_GEN=1` to generate new reference files + - Set `P2H_TEST_SAVE_TMP=1` to keep the temporary files in `/tmp/pdf2htmlEX_test` + - Set `P2H_TEST_GEN=1` to generate new reference files + - Set `P2H_TEST_REMOTE=1` to test different browsers using Sauce Labs + - Install `sauceclient` for Python + - Set correct values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` + - Setup a HTTP server at `/` on port 8000 + - Enable Sauce Connect + - See `.travis.yml` as an example -### Guidelines for test cases +### Add new test cases - Make sure you have the proper copyrights. - Using meaningful file names, a description of the file, or issueXXX.pdf. @@ -27,4 +33,6 @@ - Grayscale only, unless the test case is about colors. - Remove unnecessary elements. - [Optional] Include the source files that the PDF file is generated from. +- Add the new PDF file to the correct folder in `test/`, and add a new function in the corresponding Python file +- Run `P2H_TEST_GEN=1 test/test.py test_issueXXX` to generate the reference, assuming that the new function is called `test_issueXXX` diff --git a/test/browser_tests.py b/test/browser_tests.py index 9323e07..c6cbba4 100644 --- a/test/browser_tests.py +++ b/test/browser_tests.py @@ -3,6 +3,7 @@ import os import subprocess import shutil +import unittest from PIL import Image, ImageChops from test import Common @@ -73,6 +74,14 @@ class BrowserTests(Common): diff_img.crop(diff_img.getbbox()).convert('RGB').save(os.path.join(png_out_dir, basefilename + '.diff.png')) self.fail('PNG files differ') + @unittest.skipIf(Common.GENERATING_MODE, 'Do not auto generate reference for test_fail') + def test_fail(self): + # The HTML reference is generated manually, which mismatches the PDF + # To test if the environment can detect any errors + # E.g. when network is down, 404 message is shown for any HTML message + with self.assertRaises(AssertionError): + self.run_test_case('test_fail.pdf') + def test_basic_text(self): self.run_test_case('basic_text.pdf') diff --git a/test/browser_tests/test_fail.pdf b/test/browser_tests/test_fail.pdf new file mode 100644 index 0000000..86882f8 Binary files /dev/null and b/test/browser_tests/test_fail.pdf differ diff --git a/test/browser_tests/test_fail.tex b/test/browser_tests/test_fail.tex new file mode 100644 index 0000000..48e5045 --- /dev/null +++ b/test/browser_tests/test_fail.tex @@ -0,0 +1,9 @@ +\documentclass{article} +\begin{document} +\Huge +The quick brown fox jumps over the lazy dog + +The quick brown fox jumps over the lazy dog + +The quick brown fox jumps over the lazy dog +\end{document} diff --git a/test/browser_tests/test_fail/THIS_FOLDER_SHOULD_BE_EMPTY b/test/browser_tests/test_fail/THIS_FOLDER_SHOULD_BE_EMPTY new file mode 100644 index 0000000..e69de29 diff --git a/test/test.py b/test/test.py index 294e2f9..8fa0494 100755 --- a/test/test.py +++ b/test/test.py @@ -13,8 +13,8 @@ class Common(object): DATA_DIR = os.path.join(SRC_DIR, 'share') PDF2HTMLEX_PATH = os.path.join(SRC_DIR, 'pdf2htmlEX') - SAVE_TMP = os.environ.get('P2H_TEST_SAVE_TMP') - GENERATING_MODE = os.environ.get('P2H_TEST_GEN') + SAVE_TMP = bool(os.environ.get('P2H_TEST_SAVE_TMP')) + GENERATING_MODE = bool(os.environ.get('P2H_TEST_GEN')) CANONICAL_TEMPDIR = '/tmp/pdf2htmlEX_test' @@ -91,7 +91,7 @@ if __name__ == '__main__': all_modules.append(__import__('test_local_browser')) all_classes = ['test_output', 'test_local_browser'] - if os.environ.get('P2H_TEST_REMOTE'): + if bool(os.environ.get('P2H_TEST_REMOTE')): m = __import__('test_remote_browser') all_modules.append(m) all_classes += m.test_classnames diff --git a/test/test_output.py b/test/test_output.py index 741cd82..e7e30a3 100644 --- a/test/test_output.py +++ b/test/test_output.py @@ -7,10 +7,9 @@ import os from test import Common +@unittest.skipIf(Common.GENERATING_MODE, 'Skipping test_output in generating mode') class test_output(Common, unittest.TestCase): def run_test_case(self, input_file, expected_output_files, args=[]): - if self.GENERATING_MODE: - self.skipTest("Skipping test_output test cases in generating mode") args = list(args) args.insert(0, os.path.join(self.TEST_DIR, 'test_output', input_file)) self.assertItemsEqual(self.run_pdf2htmlEX(args)['output_files'], expected_output_files) diff --git a/test/test_remote_browser.py b/test/test_remote_browser.py index 9fcc266..f24be67 100755 --- a/test/test_remote_browser.py +++ b/test/test_remote_browser.py @@ -21,9 +21,8 @@ BASEURL='http://localhost:8000/' SAUCE_OPTIONS = { 'record-video': False, - 'record-screenshots': False, + 'record-screenshots': True, 'record-logs': False, - 'sauce-advisor': False, } # we want to test the latest stable version @@ -99,7 +98,7 @@ class test_remote_browser_base(BrowserTests): branch = os.environ.get('TRAVIS_BRANCH', 'manual') pull_request = os.environ.get('TRAVIS_PULL_REQUEST') self.sauce.jobs.update_job(self.browser.session_id, - build_num=os.environ.get('TRAVIS_BUILD_NUMBER', 0), + build_num=os.environ.get('TRAVIS_BUILD_NUMBER', '0'), name='pdf2htmlEX', passed=passed, public='public restricted',