imporving tests[ci skip]

test/README.md

@@ -15,10 +15,15 @@
   - `./test.py test_local_browser.test_basic_text`
   - Or `./test.py test_basic_text
 - Environment variables:
-  - set `P2H_TEST_SAVE_TMP=1` to keep the temporary files in `/tmp/pdf2htmlEX_test`
-  - set `P2H_TEST_GEN=1` to generate new reference files
+  - Set `P2H_TEST_SAVE_TMP=1` to keep the temporary files in `/tmp/pdf2htmlEX_test`
+  - Set `P2H_TEST_GEN=1` to generate new reference files
+  - Set `P2H_TEST_REMOTE=1` to test different browsers using Sauce Labs
+    - Set correct values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY`
+    - Setup a HTTP server at `/` on port 8000
+    - Enable Sauce Connect
+    - See `.travis.yml` as an example
 
-### Guidelines for test cases
+### Add new test cases
 
 - Make sure you have the proper copyrights.
 - Using meaningful file names, a description of the file, or issueXXX.pdf.
@@ -27,4 +32,6 @@
   - Grayscale only, unless the test case is about colors.
   - Remove unnecessary elements.
 - [Optional] Include the source files that the PDF file is generated from.
+- Add the new PDF file to the correct folder in `test/`, and add a new function in the corresponding Python file
+- Run `P2H_TEST_GEN=1 test/test.py test_issueXXX` to generate the reference, assuming that the new function is called `test_issueXXX`
 

test/browser_tests.py

@@ -3,6 +3,7 @@
 import os
 import subprocess
 import shutil
+import unittest
 
 from PIL import Image, ImageChops
 from test import Common
@@ -73,6 +74,14 @@ class BrowserTests(Common):
                 diff_img.crop(diff_img.getbbox()).convert('RGB').save(os.path.join(png_out_dir, basefilename + '.diff.png'))
             self.fail('PNG files differ')
 
+    @unittest.skipIf(Common.GENERATING_MODE, 'Do not auto generate reference for test_fail')
+    def test_fail(self):
+        # The HTML reference is generated manually, which mismatches the PDF 
+        # To test if the environment can detect any errors
+        # E.g. when network is down, 404 message is shown for any HTML message
+        with self.assertRaises(AssertionError):
+            self.run_test_case('test_fail.pdf')
+
     def test_basic_text(self):
         self.run_test_case('basic_text.pdf')
 

test/browser_tests/test_fail.tex

@@ -0,0 +1,9 @@
+\documentclass{article}
+\begin{document}
+\Huge
+The quick brown fox jumps over the lazy dog
+
+The quick brown fox jumps over the lazy dog
+
+The quick brown fox jumps over the lazy dog
+\end{document}

test/test.py

@@ -13,8 +13,8 @@ class Common(object):
     DATA_DIR = os.path.join(SRC_DIR, 'share')
     PDF2HTMLEX_PATH = os.path.join(SRC_DIR, 'pdf2htmlEX')
 
-    SAVE_TMP = os.environ.get('P2H_TEST_SAVE_TMP')
-    GENERATING_MODE = os.environ.get('P2H_TEST_GEN')
+    SAVE_TMP = bool(os.environ.get('P2H_TEST_SAVE_TMP'))
+    GENERATING_MODE = bool(os.environ.get('P2H_TEST_GEN'))
 
     CANONICAL_TEMPDIR = '/tmp/pdf2htmlEX_test'
     
@@ -91,7 +91,7 @@ if __name__ == '__main__':
     all_modules.append(__import__('test_local_browser'))
     all_classes = ['test_output', 'test_local_browser']
 
-    if os.environ.get('P2H_TEST_REMOTE'):
+    if bool(os.environ.get('P2H_TEST_REMOTE')):
         m = __import__('test_remote_browser')
         all_modules.append(m)
         all_classes += m.test_classnames

test/test_output.py

@@ -7,10 +7,9 @@ import os
 
 from test import Common
 
+@unittest.skipIf(Common.GENERATING_MODE, 'Skipping test_output in generating mode')
 class test_output(Common, unittest.TestCase):
     def run_test_case(self, input_file, expected_output_files, args=[]):
-        if self.GENERATING_MODE:
-            self.skipTest("Skipping test_output test cases in generating mode")
         args = list(args)
         args.insert(0, os.path.join(self.TEST_DIR, 'test_output', input_file))
         self.assertItemsEqual(self.run_pdf2htmlEX(args)['output_files'], expected_output_files)

test/test_remote_browser.py

@@ -99,7 +99,7 @@ class test_remote_browser_base(BrowserTests):
             branch = os.environ.get('TRAVIS_BRANCH', 'manual')
             pull_request = os.environ.get('TRAVIS_PULL_REQUEST')
             self.sauce.jobs.update_job(self.browser.session_id, 
-                build_num=os.environ.get('TRAVIS_BUILD_NUMBER', 0),
+                build_num=os.environ.get('TRAVIS_BUILD_NUMBER', '0'),
                 name='pdf2htmlEX',
                 passed=passed,
                 public='public restricted',