Always conserve output files in /tmp/pdf2htmlEX.

CMakeLists.txt

@@ -215,8 +215,19 @@ set(PDF2HTMLEX_RESOURCE
 install (FILES ${PDF2HTMLEX_RESOURCE} DESTINATION share/pdf2htmlEX)
 install (FILES pdf2htmlEX.1 DESTINATION share/man/man1)
 
-include(CTest)
+## tests:
+
 set(PDF2HTMLEX_PATH ${CMAKE_BINARY_DIR}/pdf2htmlEX)
+set(PDF2HTMLEX_TMPDIR /tmp/pdf2htmlEX/tmp)
+set(PDF2HTMLEX_DATDIR /tmp/pdf2htmlEX/dat)
+set(PDF2HTMLEX_PNGDIR /tmp/pdf2htmlEX/png)
+set(PDF2HTMLEX_OUTDIR /tmp/pdf2htmlEX/out)
+file(MAKE_DIRECTORY ${PDF2HTMLEX_TMPDIR})
+file(MAKE_DIRECTORY ${PDF2HTMLEX_DATDIR})
+file(MAKE_DIRECTORY ${PDF2HTMLEX_PNGDIR})
+file(MAKE_DIRECTORY ${PDF2HTMLEX_OUTDIR})
 configure_file(${CMAKE_SOURCE_DIR}/test/test.py.in ${CMAKE_SOURCE_DIR}/test/test.py)
+
+include(CTest)
 add_test(test_basic   python ${CMAKE_SOURCE_DIR}/test/test_output.py)
 add_test(test_browser python ${CMAKE_SOURCE_DIR}/test/test_local_browser.py)

test/README.md

@@ -9,14 +9,9 @@
 
 ### Usage
 - Run all tests:
-  - `./test.py`
+  - python test_output.py
-- Run selected test suites:
+  - python test_local_browser.py
-  - `./test.py test_local_browser`
-- Run selected test case:
-  - `./test.py test_local_browser.test_basic_text`
-  - Or `./test.py test_basic_text`
 - Environment variables:
-  - Set `P2H_TEST_SAVE_TMP=1` to keep the temporary files in `/tmp/pdf2htmlEX_test`
   - Set `P2H_TEST_GEN=1` to generate new reference files
   - Set `P2H_TEST_REMOTE=1` to test different browsers using Sauce Labs
     - Install `sauceclient` for Python
@@ -24,6 +19,7 @@
     - Setup a HTTP server at `/` on port 8000
     - Enable Sauce Connect
     - See `.travis.yml` as an example
+    - python test_remote_browser.py
 
 ### Add new test cases
 
@@ -36,4 +32,3 @@
 - [Optional] Include the source files that the PDF file is generated from.
 - Add the new PDF file to the correct folder in `test/`, and add a new function in the corresponding Python file
 - Run `P2H_TEST_GEN=1 test/test.py test_issueXXX` to generate the reference, assuming that the new function is called `test_issueXXX`
-

test/browser_tests.py

@@ -35,7 +35,7 @@ class BrowserTests(Common):
         ref_htmlfolder = os.path.join(self.TEST_DATA_DIR, basefilename)
         ref_htmlfilename = os.path.join(ref_htmlfolder, htmlfilename)
 
-        out_htmlfilename = os.path.join(self.cur_output_dir, htmlfilename)
+        out_htmlfilename = os.path.join(self.OUTDIR, htmlfilename)
 
         self.assertEquals(extension.lower(), '.pdf', 'Input file is not PDF')
 
@@ -51,17 +51,14 @@ class BrowserTests(Common):
         if self.GENERATING_MODE:
             # copy generated html files
             shutil.rmtree(ref_htmlfolder, True)
-            shutil.copytree(self.cur_output_dir, ref_htmlfolder)
+            shutil.copytree(self.OUTDIR, ref_htmlfolder)
             return
 
-        png_out_dir = os.path.join(self.cur_temp_dir, 'png_out')
+        pngfilename_out_fullpath = os.path.join(self.PNGDIR, basefilename + '.out.png')
-        os.mkdir(png_out_dir)
-
-        pngfilename_out_fullpath = os.path.join(png_out_dir, basefilename + '.out.png')
         self.generate_image(out_htmlfilename, pngfilename_out_fullpath)
         out_img = Image.open(pngfilename_out_fullpath)
 
-        pngfilename_ref_fullpath = os.path.join(png_out_dir, basefilename + '.ref.png')
+        pngfilename_ref_fullpath = os.path.join(self.PNGDIR, basefilename + '.ref.png')
         self.generate_image(ref_htmlfilename, pngfilename_ref_fullpath, page_must_load=page_must_load)
         ref_img = Image.open(pngfilename_ref_fullpath)
 
@@ -71,11 +68,15 @@ class BrowserTests(Common):
         if diff_bbox is not None:
             diff_size = (diff_bbox[2] - diff_bbox[0]) * (diff_bbox[3] - diff_bbox[1])
             img_size = ref_img.size[0] * ref_img.size[1]
-            if self.SAVE_TMP:
+            diff_file_name = "<not saved>"
             # save the diff image
             # http://stackoverflow.com/questions/15721484/saving-in-png-using-pil-library-after-taking-imagechops-difference-of-two-png
-                diff_img.convert('RGB').save(os.path.join(png_out_dir, basefilename + '.diff.png'))
+            diff_file_name = os.path.join(self.PNGDIR, basefilename + '.diff.png')
-            self.fail('PNG files differ by <= %d pixels, (%f%% of %d pixels in total)' % (diff_size, 1.0*diff_size/img_size, img_size))
+            diff_img.convert('RGB').save(diff_file_name)
+            self.fail(('PNG files %s and %s differ by <= %d pixels, (%f%% of %d pixels in total), '+
+                       'difference: %s') %
+                      (pngfilename_out_fullpath, pngfilename_ref_fullpath,
+                       diff_size, 1.0*diff_size/img_size, img_size, diff_file_name))
 
     @unittest.skipIf(Common.GENERATING_MODE, 'Do not auto generate reference for test_fail')
     def test_fail(self):

test/test.py.in

@@ -21,27 +21,18 @@ class Common(object):
     TEST_DIR = os.path.join(SRC_DIR, 'test')
     DATA_DIR = os.path.join(SRC_DIR, 'share')
 
-    SAVE_TMP = bool(os.environ.get('P2H_TEST_SAVE_TMP'))
     GENERATING_MODE = bool(os.environ.get('P2H_TEST_GEN'))
 
-    CANONICAL_TEMPDIR = '/tmp/pdf2htmlEX_test'
+    # temporary directories defined in CMakeLists.txt:
+    TMPDIR = "@PDF2HTMLEX_TMPDIR@"
+    PNGDIR = "@PDF2HTMLEX_PNGDIR@"
+    DATDIR = "@PDF2HTMLEX_DATDIR@"
+    OUTDIR = "@PDF2HTMLEX_OUTDIR@"
 
     def setUp(self):
-        if not self.SAVE_TMP:
-            self.cur_temp_dir = tempfile.mkdtemp(prefix='pdf2htmlEX_test')
-        else:
-            shutil.rmtree(self.CANONICAL_TEMPDIR, True)
-            os.mkdir(self.CANONICAL_TEMPDIR)
-            self.cur_temp_dir = self.CANONICAL_TEMPDIR
-
-        self.cur_data_dir = os.path.join(self.cur_temp_dir, 'share')
-        self.cur_output_dir = os.path.join(self.cur_temp_dir, 'out')
-        os.mkdir(self.cur_data_dir)
-        os.mkdir(self.cur_output_dir)
-
         # filter manifest
         with open(os.path.join(self.DATA_DIR, 'manifest')) as inf:
-            with open(os.path.join(self.cur_data_dir, 'manifest'), 'w') as outf:
+            with open(os.path.join(self.DATDIR, 'manifest'), 'w') as outf:
                 ignore = False
                 for line in inf:
                     if ignore:
@@ -54,13 +45,9 @@ class Common(object):
 
         # copy files
         shutil.copy(os.path.join(self.DATA_DIR, 'base.min.css'),
-                os.path.join(self.cur_data_dir, 'base.min.css'))
+                os.path.join(self.DATDIR, 'base.min.css'))
         shutil.copy(os.path.join(self.TEST_DIR, 'fancy.min.css'),
-                os.path.join(self.cur_data_dir, 'fancy.min.css'))
+                os.path.join(self.DATDIR, 'fancy.min.css'))
-
-    def tearDown(self):
-        if not self.SAVE_TMP:
-            shutil.rmtree(self.cur_temp_dir, True)
 
     def run_pdf2htmlEX(self, args):
         """
@@ -71,9 +58,12 @@ class Common(object):
         :return: an object of relevant info
         """
 
+        shutil.rmtree(self.TMPDIR, ignore_errors=False, onerror=None)
+        os.mkdir(self.TMPDIR)
+
         args = [Common.PDF2HTMLEX_PATH,
-               '--data-dir', self.cur_data_dir,
+               '--data-dir', self.DATDIR,
-               '--dest-dir', self.cur_output_dir
+               '--dest-dir', self.TMPDIR
               ] + args
 
         with open(os.devnull, 'w') as fnull:
@@ -81,7 +71,9 @@ class Common(object):
 
         self.assertEquals(return_code, 0, 'cannot execute pdf2htmlEX')
 
-        files = os.listdir(self.cur_output_dir)
+        files = os.listdir(self.TMPDIR)
+        for file in files:
+            shutil.copy(os.path.join(self.TMPDIR,file), self.OUTDIR)
 
         return {
             'return_code' : return_code,

test/test_output.py

@@ -14,6 +14,7 @@ class test_output(Common, unittest.TestCase):
         args = list(args)
         args.insert(0, os.path.join(self.TEST_DIR, 'test_output', input_file))
         result = self.run_pdf2htmlEX(args)
+        self.maxDiff = None
         if expected_output_files:
             self.assertItemsEqual(result['output_files'], expected_output_files)
             print("test_output ", input_file, ": matched ", expected_output_files)