test remote browsers using sauce labs

.travis.yml

@@ -1,5 +1,7 @@
 language: cpp
 compiler: gcc
+addons:
+  sauce_connect: true
 before_install:
   - sudo add-apt-repository ppa:fontforge/fontforge --yes
   - sudo add-apt-repository ppa:coolwanglu/pdf2htmlex --yes
@@ -8,12 +10,15 @@ before_install:
   - sudo pip install selenium
   - export DISPLAY=:99.0
   - test/start_xvfb.sh
+  - pushd tests/browser_tests
+  - python -m SimpleHTTPServer 8000 &
+  - popd
   - sleep 5
 before_script: 
   - cmake -DENABLE_SVG=ON .
 script:
   - make
-  - ctest --output-on-failure
+  - P2H_TEST_REMOTE=1 ctest --output-on-failure
   - sudo make install
   - /usr/local/bin/pdf2htmlEX -v
 branches:
@@ -21,3 +26,7 @@ branches:
     - master
     - incoming
     - wl
+env:
+  global:
+    - secure: V0yGXROTAsRc3ExcECj7X/CrJLbodUeqZyfQGkA6x0iLV7Lh8/hgTjSsvuj7ef/DIWMqJ5cAIzZuXiF0KIxiVllF1v0I3w+LScxynT7B1NsyH16hvGIc7EvrsRmGVeTv8n9I+cCIwQxjtliNKfeZjV4Rk2+u6LioUzTszmW2etc=
+    - secure: Q5ZSrdFEgN0JvUp90nY5Wh58iukmGZQ2EW7crOibWH2yuUsxAnMELxpY+9yV3+eA7kbjJf/I0NCa5ZY1gkxK60ugUj+zuUDTL+BV1XCbO37e0uwh3ae99iyQWpXc8e8wBp10sthoX7U6Hvypa5tD9r1JJib8jxJV/MzIFpb7H9s=

test/test.py

@@ -85,21 +85,30 @@ if __name__ == '__main__':
         exit(1)
     suites = []
     loader = unittest.TestLoader()
-    all_modules = ['test_output', 'test_local_browser']
+
+    all_modules = []
+    all_modules.append(__import__('test_output'))
+    all_modules.append(__import__('test_local_browser'))
+    all_classes = ['test_output', 'test_local_browser']
+
+    if os.environ.get('P2H_TEST_REMOTE'):
+        m = __import__('test_remote_browser')
+        all_modules.append(m)
+        all_classes += m.test_classnames
+
     test_names = []
     for name in sys.argv[1:]:
-        if name in all_modules or name.find('.') != -1:
+        if name.find('.') != -1:
             test_names.append(name)
         else:
-            for m in all_modules:
+            for m in all_classes:
                 test_names.append(m + '.' + name)
     
-    for module_name in all_modules:
-        __import__(module_name)
+    for module in all_modules:
         if len(test_names) > 0:
             for n in test_names:
                 try:
-                    suites.append(loader.loadTestsFromName(n, sys.modules[module_name]))
+                    suites.append(loader.loadTestsFromName(n, module))
                 except:
                     pass
         else:

test/test_local_browser.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+# Run browsers tests with a local Firefox
+
 import unittest
 
 from selenium import webdriver

test/test_remote_browser.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+# Run browser tests through Sauce Labs
+
+import unittest
+import sys
+import os
+
+from selenium import webdriver
+from sauceclient import SauceClient
+from browser_tests import BrowserTests
+
+# Set your own environment variables
+USERNAME = os.environ.get('SAUCE_USERNAME')
+ACCESS_KEY = os.environ.get('SAUCE_ACCESS_KEY')
+
+# The base url that remote browser will access
+# Usually a HTTP server should be set up in the folder containing the test cases
+# Also Sauce Connect should be enabled
+BASEURL='http://localhost:8000/'
+
+SAUCE_OPTIONS = {
+    'record-video': False,
+    'record-screenshots': False,
+    'record-logs': False,
+    'sauce-advisor': False,
+}
+
+# we want to test the latest stable version
+# and 'beta' is usually the best estimation
+BROWSER_MATRIX = [
+    ('win_ie', {
+        'platform': 'Windows 8.1',
+        'browserName': 'internet explorer',
+        'version': '11',
+    }),
+    ('win_firefox', {
+        'platform': 'Windows 8.1',
+        'browserName': 'firefox',
+        'version': 'beta',
+    }),
+    ('win_chrome', {
+        'platform': 'Windows 8.1',
+        'browserName': 'chrome',
+        'version': 'beta',
+    }),
+    ('mac_firefox', {
+        'platform': 'OS X 10.9',
+        'browserName': 'firefox',
+        'version': 'beta',
+    }),
+    ('mac_chrome', {
+        'platform': 'OS X 10.9',
+        'browserName': 'chrome',
+        'version': '38', # beta is not supported
+    }),
+    ('linux_firefox', {
+        'platform': 'Linux',
+        'browserName': 'firefox',
+        'version': 'beta',
+    }),
+    ('linux_chrome', {
+        'platform': 'Linux',
+        'browserName': 'chrome',
+        'version': 'beta',
+    }),
+]
+
+class test_remote_browser_base(BrowserTests):
+    @classmethod
+    def setUpClass(cls):
+        super(test_remote_browser_base, cls).setUpClass()
+        if not cls.GENERATING_MODE:
+            cls.sauce = SauceClient(USERNAME, ACCESS_KEY)
+            cls.sauce_url = 'http://%s:%s@ondemand.saucelabs.com:80/wd/hub' % (USERNAME, ACCESS_KEY)
+
+            cls.browser = webdriver.Remote(
+                desired_capabilities=cls.desired_capabilities,
+                command_executor=cls.sauce_url
+            )
+
+            cls.browser.implicitly_wait(30)
+            # remote screen may not be large enough for the whole page
+            cls.browser.set_window_size(cls.BROWSER_WIDTH, cls.BROWSER_HEIGHT)
+
+    @classmethod
+    def tearDownClass(cls):
+        if not cls.GENERATING_MODE:
+            cls.browser.quit()
+        super(test_remote_browser_base, cls).tearDownClass()
+
+    def setUp(self):
+        super(test_remote_browser_base, self).setUp()
+        sys.exc_clear()
+
+    def tearDown(self):
+        try:
+            passed = (sys.exc_info() == (None, None, None))
+            branch = os.environ.get('TRAVIS_BRANCH', 'manual')
+            pull_request = os.environ.get('TRAVIS_PULL_REQUEST')
+            self.sauce.jobs.update_job(self.browser.session_id, 
+                build_num=os.environ.get('TRAVIS_BUILD_NUMBER', 0),
+                name='pdf2htmlEX',
+                passed=passed,
+                public='public restricted',
+                tags = [pull_request] if pull_request else [branch]
+            )
+        except:
+            pass
+
+    def generate_image(self, html_file, png_file):
+        self.browser.get(BASEURL + html_file)
+        self.browser.save_screenshot(png_file)
+
+test_classnames = []
+
+def generate_classes():
+    module = globals()
+    for browser in BROWSER_MATRIX:
+        d = dict(test_remote_browser_base.__dict__)
+        caps = SAUCE_OPTIONS.copy()
+        caps.update(browser[1])
+        tunnel_identifier = os.environ.get('TRAVIS_JOB_NUMBER')
+        if tunnel_identifier:
+            caps['tunnel-identifier'] = tunnel_identifier
+        d['desired_capabilities'] = caps
+        name = "test_remote_%s" % (browser[0], )
+        module[name] = type(name, (test_remote_browser_base, unittest.TestCase), d)
+        test_classnames.append(name)
+
+generate_classes()