Removed flaky visual regression tests

This commit is contained in:
squidfunk
2017-05-15 10:59:31 +02:00
committed by Martin Donath
parent 58288f87ac
commit 2bc2517658
607 changed files with 13689 additions and 1922 deletions

View File

@@ -1,65 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import ecstatic from "ecstatic"
import * as http from "http"
/* ----------------------------------------------------------------------------
* Locals
* ------------------------------------------------------------------------- */
/* Static file server */
let server = null
/* ----------------------------------------------------------------------------
* Functions
* ------------------------------------------------------------------------- */
/**
* Start static file server
*
* @param {string} directory - Directory to serve
* @param {number} port - Port to listen on
* @param {Function} done - Resolve callback
*/
export const start = (directory, port, done) => {
server = http.createServer(ecstatic({
root: directory
}))
/* Listen and register signal handlers */
server.listen(port, "127.0.0.1", done)
for (const signal of ["SIGTERM", "SIGINT", "exit"])
process.on(signal, stop)
}
/**
* Stop static file server
*
* @param {Function} done - Resolve callback
*/
export const stop = done => {
if (server) {
server.close(done)
server = null
}
}

View File

@@ -1,71 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import launcher from "sauce-connect-launcher"
/* ----------------------------------------------------------------------------
* Locals
* ------------------------------------------------------------------------- */
/* SauceConnect process */
let server = null
/* ----------------------------------------------------------------------------
* Functions
* ------------------------------------------------------------------------- */
/**
* Open SauceConnect tunnel
*
* @param {string} id - Unique identifier
* @param {string} username - SauceConnect username
* @param {string} accesskey - SauceConnect accesskey
* @param {Function} done - Resolve callback
*/
export const start = (id, username, accesskey, done) => {
launcher({
username,
accessKey: accesskey,
tunnelIdentifier: id
}, (err, proc) => {
if (err)
throw new Error(err)
server = proc
done()
})
/* Register signal handlers */
for (const signal of ["SIGTERM", "SIGINT", "exit"])
process.on(signal, stop)
}
/**
* Close SauceConnect tunnel
*
* @param {Function} done - Resolve callback
*/
export const stop = done => {
if (server) {
server.close(done)
server = null
}
}

View File

@@ -1,87 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import selenium from "selenium-standalone"
/* ----------------------------------------------------------------------------
* Locals
* ------------------------------------------------------------------------- */
/* Selenium server */
let server = null
/* ----------------------------------------------------------------------------
* Definition
* ------------------------------------------------------------------------- */
/**
* Start Selenium
*
* @param {Function} done - Resolve callback
*/
export const start = done => {
selenium.start({}, (err, proc) => {
/* Register signal handlers */
for (const signal of ["SIGTERM", "SIGINT", "exit"])
process.on(signal, stop)
if (err) {
/* Install selenium, if not present */
if (/^Missing(.*)chromedriver$/.test(err.message)) {
new Promise(resolve => {
selenium.install({}, resolve)
})
/* Start selenium again */
.then(() => {
selenium.start({}, (err_, proc_) => {
server = proc_
done()
})
})
/* Otherwise, throw error */
} else {
throw err
}
}
/* Remember process handle */
server = proc
done()
})
}
/**
* Stop Selenium
*
* @param {Function} done - Resolve callback
*/
export const stop = done => {
if (server) {
if (typeof done === "function")
server.on("exit", done)
server.kill()
server = null
}
}

View File

@@ -1,45 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import path from "path"
import { Server, stopper } from "karma"
/* ----------------------------------------------------------------------------
* Task: start karma test runner
* ------------------------------------------------------------------------- */
export default () => {
return done => {
new Server({
configFile: path.join(process.cwd(), "tests/karma.conf.js")
}, done).start()
/* Register signal handler for all relevant events */
for (const signal of ["SIGTERM", "SIGINT", "exit"])
process.on(signal, () => {
return stopper.stop({
port: 9876,
logLevel: "OFF"
})
})
}
}

View File

@@ -1,38 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import clean from "del"
import vinyl from "vinyl-paths"
/* ----------------------------------------------------------------------------
* Task: clean files generated by visual tests
* ------------------------------------------------------------------------- */
export default (gulp, config) => {
return () => {
return gulp.src([
`${config.tests.visual}/data`,
"./gemini-report"
])
.pipe(vinyl(clean))
}
}

View File

@@ -1,63 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import child from "child_process"
import path from "path"
import through from "through2"
import util from "gulp-util"
/* ----------------------------------------------------------------------------
* Task: generate visual tests
* ------------------------------------------------------------------------- */
export default (gulp, config) => {
const theme = path.resolve(process.cwd(), config.views.build)
return () => {
return gulp.src(`${config.tests.visual}/suites/**/mkdocs.yml`)
.pipe(
through.obj(function(file, enc, done) {
if (file.isNull() || file.isStream())
return done()
/* Resolve test name and destination */
const name = path.relative(`${config.tests.visual}/suites`,
path.dirname(file.path))
const site = path.resolve(process.cwd(),
`${config.tests.visual}/data`, name, "_")
/* Generate test fixtures with freshly built theme */
const proc = child.spawnSync("mkdocs", [
"build", "--site-dir", site, "--theme-dir", theme
], {
cwd: path.dirname(file.path)
})
/* Emit error, if any */
if (proc.status)
this.emit("error", new util.PluginError("mkdocs",
`Terminated with errors: ${proc.stderr.toString()}`))
/* Terminate */
done()
}))
}
}

View File

@@ -1,166 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import moniker from "moniker"
import path from "path"
import * as ecstatic from "~/lib/servers/ecstatic"
import * as sauce from "~/lib/servers/sauce-connect"
import * as selenium from "~/lib/servers/selenium"
import Gemini from "gemini"
import SauceLabs from "saucelabs"
/* ----------------------------------------------------------------------------
* Locals
* ------------------------------------------------------------------------- */
/* SauceLabs job name */
const id = process.env.TRAVIS
? `Travis #${process.env.TRAVIS_BUILD_NUMBER}`
: `Local #${moniker.choose()}`
/* SauceLabs test results */
const passed = {}
/* ----------------------------------------------------------------------------
* Task: run visual tests
* ------------------------------------------------------------------------- */
export default (gulp, config, args) => {
return done => {
/* Start static file server */
let error = false
new Promise(resolve => {
ecstatic.start(`${config.tests.visual}/data`, 8000, resolve)
/* Create and start test runner */
}).then(() => {
return new Promise((resolve, reject) => {
/* Start SauceConnect tunnel */
if (process.env.CI || process.env.SAUCE) {
if (!process.env.SAUCE_USERNAME ||
!process.env.SAUCE_ACCESS_KEY)
throw new Error(
"SauceConnect: please provide SAUCE_USERNAME " +
"and SAUCE_ACCESS_KEY")
/* Start tunnel, if credentials are given */
sauce.start(
id,
process.env.SAUCE_USERNAME,
process.env.SAUCE_ACCESS_KEY,
err => {
return err ? reject(err) : resolve(sauce)
})
/* Start Selenium */
} else {
selenium.start(() => resolve(selenium))
}
})
/* Setup and run Gemini */
.then(runner => {
const setup = require(
path.join(process.cwd(), `${config.tests.visual}/config`,
process.env.CI || process.env.SAUCE
? "gemini.sauce-connect.json"
: "gemini.selenium.json"))
/* Add dynamic configuration to capabilities */
for (const key of Object.keys(setup.browsers)) {
const caps = setup.browsers[key].desiredCapabilities
caps.tunnelIdentifier = id
caps.public = "private"
caps.name = id
/* Adjust configuration for Travis CI */
if (process.env.CI && process.env.TRAVIS)
caps.public = "public"
}
/* Setup Gemini and test listeners */
const gemini = new Gemini(setup)
if (process.env.CI || process.env.SAUCE) {
/* Initialize test run */
gemini.on(gemini.events.START_BROWSER, job => {
passed[job.sessionId] = true
})
/* Update state of test run */
gemini.on(gemini.events.TEST_RESULT, job => {
passed[job.sessionId] = passed[job.sessionId] && job.equal
})
}
/* Run tests */
return gemini.test(`${config.tests.visual}/suites`, {
reporters: ["flat", "html"],
browsers: args.browser ? [].concat(args.browser) : null
})
/* Return runner for graceful stop */
.then(status => {
error = status.failed + status.errored > 0
return runner
})
})
/* Stop test runner */
.then(runner => {
return new Promise(resolve => {
runner.stop(resolve)
})
})
/* Update SauceLabs jobs with test results */
.then(() => {
const saucelabs = new SauceLabs({
username: process.env.SAUCE_USERNAME,
password: process.env.SAUCE_ACCESS_KEY
})
const updates = Object.keys(passed).map(sessionId => {
return new Promise(resolve => {
saucelabs.updateJob(sessionId, {
passed: passed[sessionId]
}, resolve)
})
})
return Promise.all(updates)
})
/* Stop static file server */
})
.then(() => {
ecstatic.stop(() => {
return error
? done(new Error("Gemini terminated with errors"))
: done()
})
}, err => {
return done(new Error(err))
})
}
}

View File

@@ -1,82 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import moniker from "moniker"
import * as ecstatic from "~/lib/servers/ecstatic"
import * as sauce from "~/lib/servers/sauce-connect"
/* ----------------------------------------------------------------------------
* Task: run visual tests
* ------------------------------------------------------------------------- */
export default (gulp, config) => {
return done => {
/* Start static file server */
new Promise(resolve => {
ecstatic.start(`${config.tests.visual}/data`, 8000, resolve)
/* Open SauceConnect tunnel */
}).then(() => {
return new Promise((resolve, reject) => {
/* Start SauceConnect tunnel */
if (process.env.CI || process.env.SAUCE) {
if (!process.env.SAUCE_USERNAME ||
!process.env.SAUCE_ACCESS_KEY)
throw new Error(
"SauceConnect: please provide SAUCE_USERNAME " +
"and SAUCE_ACCESS_KEY")
/* Open tunnel */
sauce.start(
`Local #${moniker.choose()}`,
process.env.SAUCE_USERNAME,
process.env.SAUCE_ACCESS_KEY,
err => {
return err ? reject(err) : resolve(sauce)
})
} else {
resolve()
}
})
/* Close tunnel on CTRL-C */
.then(runner => {
return new Promise(resolve => {
process.on("SIGINT", () => {
return runner
? runner.stop(resolve)
: resolve()
})
})
})
/* Stop static file server */
})
.then(() => {
ecstatic.stop(done)
}, err => {
return done(err)
})
}
}

View File

@@ -1,72 +0,0 @@
/*
* Copyright (c) 2016-2017 Martin Donath <martin.donath@squidfunk.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
import fs from "fs"
import path from "path"
import through from "through2"
/* ----------------------------------------------------------------------------
* Task: update reference images for visual tests
* ------------------------------------------------------------------------- */
export default (gulp, config) => {
return () => {
const base = path.join(
process.cwd(), `${config.tests.visual}/config`)
/* Read Gemini configs and map browsers to screenshot directories */
const mapping = fs.readdirSync(base)
.reduce((result, filename) => {
return Object.assign(result, (gemini => {
return Object.keys(gemini.browsers)
.reduce((browsers, name) => {
browsers[name] = gemini.screenshotsDir
return browsers
}, {})
})(require(path.join(base, filename))))
}, {})
/* Prepare filenames */
const dest = path.join(process.cwd(), `${config.tests.visual}/baseline`)
return gulp.src("gemini-report/images/**/*~current.png")
.pipe(
through.obj(function(file, enc, done) {
if (file.isNull() || file.isStream())
return done()
/* Remove the state from the filename */
file.path = file.path.replace("~current", "")
/* Retrieve the folder for the environment of the baseline */
const folder = path.relative(dest,
mapping[path.basename(file.path, ".png")])
file.path = file.path.replace("images", `images/${folder}`)
/* Push file to next stage */
this.push(file)
done()
}))
/* Update reference images */
.pipe(gulp.dest(dest))
}
}