Fix work with promise chains (#568)

* Run after block after test fails

* fix interface test

* fix formatting

* refactor work with chains

* fix typo

* remove second after

* correct chain order

* fix formatting

* Add compliting _after _afterSuite if after hooks failed

* fix formatting

* Add new hook finishTest

* fix formatting

* return recorder back

* fix formatting

* add closing tabs for Protractor and selenium

* fix formatting

* fix uppercase

* fix hooks

* clear localstorage and close tabs with keepCookies flag

* fix formatting

* Fixes

* remove defenition

* add step execution time in Verbose mode

* fix formatting

* Add step execution time to output

* fix formatting

* fixes after review

* fix formatting

* fix typo

* fix tests with generator

* fix formatting

* add mochaFile override

* add report file name in override config for junit reporter

Author: APshenkin

lib/actor.js

@@ -61,10 +61,17 @@ function recordStep(step, args) {
     val = step.run.apply(step, args);
   });
 
+  // ensure step result is passed to next step
+  recorder.add('return step result', () => val);
+
+  //return execution time for logging
+  recorder.add(`return execution time`, () => {
+    output.stepExecutionTime(step, global.suiteDetails);
+    return val;
+  });
+
   // run async after step hooks
   event.emit(event.step.after, step);
 
-  // ensure step result is passed to next step
-  recorder.add('return step result', () => val);
   return recorder.promise();
 }

lib/command/run-multiple.js

@@ -93,6 +93,7 @@ function runSuite(suite, suiteConf, browser) {
   // tweaking default output directories and for mochawesome
   overriddenConfig = replaceValue(overriddenConfig, 'output', path.join(config.output, outputDir));
   overriddenConfig = replaceValue(overriddenConfig, 'reportDir', path.join(config.output, outputDir));
+  overriddenConfig = replaceValue(overriddenConfig, 'mochaFile', path.join(config.output, outputDir, 'report.xml'));
 
   // override grep param and collect all params
   let params = ['run',

lib/event.js

@@ -11,6 +11,14 @@ module.exports = {
     passed: 'test.passed',
     failed: 'test.failed',
   },
+  hook: {
+    afterSuite: {
+      failed: 'afterSuiteHook.failed',
+    },
+    after: {
+      failed: 'afterTestHook.failed'
+    }
+  },
   suite: {
     before: 'suite.before',
     after: 'suite.after'

lib/helper.js

@@ -105,6 +105,15 @@ class Helper {
 
   }
 
+  /**
+   * Hook executed after all tests are executed
+   *
+   * @param suite
+   */
+  _finishTest() {
+
+  }
+
   /**
    * Access another configured helper: `this.helpers['AnotherHelper']`
    */

lib/helper/Appium.js

@@ -133,16 +133,6 @@ class Appium extends WebdriverIO {
     return this.browser;
   }
 
-  _after() {
-    if (this.options.restart) return this.browser.end();
-    if (this.options.desiredCapabilities.browserName) {
-      this.debugSection('Session', 'cleaning cookies and localStorage');
-      return this.browser.execute('localStorage.clear();').then(() => {
-        return this.browser.deleteCookie();
-      });
-    }
-  }
-
   /**
    * Check if an app is installed.
    *

lib/helper/Nightmare.js

@@ -55,6 +55,8 @@ class Nightmare extends Helper {
       js_errors: null
     };
 
+    this.isRunning = false;
+
     // override defaults with config
     Object.assign(this.options, config);
 
@@ -172,7 +174,8 @@ class Nightmare extends Helper {
   }
 
   _beforeSuite() {
-    if (!this.options.restart) {
+    if (!this.options.restart && !this.isRunning) {
+      this.isRunning = true;
       return this._startBrowser();
     }
   }
@@ -189,10 +192,13 @@ class Nightmare extends Helper {
       return this._stopBrowser();
     }
     if (this.options.keepCookies) return;
-    return this.browser.cookies.clearAll();
+    return Promise.all([this.browser.cookies.clearAll(), this.executeScript('localStorage.clear();')]);
   }
 
   _afterSuite() {
+  }
+
+  _finishTest() {
     if (!this.options.restart) {
       this._stopBrowser();
     }
@@ -822,9 +828,9 @@ class Nightmare extends Helper {
     }, lctype(locator), lcval(locator));
   }
 
-    /**
-     * {{> ../webapi/saveScreenshot }}
-     */
+  /**
+   * {{> ../webapi/saveScreenshot }}
+   */
   saveScreenshot(fileName, fullPage = this.options.fullPageScreenshots) {
     let outputFile = path.join(global.output_dir, fileName);
     this.debug('Screenshot is saving to ' + outputFile);
@@ -837,9 +843,9 @@ class Nightmare extends Helper {
       height: document.body.scrollHeight,
       width: document.body.scrollWidth
     })).then(({
-        width,
-        height
-      }) => {
+                width,
+                height
+              }) => {
       this.browser.viewport(width, height);
       return this.browser.screenshot(outputFile);
     });
@@ -907,7 +913,7 @@ function *proceedSeeInField(assertType, field, value) {
   let fieldVal = yield this.browser.evaluate(function (el) {
     return codeceptjs.fetchElement(el).value;
   }
-  , el);
+    , el);
   if (tag == 'select') {
     // locate option by values and check them
     let text = yield this.browser.evaluate(function (el, val) {

lib/helper/SeleniumWebdriver.js

@@ -78,6 +78,9 @@ class SeleniumWebdriver extends Helper {
       manualStart: false,
       capabilities: {}
     };
+
+    this.isRunning = false;
+
     if (this.options.waitforTimeout) {
       console.log(`waitforTimeout is deprecated in favor of waitForTimeout, please update config`);
       this.options.waitForTimeout = this.options.waitforTimeout;
@@ -132,8 +135,9 @@ class SeleniumWebdriver extends Helper {
   }
 
   _beforeSuite() {
-    if (!this.options.restart && !this.options.manualStart) {
+    if (!this.options.restart && !this.options.manualStart && !this.isRunning) {
       this.debugSection('Session', 'Starting singleton browser session');
+      this.isRunning = true;
       return this._startBrowser();
     }
   }
@@ -146,13 +150,16 @@ class SeleniumWebdriver extends Helper {
 
   _after() {
     if (this.options.restart) return this.browser.quit();
-    if (this.options.keepCookies) return;
+    if (this.options.keepCookies) return Promise.all([this.browser.executeScript('localStorage.clear();'), this.closeOtherTabs()]);
     // if browser should not be restarted
     this.debugSection('Session', 'cleaning cookies and localStorage');
-    return Promise.all([this.browser.manage().deleteAllCookies(), this.browser.executeScript('localStorage.clear();')]);
+    return Promise.all([this.browser.manage().deleteAllCookies(), this.browser.executeScript('localStorage.clear();'), this.closeOtherTabs()]);
   }
 
   _afterSuite() {
+  }
+
+  _finishTest() {
     if (!this.options.restart) return this.browser.quit();
   }
 
@@ -667,6 +674,27 @@ class SeleniumWebdriver extends Helper {
     return this.browser.manage().window().setSize(width, height);
   }
 
+  /**
+   * Close all tabs expect for one.
+   *
+   * ```js
+   * I.closeOtherTabs();
+   * ```
+   */
+  closeOtherTabs() {
+    let client = this.browser;
+
+    return client.getAllWindowHandles().then(function (handles){
+      for (var i = 1; i < handles.length; i++) {
+        client.switchTo().window(handles[i]).then(function (){
+          client.close();
+        });
+      }
+      return client.switchTo().window(handles[0]);
+    });
+  }
+
+
   /**
    * {{> ../webapi/wait }}
    */

lib/helper/WebDriverIO.js

@@ -210,6 +210,8 @@ class WebDriverIO extends Helper {
       }
     };
 
+    this.isRunning = false;
+
     // override defaults with config
     Object.assign(this.options, config);
 
@@ -256,8 +258,9 @@ class WebDriverIO extends Helper {
   }
 
   _beforeSuite() {
-    if (!this.options.restart && !this.options.manualStart) {
+    if (!this.options.restart && !this.options.manualStart && !this.isRunning) {
       this.debugSection('Session', 'Starting singleton browser session');
+      this.isRunning = true;
       return this._startBrowser();
     }
   }
@@ -295,12 +298,17 @@ class WebDriverIO extends Helper {
 
   _after() {
     if (this.options.restart) return this.browser.end();
-    if (this.options.keepCookies) return;
-    this.debugSection('Session', 'cleaning cookies and localStorage');
-    return Promise.all([this.browser.deleteCookie(), this.browser.execute('localStorage.clear();')]);
+    if (this.options.keepCookies) return Promise.all([this.browser.execute('localStorage.clear();'), this.closeOtherTabs()]);
+    if (this.options.desiredCapabilities.browserName) {
+      this.debugSection('Session', 'cleaning cookies and localStorage');
+      return Promise.all([this.browser.deleteCookie(), this.browser.execute('localStorage.clear();'), this.closeOtherTabs()]);
+    }
   }
 
   _afterSuite() {
+  }
+
+  _finishTest() {
     if (!this.options.restart) return this.browser.end();
   }
 
@@ -1088,6 +1096,24 @@ class WebDriverIO extends Helper {
     );
   }
 
+  /**
+   * Close all tabs expect for one.
+   * Appium: support web test
+   *
+   * ```js
+   * I.closeOtherTabs();
+   * ```
+   */
+  closeOtherTabs() {
+    let client = this.browser;
+    return client.getTabIds().then(function (handles) {
+      for (var i = 1; i < handles.length; i++) {
+        this.close(handles[i]);
+      }
+      return this.switchTab();
+    });
+  }
+
   /**
    * {{> ../webapi/wait }}
    * Appium: support

lib/interfaces/bdd.js

@@ -24,6 +24,7 @@ var escapeRe = require('escape-string-regexp');
 module.exports = function (suite) {
   var suites = [suite];
   suite.timeout(0);
+  var afterAllHooks, afterEachHooks, afterAllHooksAreLoaded, afterEachHooksAreLoaded;
 
   suite.on('pre-require', function (context, file, mocha) {
     var common = require('mocha/lib/interfaces/common')(suites, context);
@@ -45,6 +46,12 @@ module.exports = function (suite) {
         suites.shift();
       }
       if (!opts) opts = {};
+
+      afterAllHooks = [];
+      afterEachHooks = [];
+      afterAllHooksAreLoaded = false;
+      afterEachHooksAreLoaded = false;
+
       var suite = Suite.create(suites[0], title);
       suite.timeout(0);
 
@@ -54,28 +61,28 @@ module.exports = function (suite) {
       suite.file = file;
       suites.unshift(suite);
       suite.beforeEach('codeceptjs.before', scenario.setup);
-      suite.afterEach('finialize codeceptjs', scenario.teardown);
+      afterEachHooks.push(['finialize codeceptjs', scenario.teardown]);
 
       suite.beforeAll('codeceptjs.beforeSuite', () => scenario.suiteSetup(suite));
-      suite.afterAll('codeceptjs.afterSuite', () => scenario.suiteTeardown(suite));
+      afterAllHooks.push(['codeceptjs.afterSuite', () => scenario.suiteTeardown(suite)]);
 
       return suite;
     };
 
     context.BeforeSuite = function (fn) {
-      suites[0].beforeAll('BeforeSuite', scenario.injected(fn, suites[0]));
+      suites[0].beforeAll('BeforeSuite', scenario.injected(fn, suites[0], 'beforeSuite'));
     };
 
     context.AfterSuite = function (fn) {
-      suites[0].afterAll('AfterSuite', scenario.injected(fn, suites[0]));
+      afterAllHooks.unshift(['AfterSuite', scenario.injected(fn, suites[0], 'afterSuite')]);
     };
 
     context.Background = context.Before = function (fn) {
-      suites[0].beforeEach('Before', scenario.injected(fn, suites[0]));
+      suites[0].beforeEach('Before', scenario.injected(fn, suites[0], 'before'));
     };
 
     context.After = function (fn) {
-      suites[0].afterEach('After', scenario.injected(fn, suites[0]));
+      afterEachHooks.unshift(['After', scenario.injected(fn, suites[0]), 'after']);
     };
 
     /**
@@ -88,7 +95,22 @@ module.exports = function (suite) {
         fn = opts;
         opts = {};
       }
-
+      /**
+       * load hooks from arrays to suite to prevent reordering
+       *
+       */
+      if (!afterAllHooksAreLoaded) {
+        afterAllHooks.forEach(function (hook) {
+          suites[0].afterAll(hook[0], hook[1]);
+        });
+        afterAllHooksAreLoaded = true;
+      }
+      if (!afterEachHooksAreLoaded) {
+        afterEachHooks.forEach(function (hook) {
+          suites[0].afterEach(hook[0], hook[1]);
+        });
+        afterEachHooksAreLoaded = true;
+      }
       var suite = suites[0];
       if (suite.pending) {
         fn = null;

lib/listener/helpers.js

@@ -36,6 +36,14 @@ module.exports = function () {
     runAsyncHelpersHook('_afterSuite', suite, true);
   });
 
+  event.dispatcher.on(event.hook.after.failed, function (suite) {
+    runAsyncHelpersHook('_after', suite, true);
+  });
+
+  event.dispatcher.on(event.hook.afterSuite.failed, function (suite) {
+    runAsyncHelpersHook('_afterSuite', suite, true);
+  });
+
   event.dispatcher.on(event.test.started, function (test) {
     runHelpersHook('_test', test);
     recorder.catch((e) => error(e));
@@ -48,7 +56,7 @@ module.exports = function () {
   event.dispatcher.on(event.test.failed, function (test) {
     runAsyncHelpersHook('_failed', test, true);
     // should not fail test execution, so errors should be catched
-    recorder.catch((e) => error(e));
+    recorder.catchWithoutStop((e) => error(e));
   });
 
   event.dispatcher.on(event.test.after, function () {
@@ -62,4 +70,8 @@ module.exports = function () {
   event.dispatcher.on(event.step.after, function (step) {
     runAsyncHelpersHook('_afterStep', step);
   });
+
+  event.dispatcher.on(event.all.result, function () {
+    runAsyncHelpersHook('_finishTest', {}, true);
+  });
 };