scenarist
Class-based branching test scenario runner for mocha
// common-test.js global test classes async { await super; thiscontainer = document; } async { let self = this; await super; await self; }
// example-test.js // example scope let scope = 'example'; let example = scope 'Description of Example Suite'; // test class mixin in "example" scope example { return 'Description of Test A'; } async { console; thiselement = document } async { console; assert; //assert.isOk(false, 'Failing test A'); } ... // test class in "example" scope exampletest = static { return true; } async {} async {} ... // scenarios exampletest = // test class mixins '': TestA: TestB: 'TestAThenB' TestB: TestA: 'TestBThenA' Suite // test classes TestC: TestAThenB: 'TestCAB' TestD: 'TestDAlias' TestE: TestAThenB: 'TestEAB' TestA: TestB: Test1: Test2: 'TestEAB12' TestBThenA: 'TestEABA' TestB: Test1: '' TestAB3: 'TestEAB3; Description of "Test EAB3"' Suite ; let match = ; if match // Runner // match[1] = '0' for the first round of test suites runnable without reloading example; // example scope
// In Driver page// example for web-component-testervar suites = ;for var scope in Suitescopes Suitescopesscopetest;WCT;
Design Principles
- Contexts must be explicitly handled in a concise and intuitive way in JavaScript classes
suite()
andtest()
in mocha are wrapped for contexts
Alternative viewpoints for test scenarios with long and branching operations
- Test target systems are a collection of state machines
- Operations for test suites are a series of the branches of states
- Test assertions are targeted "checkpoints" for the expected states
Depicted test scenarios
Initial -(setup op)-> First checkpoint -(op)-> 2nd CP -> ... -> Final CP for scenario A
+--> ... -> Final CP for scenario B
...
- Initial state is without instances of the target system
- Operations from the initial state set up test fixtures
- The first checkpoint on the first operation asserts the target instances
- Setting up the fixtures may take more steps (operations)
- Successive checkpoints constitute a series of pairs of a (mock) operation and its corresponding test assertion(s)
- Different test suites can share parts of operations and then branch in the latter parts
Conceptual mappings for JavaScript classes
- Operation: a (mock) operation
- Checkpoint: a collection of test assertions for a checkpoint
- Scenario: a series of Operation and Checkpoint pairs (by a prototype chain of classes)
- Suite: the base class of test scenarios
- Driver: driver of test suites
- Parameters: test parameters handed to constructors of test classes
Comparison with BDD framework
- "Branching" of test contexts: Shared steps and test assertions for scenarios
Install
Browsers
bower install --save-dev scenarist
NodeJS
npm install --save-dev scenarist
Import
Browsers
Raw ES module class version
;
UMD ES6 class version
ES5 version
babel-polyfill/browser.js
is required for the ES5 version to work
Note:
NodeJS
Node 8.x or later
Command Line
mocha test.js
Test Script
//require('babel-polyfill');const chai = ;const assert = chaiassert;const Suite = ;// test classes...
Node 4.x - 6.x with Babel es2015 Transpilation
Command Line
mocha test.js
Test Script
;const chai = ;const assert = chaiassert;const Suite = ;// test classes...
Node with Multiple Scopes
Command Line
mocha test.js
Driver Test Script (test.js)
;const chai = ;globalassert = chaiassert;globalSuite = ; ;; for var scope in Suitescopes Suitescopesscopetest;
Test Script 1 (scope1-test.js)
let scope1 = 'scope1' 'Scope 1 Suites';// test classesscope1test = ...
Test Script 2 (scope2-test.js)
let scope2 = 'scope2' 'Scope 2 Suites';// test classesscope2test = ...
Compatibility
Version | Chrome | Firefox | IE/Edge | Safari | Opera | Node | ECMAScript |
---|---|---|---|---|---|---|---|
Suite.esm.js | 61+ | 60 | 17+ | 10.1+ | 47+ | 10+ w/ esm | ES6 + ES modules |
Suite.js | 55+ | 52+ | 17+ | 10.1+ | 42+ | 7+ w/ async | ES6 + async/await |
Suite.min.js | 55+ | 50+ | 11+ | 10+ | 42+ | 4+ w/ babel | ES5 + babel-polyfill |
- Suite.mjs and Suite.esm.js have the same contents but have different file name extensions.
API
Suite
class
- Index
- Scope Object as a
Suite
class instance - Scope Object Instance Properties other than
test
property setter - Test Suite Subclass as a direct or descendent subclass of
Suite
class - Test Runner Subclass as a subclass of
Suite
class - Utility Instance Methods
- Utility Static Methods
- Static Properties for
Suite
class - Static Properties for subclasses
- Scope Object as a
Suite
class instance
Scope Object as a - constructor(scope: string, description: string = scope + ' suite') - Create a scope typically in a block scope; The scope object is globally accessible via
Suite.scopes[scope]
// "scope1" block scope let scope1 = 'scope1' 'Description of scope1 Suite'; scope1 === Suitescopesscope1; // is true
- test property setter with a class expression - Define a test runner subclass or a test suite subclass in the scope
let scope = 'example'; let example = scope 'Description of Example Suite'; // Defined test suite class is accessible via example.classes.ExampleSuite exampletest = async { await super; ... } async { await super; ... } // Defined test runner class is accessible via example.classes.ExampleTest1 exampletest = classesExampleSuite async { ... } async { ... }
- test property setter with a class expression mixin - Define a test runner class mixin in the scope
// Defined test runner class mixin is accessible via example.mixins.ExampleTestMixin1 example async { ... } async { ... }
- test property setter with a test scenario object - Define test scenarios with a scenario object in the scope
- Arrays in scenarios iterate over their items
- Description can be optionally specified after ';' of the test class name
- Default description string is generated by uncamelcasing the test class name, e.g., 'TestAAndB' -> 'test a and b'
exampletest = // test class mixins '': ExampleTestMixin1: ExampleTestMixin2: 'Test1Then2' // Define Test1Then2 test class mixin // test classes ExampleTest1: Test1Then2: 'Test1ThenTest1Then2' // Define Test1ThenTest1Then2 test class
exampletest = // test class mixins '': Suite Suite // test classes ExampleTest1: Test1Then2: 'Test1ThenTest1Then2; Description of the Test'
- test property setter with a class from another scope - Define a subscope of a scope
// Meta-depiction of Subscoping as Prototype chaining
Suite <-- GlobalScope <-- ScopeA <-- SubscopeAB ...
+- SubscopeAC ...
<!-- subscope_ac-test.html -->
// global_scope.js ... ...
// scope_a.js let scope_a = 'scope_a'; scope_atest = ... scope_a ... scope_atest = ScopeATest1: ScopeATest2: 'ExportedScopeATest' // The exported test name is not necessarily explicit // no call of run() in scope_a.js since it will be called in scope_a-test.html
// subscope_ac.js let subscope_ac = 'subscope_ac'; // Define a subscope 'subscope_ac' of the scope 'scope_a' via 'ExportedScopeATest' subscope_actest = Suitescopesscope_aclassesExportedScopeATest; // Tests for subscope_ac subscope_ac ... subscope_ac ... subscope_actest = ExportedScopeATest: // Extend scope_a test SubscopeACTest1: 'Instantiate_ScopeATest1_ScopeATest2_SubscopeACTest1_Test' SubscopeACTest2: 'Instantiate_ScopeATest1_ScopeATest2_SubscopeACTest2_Test' ... // no call of run() in subscope_ac.js since it will be called in subscope_ac-test.html
- testClasses(tests) instance method - Get Array of test classes
- tests as number - List classes in CSV
this.test[tests]
- tests as CSV string - List classes in the CSV
- tests as number - List classes in CSV
- async run(classes, target: string) instance method - Run the specified test classes in the scope;
target
is handed to constructors of target classes- classes as number - Target classes are
testClasses(test[classes])
- classes as CSV string - Target classes are
testClasses(classes)
- classes as Array of string - Target classes are
classes.map((item) => self.classes[item])
- classes as Array of classes - Target classes are
classes
- classes as object with class properties - Target classes are properties of
classes
- it can throw MultiError exception, which has Array property
.errors
as[ [ 'TestClassName', TestClassInstance, Exception ], ... ]
- classes as number - Target classes are
{ for let scope in Suitescopes for let index in Suitescopesscopetest try await Suitescopesscope; catch errors errorsmessage === 'Error: Suite.error5.run(Test1,Test2,...): exception(s) thrown. See .errors for details'; errorserrors; }
for var scope in Suitescopes Suitescopesscopetest;
test
property setter
Scope Object Instance Properties other than - scope string property - Scope name set by
new Suite(scope)
- description string property - Default: scope + ' suite'; Scope description set by
new Suite(scope, description)
- classes object property - Object containing the currently defined test classes
- mixins object property - Object containing the currently defined test class mixins
- classSyntaxSupport boolean property -
true
if ES6 class syntax is natively supported - arrowFunctionSupport boolean property -
true
if ES6 arrow function syntax is natively supported - leafClasses object property - Object containing the current leaf (non-redundant) test classes
- leafScenarios object property - Object with CSV strings of leaf test scenarios as its properties
- branchScenarios object property - Object with CSV strings of branch test scenarios as its properties
- test Array property - Array of CSV strings, each of which constitutes a group of reconnectable tests
Suitescopesexampletest returns 'ReconnectableTest1,ReconnectableTest2' 'NonReconnectableTest1' 'ReconnectableTest3,ReconnectableTest4' ... // Test page has to be reloaded for each reconnectable test group
// In Driver page // example for web-component-tester var suites = ; for var scope in Suitescopes Suitescopesscopetest; WCT;
Suite
class
Test Suite Subclass as a direct or descendent subclass of - async setup() instance method - Overridden methods called once at the beginning of each running test via
suiteSetup()
; Overridden methods in subclasses should callawait super.setup()
- async teardown() instance method - Overridden methods called once at the end of each running test via
suiteTeardown()
; Overridden methods in subclasses should callawait super.teardown()
// Define a common test suite as a direct subclass of Suite async { await super; ... } async { await super; ... } ... // custom methods // example scope let example = 'example' 'Example Subsuite'; // [Optional] Define a test subsuite class in the 'example' scope exampletest = async { await super; ... } async { ...; await super; } // may need teardown operation before super.teardown() ... // more custom methods for the subsuite
Suite
class
Test Runner Subclass as a subclass of - constructor(target: string) - Instantiate a test runner; Optional for overriding
- async run() instance method - Run the test; Not for overriding
- It can throw an exception. See
exception()
instance method section below.
- It can throw an exception. See
- * scenario() instance generator method - Iterate over tests in the reversed order of the prototype chain of the test; Yield
{ name: string, iteration: function, operation: function, checkpoint: function, ctor: function }
for each test class
// Simplest example without a test suite subclass and a scope {} // Define a test runner class let testRunner = '#target'; // testRunner.target = '#target' testRunner; // run the test
// Example with a scope ... // example scope let example = 'example'; // Define a test class in the scope exampletest = async { ... } async { ... } // run the test '#target';
- async operation(parameters: optional) instance method - Perform operations of the test;
parameters
argument is omitted if*iteration()
is not defined - async checkpoint(parameters: optional) instance method - Perform test assertions of the test;
parameters
argument is omitted if*iteration()
is not defined - *iteration() instance generator method - [Optional] Provide parameters to
operation
andcheckpoint
methods
* { 1 2 3 ; } // parameters iterate through 1 to 3 async { ... } async { ... }
- exception(reject: function, exception: Error) instance method - [Optional] Exception handler for errors outside of test callback function
- If it calls
reject()
, it must return non-null to tell the runner not to callresolve()
- The method can be inherited and overridden by subclasses
- If it calls
async { ... } async { ... } { // default action when exception() is not defined ; return true; } try await '#example'; catch exception // Handle exception in runner ...
async { ... } async { ... } { // Treat the exception as a test failure by mocha typeof test === 'function' ? test : it'exception on scenario' { throw exception; }; }
Utility Instance Methods
- async forEvent(element: Element, type: string, trigger: function, condition: function) - Invoke the
trigger()
and wait for the eventtype
for theelement
untilcondition(element: Element, type: string, event: Event)
returnstrue
async { let self = this; // wait for 'track' event until the event state becomes 'end' await self; }
Utility Static Methods
- Suite.repeat(target: string, count: number, subclass: string/object) - Repetition operator for scenario objects
Suite returns TargetTest: TargetTest: TargetTest: 'RepeatTarget3Times'
- Suite.permute(targets: Array of string, subclass: function (scenario: Array of string)) - Permutation operator for scenario objects
Suite returns TestA: TestB: TestC: "Test_A_B_C" TestC: TestB: "Test_A_C_B" TestB: TestA: TestC: "Test_B_A_C" TestC: TestA: "Test_B_C_A" TestC: TestB: TestA: "Test_C_B_A" TestA: TestB: "Test_C_A_B"
Suite
class
Static Properties for - Suite.scopes static object property - Object containing scope objects as
Suite.scopes[scope]
such asSuite.scopes.example
for'example'
scope
Static Properties for subclasses
- reconnectable static boolean read-only property - Default:
true
; Override the value asfalse
to treat the test and its subclasses are not reconnectable and need page reloading to perform further tests
{ return false; } ...
- skipAfterFailure static boolean read-only property - Default:
false
; Override the value astrue
to skip subsequent tests of the suite after an assertion failure
{ return true; } ...
Complex Examples
dialog-test.js for live-localizer test suites
Test Class Mixin with Parameterized Iterations in let example = 'example'; example * { let dx = 10; let dy = 10; mode: 'position' dx: dx dy: dy expected: x: dx y: dy width: 0 height: 0 mode: 'upper-left' dx: -dx dy: -dy expected: x: -dx y: -dy width: dx height: dy mode: 'upper' dx: -dx dy: -dy expected: x: 0 y: -dy width: 0 height: dy mode: 'upper-right' dx: dx dy: -dy expected: x: 0 y: -dy width: dx height: dy mode: 'middle-left' dx: -dx dy: dy expected: x: -dx y: 0 width: dx height: 0 mode: 'middle-right' dx: dx dy: dy expected: x: 0 y: 0 width: dx height: 0 mode: 'lower-left' dx: -dx dy: dy expected: x: -dx y: 0 width: dx height: dy mode: 'lower' dx: dx dy: dy expected: x: 0 y: 0 width: 0 height: dy mode: 'lower-right' dx: dx dy: dy expected: x: 0 y: 0 width: dx height: dy mode: '.title-pad' dx: dx dy: dy expected: x: 0 y: 0 width: 0 height: 0 ; } async { let self = this; let handle = selfdialog$handle; selforigin = {}; 'x' 'y' 'width' 'height' ; handle; await self; } async { for let prop in parametersexpected assert; }
demo.js
Test Class Mixin Generator for Common Operations and Checkpoints inlet demo = scope 'Scenarist Demo Suite';const labels = // op: [ 'Class', 'id' ] '0': 'Number0' '0' '1': 'Number1' '1' ... '=': 'Equal' '=' 'A': 'Ac' 'AC' 'B': 'Bs' 'BS' ;demoexpected = "AC": "0" "AC1": "1" "AC12": "12" "AC12+": "12" "AC12+3": "3" "AC12+34": "34" "AC12+34=": "46" ...;for let ex in labels demotest = 'demo' { // generate ES5 class by manipulating transpiled func.toString() return 'return ' + { return '__LABEL__'; } async { await this; thishistory = this; } async { assert; } // trim istanbul coverage counters ; }labelsex0 labelsex1demo;
demo.js
Custom Test Scenario Object Operator in { let result = null; if !name let description = ; name = ; for let j of expression name; description; name = name; description = description; description += description ? ' ' : ' = ' + demoexpected'AC' + description; name += '; ' + description; for let i = expressionlength - 1; i >= 0; i-- let op = expressioni; let mixin = labelsop0; if !mixin throw 'Invalid operation ' + op + ' in "' + expression + '"'; if result result = mixin: result ; else result = mixin: name ; return result;}demotest = // test class mixins '': // test classes Connect: Ac: N12: '+' '-' '*' '/' ;