1 |
|
2 | common = require './common'
|
3 | protocol = require './protocol'
|
4 | testsuite = require './testsuite'
|
5 | expectation = require './expectation'
|
6 |
|
7 | fbp = require 'fbp'
|
8 | fbpClient = require 'fbp-protocol-client'
|
9 | debug = require('debug')('fbp-spec:runner')
|
10 | Promise = require 'bluebird'
|
11 |
|
12 | debugReceivedMessages = (client) ->
|
13 | debugReceived = require('debug')('fbp-spec:runner:received')
|
14 | client.on 'graph', ({command, payload}) ->
|
15 | debugReceived 'graph', command, payload
|
16 | client.on 'network', ({command, payload}) ->
|
17 | debugReceived 'network', command, payload
|
18 | client.on 'runtime', ({command, payload}) ->
|
19 | debugReceived 'runtime', command, payload
|
20 | client.on 'component', ({command, payload}) ->
|
21 | debugReceived 'component', command, payload
|
22 | client.on 'execution', (status) ->
|
23 | debugReceived 'execution', status
|
24 |
|
25 | synthesizeTopicFixture = (topic, components, callback) ->
|
26 | debug 'synthesizing fixture', topic
|
27 |
|
28 |
|
29 | component = components[topic]
|
30 | return callback new Error "Could not find component for topic: #{topic}" if not component
|
31 |
|
32 | graph =
|
33 | properties: {}
|
34 | inports: {}
|
35 | outports: {}
|
36 | processes: {}
|
37 | connections: []
|
38 |
|
39 | processName = 'testee'
|
40 | graph.processes[processName] =
|
41 | component: topic
|
42 |
|
43 | for port in component.inPorts
|
44 | portName = port.id
|
45 | graph.inports[portName] =
|
46 | process: processName
|
47 | port: portName
|
48 | for port in component.outPorts
|
49 | portName = port.id
|
50 | graph.outports[portName] =
|
51 | process: processName
|
52 | port: portName
|
53 |
|
54 |
|
55 | if Object.keys(graph.outports) < 1
|
56 | return callback new Error "Component '#{topic}' used as fixture has no outports"
|
57 | if Object.keys(graph.inports) < 1
|
58 | return callback new Error "Component '#{topic}' used as fixture has no inports"
|
59 |
|
60 | return callback null, graph
|
61 |
|
62 |
|
63 |
|
64 | getFixtureGraph = (context, suite, callback) ->
|
65 |
|
66 |
|
67 | hasComponents = (s) ->
|
68 | return s.components? and typeof s.components == 'object' and Object.keys(s.components).length
|
69 |
|
70 | updateComponents = (cb) ->
|
71 | return cb null if hasComponents context
|
72 | protocol.getComponents context.client, (err, components) ->
|
73 | return cb err if err
|
74 | context.components = components
|
75 | return cb null
|
76 |
|
77 | updateComponents (err) ->
|
78 | return callback err if err
|
79 |
|
80 | if not suite.fixture?
|
81 | return synthesizeTopicFixture suite.topic, context.components, callback
|
82 | else if suite.fixture.type == 'json'
|
83 | try
|
84 | graph = JSON.parse suite.fixture.data
|
85 | catch e
|
86 | return callback e
|
87 | return callback null, graph
|
88 | else if suite.fixture.type == 'fbp'
|
89 | try
|
90 | graph = fbp.parse suite.fixture.data
|
91 | catch
|
92 | return callback e
|
93 | graph.properties = {} if not graph.properties
|
94 | return callback null, graph
|
95 | else
|
96 | return callback new Error "Unknown fixture type #{suite.fixture.type}"
|
97 |
|
98 |
|
99 | sendMessageAndWait = (client, currentGraph, inputData, expectData, callback) ->
|
100 | received = {}
|
101 | onReceived = (port, data) =>
|
102 | debug 'runtest got output on', port
|
103 | received[port] = data
|
104 | nExpected = Object.keys(expectData).length
|
105 | if Object.keys(received).length == nExpected
|
106 | client.removeListener 'runtime', checkPacket
|
107 | return callback null, received
|
108 |
|
109 | checkPacket = (msg) =>
|
110 | d = msg.payload
|
111 |
|
112 | if msg.command == 'packet' and d.event == 'data'
|
113 | onReceived d.port, d.payload
|
114 | else if msg.command == 'packet' and ['begingroup', 'endgroup', 'connect', 'disconnect'].indexOf(d.event) != -1
|
115 |
|
116 | else
|
117 | debug 'unknown runtime message', msg
|
118 | client.on 'runtime', checkPacket
|
119 |
|
120 |
|
121 | protocol.sendPackets client, currentGraph, inputData, (err) =>
|
122 | return callback err if err
|
123 |
|
124 |
|
125 |
|
126 | class Runner
|
127 | constructor: (@client, options={}) ->
|
128 | if @client.protocol? and @client.address?
|
129 |
|
130 | Transport = fbpClient.getTransport @client.protocol
|
131 | @client = new Transport @client
|
132 | @currentGraphId = null
|
133 | @components = {}
|
134 | @options = options
|
135 | @options.connectTimeout = 5*1000 if not @options.connectTimeout?
|
136 |
|
137 |
|
138 | connect: (callback) ->
|
139 | debug 'connect'
|
140 |
|
141 | debugReceivedMessages @client
|
142 | @client.on 'network', ({command, payload}) ->
|
143 | console.log payload.message if command is 'output' and payload.message
|
144 |
|
145 | @client.on 'error', (err) ->
|
146 | debug 'connection failed', err
|
147 |
|
148 | timeBetweenAttempts = 500
|
149 | attempts = Math.floor(@options.connectTimeout / timeBetweenAttempts)
|
150 | isOnline = () =>
|
151 | connected = @client.isConnected()
|
152 | return if connected then Promise.resolve() else Promise.reject new Error 'Not connected to runtime'
|
153 | tryConnect = () =>
|
154 | debug 'trying to connect'
|
155 | @client.connect()
|
156 | return Promise.resolve()
|
157 | return common.retryUntil(tryConnect, isOnline, timeBetweenAttempts, attempts).asCallback callback
|
158 |
|
159 | disconnect: (callback) ->
|
160 | debug 'disconnect'
|
161 | onStatus = (status) =>
|
162 | err = if not status.online then null else new Error 'Runtime online after disconnect()'
|
163 | @client.removeListener 'status', onStatus
|
164 | debug 'disconnected', err
|
165 | return callback err
|
166 | @client.on 'status', onStatus
|
167 | @client.disconnect()
|
168 |
|
169 | setupSuite: (suite, callback) ->
|
170 | debug 'setup suite', "\"#{suite.name}\""
|
171 |
|
172 | getFixtureGraph this, suite, (err, graph) =>
|
173 | return callback err if err
|
174 | protocol.sendGraph @client, graph, (err, graphId) =>
|
175 | @currentGraphId = graphId
|
176 | return callback err if err
|
177 | protocol.startNetwork @client, graphId, (err) =>
|
178 | return callback err
|
179 |
|
180 | teardownSuite: (suite, callback) ->
|
181 | debug 'teardown suite', "\"#{suite.name}\""
|
182 |
|
183 | protocol.stopNetwork @client, @currentGraphId, (err) =>
|
184 | return callback err
|
185 |
|
186 | runTest: (testcase, callback) ->
|
187 | debug 'runtest', "\"#{testcase.name}\""
|
188 |
|
189 |
|
190 | inputs = if common.isArray(testcase.inputs) then testcase.inputs else [ testcase.inputs ]
|
191 | expects = if common.isArray(testcase.expect) then testcase.expect else [ testcase.expect ]
|
192 | sequence = []
|
193 | for i in [0...inputs.length]
|
194 | sequence.push
|
195 | inputs: inputs[i]
|
196 | expect: expects[i]
|
197 |
|
198 | sendWait = (data, cb) =>
|
199 | sendMessageAndWait @client, @currentGraphId, data.inputs, data.expect, cb
|
200 | common.asyncSeries sequence, sendWait, (err, actuals) ->
|
201 | actuals.forEach (r, idx) ->
|
202 | sequence[idx].actual = r
|
203 | return callback err, sequence
|
204 |
|
205 |
|
206 | checkResults = (results) ->
|
207 | actuals = results.filter (r) -> r.actual?
|
208 | expects = results.filter (r) -> r.expect?
|
209 | if actuals.length < expects.length
|
210 | return callback null,
|
211 | passed: false
|
212 | error: new Error "Only got #{actual.length} output messages out of #{expect.length}"
|
213 |
|
214 | results = results.map (res) ->
|
215 | res.error = null
|
216 | try
|
217 | expectation.expect res.expect, res.actual
|
218 | catch e
|
219 |
|
220 | res.error = e
|
221 | return res
|
222 |
|
223 | failures = results.filter (r) -> r.error
|
224 | if failures.length == 0
|
225 | result =
|
226 | passed: true
|
227 | else
|
228 | if expects.length == 1
|
229 | result =
|
230 | error: failures[0].error
|
231 | else if expects.length > 1 and failures.length == 1
|
232 | index = results.findIndex (r) -> r.error
|
233 | failed = results[index]
|
234 | result =
|
235 | error: new Error "Expectation #{index} of sequence failed: #{failed.error.message}"
|
236 | else
|
237 | errors = results.map (r) -> r.error?.message or ''
|
238 | result =
|
239 | error: new Error "Multiple failures in sequence: #{errors}"
|
240 |
|
241 | return result
|
242 |
|
243 | runTestAndCheck = (runner, testcase, callback) ->
|
244 | return callback null, { passed: true } if testcase.skip
|
245 |
|
246 |
|
247 |
|
248 | inputs = if common.isArray(testcase.inputs) then testcase.inputs else [ testcase.inputs ]
|
249 | expects = if common.isArray(testcase.expect) then testcase.expect else [ testcase.expect ]
|
250 | if inputs.length != expects.length
|
251 | return callback null,
|
252 | passed: false
|
253 | error: new Error "Test sequence length mismatch. Got #{inputs.length} inputs and #{expects.length} expectations"
|
254 |
|
255 | runner.runTest testcase, (err, results) ->
|
256 | return callback err, null if err
|
257 | result = checkResults results
|
258 | if result.error
|
259 | result.passed = false
|
260 | return callback null, result
|
261 |
|
262 | runSuite = (runner, suite, runTest, callback) ->
|
263 |
|
264 | runner.setupSuite suite, (err) ->
|
265 | debug 'setup suite', err
|
266 | return callback err, suite if err
|
267 |
|
268 | common.asyncSeries suite.cases, runTest, (err) ->
|
269 | debug 'testrun complete', err
|
270 |
|
271 | runner.teardownSuite suite, (err) ->
|
272 | debug 'teardown suite', err
|
273 | return callback err, suite
|
274 |
|
275 |
|
276 | exports.getComponentSuites = (runner, callback) ->
|
277 | protocol.getCapabilities runner.client, (err, caps) ->
|
278 | return callback err if err
|
279 | return callback null, [] unless 'component:getsource' in caps
|
280 |
|
281 | protocol.getComponentTests runner.client, (err, tests) ->
|
282 | return callback err if err
|
283 | suites = loadComponentSuites tests
|
284 | debug 'get component suites', tests.length, suites.length
|
285 | return callback null, suites
|
286 |
|
287 | loadComponentSuites = (componentTests) ->
|
288 | suites = []
|
289 | for name, tests of componentTests
|
290 | try
|
291 | ss = testsuite.loadYAML tests
|
292 | suites = suites.concat ss
|
293 | catch e
|
294 |
|
295 |
|
296 | continue
|
297 | return suites
|
298 |
|
299 |
|
300 |
|
301 | runAll = (runner, suites, updateCallback, doneCallback) ->
|
302 |
|
303 | runTest = (testcase, callback) ->
|
304 | done = (error) ->
|
305 | updateCallback suites
|
306 | callback error
|
307 |
|
308 | runTestAndCheck runner, testcase, (err, results) ->
|
309 | for key, val of results
|
310 | testcase[key] = val
|
311 | testcase.error = testcase.error.message if testcase.error
|
312 | debug 'ran test', '"testcase.name"', testcase.passed, err
|
313 | return done null
|
314 |
|
315 | runOneSuite = (suite, cb) ->
|
316 | runSuite runner, suite, runTest, cb
|
317 |
|
318 | debug 'running suites', (s.name for s in suites)
|
319 | common.asyncSeries suites, runOneSuite, (err) ->
|
320 | return doneCallback err
|
321 |
|
322 | exports.Runner = Runner
|
323 | exports.runAll = runAll
|
324 | exports.runTestAndCheck = runTestAndCheck
|