diff --git a/src/specificClass.js b/src/specificClass.js index 99b697d..e17b8b6 100644 --- a/src/specificClass.js +++ b/src/specificClass.js @@ -1421,8 +1421,28 @@ class MachineGroup { } async turnOffAllMachines(){ + // Cancel any deferred dispatch — turnOff is the latest user intent, + // a stale 1% keep-alive must not re-engage a pump after we shut down. + this._delayedCall = null; + // Per-pump shutdown serialization: PS calls turnOffAllMachines on + // every tick (every 2 s) once level < stopLevel. Without this guard, + // each new shutdown call hits the still-running prior shutdown's + // movement transitions and triggers abortCurrentMovement, which + // bounces the pump back to 'operational' before the sequence loop + // can reach stopping/coolingdown/idle. Net effect: pump never + // actually shuts down. Track shutdown-in-flight per pump and skip + // if already underway. + if (!this._shutdownInFlight) this._shutdownInFlight = new Set(); await Promise.all(Object.entries(this.machines).map(async ([machineId, machine]) => { - if (this.isMachineActive(machineId)) { await machine.handleInput("parent", "execsequence", "shutdown"); } + if (this._shutdownInFlight.has(machineId)) return; + if (this.isMachineActive(machineId)) { + this._shutdownInFlight.add(machineId); + try { + await machine.handleInput("parent", "execsequence", "shutdown"); + } finally { + this._shutdownInFlight.delete(machineId); + } + } })); // Update measurements to zero so the parent (PS) sees the // outflow drop immediately — without this the PS keeps the diff --git a/test/integration/turnoff-deadlock.integration.test.js b/test/integration/turnoff-deadlock.integration.test.js new file mode 100644 index 0000000..542faf7 --- /dev/null +++ b/test/integration/turnoff-deadlock.integration.test.js @@ -0,0 +1,131 @@ +// Regression: pump A in pumpingstation-complete-example demo got stuck +// running at minimum flow while basin level dropped past stopLevel and +// kept dropping all the way to dry-run threshold. +// +// Root cause (two parts): +// +// 1. rotatingMachine.executeSequence on shutdown went through an +// interruptible-abort path that returned the FSM to 'operational', +// triggering state.transitionToState's auto-pickup of the queued +// delayedMove — re-engaging the pump before the shutdown sequence +// could reach stopping/coolingdown/idle. Fix: clear delayedMove at +// the top of shutdown/emergencystop sequences. +// +// 2. PS calls turnOffAllMachines on every tick (every 2 s) while +// level < stopLevel. Each call interrupted the still-running prior +// shutdown's transitions, resetting the FSM to 'accelerating'. The +// pump bounced accelerating ↔ decelerating forever and the actual +// shutdown sequence transitions never ran. Fix: serialize per-pump +// shutdown calls in turnOffAllMachines so concurrent invocations +// are no-ops while a shutdown is already in flight. +// +// This test exercises part 2 — the per-pump serialization at the MGC +// level — by hammering turnOffAllMachines from a tight loop, mirroring +// the live tick cadence. + +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const MachineGroup = require('../../src/specificClass'); +const Machine = require('../../../rotatingMachine/src/specificClass'); + +const logCfg = { enabled: false, logLevel: 'error' }; + +const stateConfig = { + general: { logging: logCfg }, + state: { current: 'idle' }, + movement: { mode: 'staticspeed', speed: 50, maxSpeed: 100, interval: 10 }, + // Non-zero shutdown timing so a shutdown takes long enough that a + // concurrent turnOff call lands mid-sequence — exactly the live race. + time: { starting: 0, warmingup: 0, stopping: 1, coolingdown: 1 }, +}; + +function machineConfig(id) { + return { + general: { logging: logCfg, name: id, id, unit: 'm3/h' }, + functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' }, + asset: { category: 'pump', type: 'centrifugal', model: 'hidrostal-H05K-S03R', supplier: 'hidrostal' }, + mode: { + current: 'auto', + allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] }, + allowedSources: { auto: ['parent', 'GUI'] }, + }, + sequences: { + startup: ['starting', 'warmingup', 'operational'], + shutdown: ['stopping', 'coolingdown', 'idle'], + emergencystop: ['emergencystop', 'off'], + }, + }; +} + +function groupConfig() { + return { + general: { logging: logCfg, name: 'mgc', id: 'mgc' }, + functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' }, + scaling: { current: 'normalized' }, + mode: { current: 'optimalcontrol' }, + }; +} + +function buildGroup() { + const mgc = new MachineGroup(groupConfig()); + const ids = ['pump_a', 'pump_b', 'pump_c']; + const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig)); + for (const m of pumps) { + m.updateMeasuredPressure(0, 'upstream', { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` }); + m.updateMeasuredPressure(1100, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` }); + mgc.childRegistrationUtils.registerChild(m, 'downstream'); + } + mgc.calcAbsoluteTotals(); + mgc.calcDynamicTotals(); + return { mgc, pumps }; +} + +const sleep = (ms) => new Promise(r => setTimeout(r, ms)); + +test('repeated turnOffAllMachines reaches idle (serializes concurrent shutdowns)', async () => { + const { mgc, pumps } = buildGroup(); + const pumpA = pumps[0]; + + // Start pump A and queue a delayedMove the way MGC's optimalControl + // would when PS sends a 1% dead-zone keep-alive. + await pumpA.handleInput('parent', 'execsequence', 'startup'); + assert.equal(pumpA.state.getCurrentState(), 'operational'); + pumpA.setpoint(80); // start a slow move (not awaited) + await sleep(50); + assert.equal(pumpA.state.getCurrentState(), 'accelerating'); + pumpA.state.delayedMove = 75; + + // Mimic PS's tick loop: fire turnOffAllMachines on a tight cadence + // without awaiting. Without the per-pump serialization in + // turnOffAllMachines, each call hits the still-running prior shutdown + // and bounces the pump back to accelerating — the live deadlock. + const ticks = []; + for (let i = 0; i < 6; i++) { + ticks.push(mgc.turnOffAllMachines()); + await sleep(80); // half the realtime tick — tighter race + } + await Promise.all(ticks); + // Allow the (single) in-flight shutdown to finish its 1+1 s timed + // transitions through stopping → coolingdown → idle. + await sleep(2500); + + assert.equal(pumpA.state.getCurrentState(), 'idle', + `pump must reach idle under repeated turnOff calls; got ${pumpA.state.getCurrentState()} (delayedMove=${pumpA.state.delayedMove})`); + assert.equal(pumpA.state.delayedMove, null, + 'delayedMove must be cleared after shutdown'); +}); + +test('turnOffAllMachines clears MGC._delayedCall to cancel any deferred dispatch', async () => { + // PS sends a 1% keep-alive while MGC is mid-dispatch. MGC parks it in + // _delayedCall. PS then crosses stopLevel and calls turnOffAllMachines. + // Without clearing _delayedCall, MGC's finally block fires the parked + // 1% call AFTER the shutdown — re-engaging the pump. + const { mgc } = buildGroup(); + mgc._delayedCall = { source: 'parent', demand: 1, powerCap: Infinity, priorityList: null }; + + await mgc.turnOffAllMachines(); + + assert.equal(mgc._delayedCall, null, + 'turnOff must cancel any deferred dispatch so it cannot re-engage pumps post-shutdown'); +});