// MGC + idle pumps under realistic startup times — three scenarios that // pin down WHERE the live deadlock is happening when PS sends 100% but // pumps "show on" without adopting the control value. // // All three scenarios start with idle pumps (NOT pre-started) and use // non-zero state.time values so startup is observable. Each scenario // prints the per-pump snapshot at the end. The asserts state what we // EXPECT to happen — failures point at the exact codepath that breaks. // // Compare to demand-cycle-walkthrough.integration.test.js which // pre-starts every pump to 'operational' and therefore CANNOT exercise // the idle-during-rapid-retarget paths described here. const test = require('node:test'); const assert = require('node:assert/strict'); const MachineGroup = require('../../src/specificClass'); const Machine = require('../../../rotatingMachine/src/specificClass'); const HEAD_MBAR_UP = 0; const HEAD_MBAR_DOWN = 1100; const N_PUMPS = 3; const LOG_DEBUG = process.env.LOG_DEBUG === '1'; const logCfg = { enabled: LOG_DEBUG, logLevel: LOG_DEBUG ? 'debug' : 'error' }; // Production-realistic-but-shrunk: starting=1s, warmingup=2s. Total // startup ~3s. Long enough for rapid retargeting (every 200ms) to land // 10+ extra calls during the transient, short enough to keep the test // well under 30s. const stateConfig = { general: { logging: logCfg }, state: { current: 'idle' }, movement: { mode: 'staticspeed', speed: 200, maxSpeed: 200, interval: 50 }, time: { starting: 1, warmingup: 2, stopping: 1, coolingdown: 2 }, }; function machineConfig(id) { return { general: { logging: logCfg, name: id, id, unit: 'm3/h' }, functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' }, asset: { category: 'pump', type: 'centrifugal', model: 'hidrostal-H05K-S03R', supplier: 'hidrostal' }, mode: { current: 'auto', allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] }, allowedSources: { auto: ['parent', 'GUI'] }, }, sequences: { startup: ['starting', 'warmingup', 'operational'], shutdown: ['stopping', 'coolingdown', 'idle'], emergencystop: ['emergencystop', 'off'], }, }; } function groupConfig() { return { general: { logging: logCfg, name: 'mgc', id: 'mgc' }, functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' }, scaling: { current: 'normalized' }, mode: { current: 'optimalcontrol' }, }; } function buildGroup({ withPressure = true } = {}) { const mgc = new MachineGroup(groupConfig()); const ids = Array.from({ length: N_PUMPS }, (_, i) => `pump_${String.fromCharCode(97 + i)}`); const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig)); for (const m of pumps) { if (withPressure) { m.updateMeasuredPressure(HEAD_MBAR_UP, 'upstream', { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` }); m.updateMeasuredPressure(HEAD_MBAR_DOWN, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` }); } mgc.childRegistrationUtils.registerChild(m, 'downstream'); } mgc.calcAbsoluteTotals(); mgc.calcDynamicTotals(); return { mgc, pumps }; } const sleep = (ms) => new Promise(r => setTimeout(r, ms)); const NON_RUNNING = new Set(['idle', 'off', 'stopping', 'coolingdown', 'emergencystop']); function snapshot(pump) { const state = pump.state.getCurrentState(); const ctrl = Number(pump.state.getCurrentPosition?.() ?? 0); const running = !NON_RUNNING.has(state); const flow = running ? Number(pump.predictFlow?.outputY ?? 0) * 3600 : 0; const power = running ? Number(pump.predictPower?.outputY ?? 0) / 1000 : 0; return { state, ctrl, flow, power, delayedMove: pump.state.delayedMove }; } function printSnapshots(label, pumps) { console.log(`\n --- ${label} ---`); console.log(' ' + ['id'.padEnd(8), 'state'.padEnd(14), 'ctrl%'.padStart(6), 'Q m³/h'.padStart(8), 'kW'.padStart(6), 'delayedMove'.padStart(12)].join(' ')); console.log(' ' + '-'.repeat(60)); for (const p of pumps) { const s = snapshot(p); console.log(' ' + [ p.config.general.id.padEnd(8), s.state.padEnd(14), s.ctrl.toFixed(1).padStart(6), s.flow.toFixed(1).padStart(8), s.power.toFixed(1).padStart(6), String(s.delayedMove).padStart(12), ].join(' ')); } } function expectAllRunningAt100(pumps, label) { // After settle every pump should be operational with high ctrl% and // measurable flow. "high" is conservative — at 100% normalized demand, // 3-pump split puts each pump near 100% ctrl. Allow >70% as the floor // (accommodates BEP-Gravitation's slight asymmetry at the curve edges). for (const p of pumps) { const s = snapshot(p); assert.equal(s.state, 'operational', `${label}: pump ${p.config.general.id} expected operational, got '${s.state}' (ctrl=${s.ctrl.toFixed(1)}, delayedMove=${s.delayedMove})`); assert.ok(s.ctrl > 70, `${label}: pump ${p.config.general.id} expected ctrl% > 70 at 100% demand, got ${s.ctrl.toFixed(2)} (state=${s.state}, delayedMove=${s.delayedMove})`); assert.ok(s.flow > 100, `${label}: pump ${p.config.general.id} expected flow > 100 m³/h, got ${s.flow.toFixed(2)} (state=${s.state}, ctrl=${s.ctrl.toFixed(1)})`); } } // --------------------------------------------------------------------------- test('Scenario 1 — single-shot 100% demand to idle pumps', async () => { // Hypothesis A: a SINGLE handleInput call to MGC with all pumps idle is // enough to surface the bug. If pumps end up at 100% ctrl, the bug is // elsewhere (rapid retargeting OR pressure plumbing). If pumps stay at // 0%, the dispatch loop itself doesn't follow through on // execsequence-startup → flowmovement. const { mgc, pumps } = buildGroup(); console.log(`\n[Scenario 1] head=${HEAD_MBAR_DOWN} mbar, time.starting=${stateConfig.time.starting}s, time.warmingup=${stateConfig.time.warmingup}s`); printSnapshots('before handleInput', pumps); await mgc.handleInput('parent', 100); printSnapshots('immediately after handleInput returns', pumps); // Wait for full startup (3s) + movement (~0.5s) + slack await sleep(6000); printSnapshots('after 6s settle', pumps); expectAllRunningAt100(pumps, 'Scenario 1'); }); // --------------------------------------------------------------------------- test('Scenario 2 — rapid 100% retargeting during startup window', async () => { // Hypothesis B: PS fires _applyMachineGroupLevelControl on every level // tick (every few hundred ms). While pumps are in 'starting' / // 'warmingup', MGC's optimalControl loop snapshots them, hits NONE of // its three branches (idle / operational / flow<=0), and dispatches // nothing. The only reason pumps eventually move is the FIRST call's // queued `await flowmovement` after `await execsequence startup` — // unless a subsequent call's abortActiveMovements aborts that move // mid-flight, parking it in 'accelerating'/'decelerating'. const { mgc, pumps } = buildGroup(); console.log(`\n[Scenario 2] firing mgc.handleInput('parent', 100) every 200ms for 5s`); printSnapshots('before any handleInput', pumps); // First call (kicks off startup); not awaited so retargets can layer on. mgc.handleInput('parent', 100).catch(e => console.log(`first call rejected: ${e.message}`)); // Spam additional retargets every 200ms for 5s — covers the 3s startup // window with 25 extra retargeting calls. const interval = setInterval(() => { mgc.handleInput('parent', 100).catch(e => console.log(`retarget rejected: ${e.message}`)); }, 200); await sleep(5000); clearInterval(interval); printSnapshots('right after retarget barrage stops', pumps); // Drain: let any pending moves finish and let the FSM settle. await sleep(3000); printSnapshots('after 3s drain', pumps); expectAllRunningAt100(pumps, 'Scenario 2'); }); // --------------------------------------------------------------------------- test('Scenario 3 — pumps with NO pressure measurements injected', async () => { // Hypothesis C: in production, MGC may receive a demand BEFORE the // first pressure measurement has propagated. Without head, the curve's // operating point is at fDimension=defaults, and currentFxyYMin/Max // may not correspond to a usable envelope. If MGC's distributor then // hands every pump flow≤0, the dispatch loop falls into the 'flow<=0 // → shutdown' branch and pumps go straight to idle. const { mgc, pumps } = buildGroup({ withPressure: false }); const sample = pumps[0].groupPredictFlow ?? pumps[0].predictFlow; const minQ = sample.currentFxyYMin * 3600; const maxQ = sample.currentFxyYMax * 3600; const dyn = mgc.calcDynamicTotals(); console.log(`\n[Scenario 3] no pressure injected. per-pump curve envelope: ${minQ.toFixed(1)} .. ${maxQ.toFixed(1)} m³/h, station: ${(dyn.flow.min*3600).toFixed(1)} .. ${(dyn.flow.max*3600).toFixed(1)} m³/h`); printSnapshots('before handleInput', pumps); await mgc.handleInput('parent', 100); await sleep(6000); printSnapshots('after 6s settle (no pressure)', pumps); // We don't assert success here — this scenario is exploratory. Just // log what happens. If pumps DO ramp despite no pressure, MGC is // resilient. If they stay idle, that's a meaningful failure mode for // the live system because a redeploy may rebuild the world before // sensors republish. console.log(' (Scenario 3 is exploratory — no asserts; review the snapshot above.)'); }); // --------------------------------------------------------------------------- test('Scenario 4 — varying demand during startup (combo flips)', async () => { // Hypothesis D: in production the demand is NOT constant — as basin // level rises, percControl ramps from startLevel→maxLevel over the // basin model. Demand can flip between 1-pump / 2-pump / 3-pump // combinations every PS tick. Each flip in optimalControl tells some // pumps to start, others to shutdown, others nothing. If a pump that // was just told "startup" is told "shutdown" 1s later (still in // 'starting' state — neither idle nor operational), nothing happens // for that pump in this snapshot. The execsequence shutdown branch // requires state to be operational/accelerating/decelerating — a // 'starting'/'warmingup' pump is silently passed over for shutdown // too. The pump then proceeds to operational AND obeys its queued // flowmovement, even though MGC's intent has since changed. const { mgc, pumps } = buildGroup(); const sequence = [25, 75, 50, 100, 30, 90, 60, 100]; console.log(`\n[Scenario 4] varying demand sequence: ${sequence.join(' → ')} (each held 400ms)`); printSnapshots('before any handleInput', pumps); for (const pct of sequence) { console.log(` → demand ${pct}%`); mgc.handleInput('parent', pct).catch(e => console.log(`call ${pct}% rejected: ${e.message}`)); await sleep(400); } printSnapshots('right after sequence ends', pumps); // Final demand was 100% — drain and verify pumps converged. await sleep(4000); printSnapshots('after 4s drain (demand was last set to 100%)', pumps); expectAllRunningAt100(pumps, 'Scenario 4'); });