// MGC + idle pumps under realistic startup times — three scenarios that // pin down WHERE the live deadlock is happening when PS sends 100% but // pumps "show on" without adopting the control value. // // All three scenarios start with idle pumps (NOT pre-started) and use // non-zero state.time values so startup is observable. Each scenario // prints the per-pump snapshot at the end. The asserts state what we // EXPECT to happen — failures point at the exact codepath that breaks. // // Compare to demand-cycle-walkthrough.integration.test.js which // pre-starts every pump to 'operational' and therefore CANNOT exercise // the idle-during-rapid-retarget paths described here. const test = require('node:test'); const assert = require('node:assert/strict'); const MachineGroup = require('../../src/specificClass'); const Machine = require('../../../rotatingMachine/src/specificClass'); const HEAD_MBAR_UP = 0; const HEAD_MBAR_DOWN = 1100; const N_PUMPS = 3; const LOG_DEBUG = process.env.LOG_DEBUG === '1'; const logCfg = { enabled: LOG_DEBUG, logLevel: LOG_DEBUG ? 'debug' : 'error' }; // Production-realistic-but-shrunk: starting=1s, warmingup=2s. Total // startup ~3s. Long enough for rapid retargeting (every 200ms) to land // 10+ extra calls during the transient, short enough to keep the test // well under 30s. const stateConfig = { general: { logging: logCfg }, state: { current: 'idle' }, movement: { mode: 'staticspeed', speed: 200, maxSpeed: 200, interval: 50 }, time: { starting: 1, warmingup: 2, stopping: 1, coolingdown: 2 }, }; function machineConfig(id) { return { general: { logging: logCfg, name: id, id, unit: 'm3/h' }, functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' }, asset: { model: 'hidrostal-H05K-S03R', unit: 'm3/h' }, mode: { current: 'auto', allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] }, allowedSources: { auto: ['parent', 'GUI'] }, }, sequences: { startup: ['starting', 'warmingup', 'operational'], shutdown: ['stopping', 'coolingdown', 'idle'], emergencystop: ['emergencystop', 'off'], }, }; } function groupConfig() { return { general: { logging: logCfg, name: 'mgc', id: 'mgc' }, functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' }, mode: { current: 'optimalcontrol' }, }; } // Post-refactor handleInput takes canonical m³/s. This helper mirrors what // the set.demand handler does for a bare-number (percent) payload, so test // scenarios that previously sent `mgc.handleInput('parent', pctToCanonical(mgc, 100))` (= 100 %) // keep their intent. function pctToCanonical(mgc, pct) { if (pct < 0) return -1; const dt = mgc.calcDynamicTotals(); return mgc.interpolation.interpolate_lin_single_point(pct, 0, 100, dt.flow.min, dt.flow.max); } function buildGroup({ withPressure = true } = {}) { const mgc = new MachineGroup(groupConfig()); const ids = Array.from({ length: N_PUMPS }, (_, i) => `pump_${String.fromCharCode(97 + i)}`); const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig)); for (const m of pumps) { if (withPressure) { m.updateMeasuredPressure(HEAD_MBAR_UP, 'upstream', { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` }); m.updateMeasuredPressure(HEAD_MBAR_DOWN, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` }); } mgc.childRegistrationUtils.registerChild(m, 'downstream'); } mgc.calcAbsoluteTotals(); mgc.calcDynamicTotals(); return { mgc, pumps }; } const sleep = (ms) => new Promise(r => setTimeout(r, ms)); const NON_RUNNING = new Set(['idle', 'off', 'stopping', 'coolingdown', 'emergencystop']); function snapshot(pump) { const state = pump.state.getCurrentState(); const ctrl = Number(pump.state.getCurrentPosition?.() ?? 0); const running = !NON_RUNNING.has(state); const flow = running ? Number(pump.predictFlow?.outputY ?? 0) * 3600 : 0; const power = running ? Number(pump.predictPower?.outputY ?? 0) / 1000 : 0; return { state, ctrl, flow, power, delayedMove: pump.state.delayedMove }; } function printSnapshots(label, pumps) { console.log(`\n --- ${label} ---`); console.log(' ' + ['id'.padEnd(8), 'state'.padEnd(14), 'ctrl%'.padStart(6), 'Q m³/h'.padStart(8), 'kW'.padStart(6), 'delayedMove'.padStart(12)].join(' ')); console.log(' ' + '-'.repeat(60)); for (const p of pumps) { const s = snapshot(p); console.log(' ' + [ p.config.general.id.padEnd(8), s.state.padEnd(14), s.ctrl.toFixed(1).padStart(6), s.flow.toFixed(1).padStart(8), s.power.toFixed(1).padStart(6), String(s.delayedMove).padStart(12), ].join(' ')); } } function expectAllRunningAt100(pumps, label) { // After settle every pump should be operational with high ctrl% and // measurable flow. "high" is conservative — at 100% normalized demand, // 3-pump split puts each pump near 100% ctrl. Allow >70% as the floor // (accommodates BEP-Gravitation's slight asymmetry at the curve edges). for (const p of pumps) { const s = snapshot(p); assert.equal(s.state, 'operational', `${label}: pump ${p.config.general.id} expected operational, got '${s.state}' (ctrl=${s.ctrl.toFixed(1)}, delayedMove=${s.delayedMove})`); assert.ok(s.ctrl > 70, `${label}: pump ${p.config.general.id} expected ctrl% > 70 at 100% demand, got ${s.ctrl.toFixed(2)} (state=${s.state}, delayedMove=${s.delayedMove})`); assert.ok(s.flow > 100, `${label}: pump ${p.config.general.id} expected flow > 100 m³/h, got ${s.flow.toFixed(2)} (state=${s.state}, ctrl=${s.ctrl.toFixed(1)})`); } } // --------------------------------------------------------------------------- test('Scenario 1 — single-shot 100% demand to idle pumps', async () => { // Hypothesis A: a SINGLE handleInput call to MGC with all pumps idle is // enough to surface the bug. If pumps end up at 100% ctrl, the bug is // elsewhere (rapid retargeting OR pressure plumbing). If pumps stay at // 0%, the dispatch loop itself doesn't follow through on // execsequence-startup → flowmovement. const { mgc, pumps } = buildGroup(); console.log(`\n[Scenario 1] head=${HEAD_MBAR_DOWN} mbar, time.starting=${stateConfig.time.starting}s, time.warmingup=${stateConfig.time.warmingup}s`); printSnapshots('before handleInput', pumps); await mgc.handleInput('parent', pctToCanonical(mgc, 100)); printSnapshots('immediately after handleInput returns', pumps); // Wait for full startup (3s) + movement (~0.5s) + slack await sleep(6000); printSnapshots('after 6s settle', pumps); expectAllRunningAt100(pumps, 'Scenario 1'); }); // --------------------------------------------------------------------------- test('Scenario 2 — rapid 100% retargeting during startup window', async () => { // Hypothesis B: PS fires _applyMachineGroupLevelControl on every level // tick (every few hundred ms). While pumps are in 'starting' / // 'warmingup', MGC's optimalControl loop snapshots them, hits NONE of // its three branches (idle / operational / flow<=0), and dispatches // nothing. The only reason pumps eventually move is the FIRST call's // queued `await flowmovement` after `await execsequence startup` — // unless a subsequent call's abortActiveMovements aborts that move // mid-flight, parking it in 'accelerating'/'decelerating'. const { mgc, pumps } = buildGroup(); console.log(`\n[Scenario 2] firing mgc.handleInput('parent', pctToCanonical(mgc, 100)) every 200ms for 5s`); printSnapshots('before any handleInput', pumps); // First call (kicks off startup); not awaited so retargets can layer on. mgc.handleInput('parent', pctToCanonical(mgc, 100)).catch(e => console.log(`first call rejected: ${e.message}`)); // Spam additional retargets every 200ms for 5s — covers the 3s startup // window with 25 extra retargeting calls. const interval = setInterval(() => { mgc.handleInput('parent', pctToCanonical(mgc, 100)).catch(e => console.log(`retarget rejected: ${e.message}`)); }, 200); await sleep(5000); clearInterval(interval); printSnapshots('right after retarget barrage stops', pumps); // Drain: let any pending moves finish and let the FSM settle. await sleep(3000); printSnapshots('after 3s drain', pumps); expectAllRunningAt100(pumps, 'Scenario 2'); }); // --------------------------------------------------------------------------- test('Scenario 3 — pumps with NO pressure measurements injected', async () => { // Hypothesis C: in production, MGC may receive a demand BEFORE the // first pressure measurement has propagated. Without head, the curve's // operating point is at fDimension=defaults, and currentFxyYMin/Max // may not correspond to a usable envelope. If MGC's distributor then // hands every pump flow≤0, the dispatch loop falls into the 'flow<=0 // → shutdown' branch and pumps go straight to idle. const { mgc, pumps } = buildGroup({ withPressure: false }); const sample = pumps[0].groupPredictFlow ?? pumps[0].predictFlow; const minQ = sample.currentFxyYMin * 3600; const maxQ = sample.currentFxyYMax * 3600; const dyn = mgc.calcDynamicTotals(); console.log(`\n[Scenario 3] no pressure injected. per-pump curve envelope: ${minQ.toFixed(1)} .. ${maxQ.toFixed(1)} m³/h, station: ${(dyn.flow.min*3600).toFixed(1)} .. ${(dyn.flow.max*3600).toFixed(1)} m³/h`); printSnapshots('before handleInput', pumps); await mgc.handleInput('parent', pctToCanonical(mgc, 100)); await sleep(6000); printSnapshots('after 6s settle (no pressure)', pumps); // We don't assert success here — this scenario is exploratory. Just // log what happens. If pumps DO ramp despite no pressure, MGC is // resilient. If they stay idle, that's a meaningful failure mode for // the live system because a redeploy may rebuild the world before // sensors republish. console.log(' (Scenario 3 is exploratory — no asserts; review the snapshot above.)'); }); // --------------------------------------------------------------------------- test('Scenario 5 — full up/down/up cycle through shutdown', async () => { // Hypothesis E: when demand goes 100% → 0% → 100% (basin fills, drains // past stopLevel, then refills), pumps pass through stopping → // coolingdown → idle. If a fresh flow>0 demand arrives while a pump is // mid-shutdown, the current MGC dispatch saves flowmovement to // delayedMove (good) but doesn't issue execsequence-startup because // state !== 'idle' (bug). The pump completes shutdown, reaches 'idle', // and stays there because transitionToState('idle') doesn't fire // delayedMove — only the transition INTO 'operational' does. Pump is // stuck with delayedMove orphaned. const { mgc, pumps } = buildGroup(); console.log('\n[Scenario 5] cycle: 100% → 0% → 100% with mid-shutdown re-engage'); printSnapshots('before any handleInput', pumps); // Phase 1: drive up to 100% from idle. await mgc.handleInput('parent', pctToCanonical(mgc, 100)); await sleep(5000); // full startup + ramp printSnapshots('after settle at 100%', pumps); for (const p of pumps) { assert.equal(p.state.getCurrentState(), 'operational', `Phase 1: pump ${p.config.general.id} not operational at 100% (got ${p.state.getCurrentState()})`); } // Phase 2: demand drops below 0 — pumps begin shutdown sequence. Use a // strictly-negative percent because 0% now means "minimum-control" // (interpolates to dt.flow.min), not shutdown. // FIRE-AND-FORGET: handleInput(-1) awaits turnOffAllMachines which // awaits the full per-pump shutdown sequence. We need the next 100% // demand to arrive WHILE pumps are still in stopping/coolingdown, // not after they've reached idle. mgc.turnOffAllMachines().catch(e => console.log(`-1% rejected: ${e.message}`)); // Wait briefly so the shutdown sequence enters but does NOT complete. // shutdown=['stopping','coolingdown','idle'] with stopping=1s, // coolingdown=2s. 500ms puts us solidly inside 'stopping'. await sleep(500); printSnapshots('mid-shutdown (pumps should be in stopping/coolingdown)', pumps); const midShutdownStates = pumps.map(p => p.state.getCurrentState()); console.log(` states mid-shutdown: ${midShutdownStates.join(', ')}`); // Phase 3: demand returns to 100% while pumps are mid-shutdown. await mgc.handleInput('parent', pctToCanonical(mgc, 100)); // Generous: full coolingdown remaining + full startup + ramp. await sleep(8000); printSnapshots('after re-engage to 100%', pumps); expectAllRunningAt100(pumps, 'Scenario 5'); }); // --------------------------------------------------------------------------- test('Scenario 6 — full up sweep then full down sweep', async () => { // Hypothesis F: the user observed "going up stuck ~60%, going down // not reacting". Mirror that with an explicit up-then-down monotonic // sweep, every step holding 600 ms (slightly longer than DWELL on // production basin model). After the sweep, we expect the LATEST // demand (the final value of the down-sweep, which is 10%) to be // honoured: pumps either at 1-pump combo's split or all idle if that // demand falls below the per-pump minimum. const { mgc, pumps } = buildGroup(); console.log('\n[Scenario 6] up-sweep 10%→100% then down-sweep 100%→10%, each step 600 ms'); printSnapshots('before any handleInput', pumps); const upSteps = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; const downSteps = [90, 80, 70, 60, 50, 40, 30, 20, 10]; console.log(' --- up sweep ---'); for (const pct of upSteps) { mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`up ${pct}% rejected: ${e.message}`)); await sleep(600); const snaps = pumps.map(snapshot); const totalQ = snaps.reduce((s, x) => s + x.flow, 0); console.log(` cmd=${pct.toFixed(0).padStart(3)}% states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`); } printSnapshots('top of up-sweep (cmd=100%) after full settle', pumps); await sleep(2000); printSnapshots('top of up-sweep + 2s drain', pumps); console.log(' --- down sweep ---'); for (const pct of downSteps) { mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`down ${pct}% rejected: ${e.message}`)); await sleep(600); const snaps = pumps.map(snapshot); const totalQ = snaps.reduce((s, x) => s + x.flow, 0); console.log(` cmd=${pct.toFixed(0).padStart(3)}% states=[${snaps.map(s=>s.state.padEnd(13)).join(', ')}] ctrl=[${snaps.map(s=>s.ctrl.toFixed(1).padStart(5)).join(', ')}] ΣQ=${totalQ.toFixed(1)}`); } printSnapshots('bottom of down-sweep (cmd=10%) after sequence', pumps); await sleep(3000); printSnapshots('bottom of down-sweep + 3s drain', pumps); // Final demand was 10% (≈ 148 m³/h). At head 1100 mbar with per-pump // min ≈ 89.5, this is solvable by a 1-pump combo near 148 m³/h. // Optimizer typically picks the 1-pump combo. Either way, pumps are // NOT supposed to be stuck at the prior up-sweep's 100% setpoint. const flowMin_m3h = mgc.calcDynamicTotals().flow.min * 3600; const flowMax_m3h = mgc.calcDynamicTotals().flow.max * 3600; const expectedQ_m3h = flowMin_m3h + (flowMax_m3h - flowMin_m3h) * 0.10; // 10% scaled console.log(` expected total flow at 10%: ~${expectedQ_m3h.toFixed(1)} m³/h`); const snaps = pumps.map(snapshot); const totalQ = snaps.reduce((s, x) => s + x.flow, 0); // Loose: total within 30 m³/h of expectation. Catches the obvious // stuck-at-old-position regression. assert.ok(Math.abs(totalQ - expectedQ_m3h) < 30, `Scenario 6: total flow ${totalQ.toFixed(1)} m³/h diverged from expected ${expectedQ_m3h.toFixed(1)} after down-sweep — pumps did not adopt latest demand. Per-pump: ${snaps.map(s => `${s.state}@${s.ctrl.toFixed(0)}%`).join(', ')}`); }); // --------------------------------------------------------------------------- test('Scenario 4 — varying demand during startup (combo flips)', async () => { // Hypothesis D: in production the demand is NOT constant — as basin // level rises, percControl ramps from startLevel→maxLevel over the // basin model. Demand can flip between 1-pump / 2-pump / 3-pump // combinations every PS tick. Each flip in optimalControl tells some // pumps to start, others to shutdown, others nothing. If a pump that // was just told "startup" is told "shutdown" 1s later (still in // 'starting' state — neither idle nor operational), nothing happens // for that pump in this snapshot. The execsequence shutdown branch // requires state to be operational/accelerating/decelerating — a // 'starting'/'warmingup' pump is silently passed over for shutdown // too. The pump then proceeds to operational AND obeys its queued // flowmovement, even though MGC's intent has since changed. const { mgc, pumps } = buildGroup(); const sequence = [25, 75, 50, 100, 30, 90, 60, 100]; console.log(`\n[Scenario 4] varying demand sequence: ${sequence.join(' → ')} (each held 400ms)`); printSnapshots('before any handleInput', pumps); for (const pct of sequence) { console.log(` → demand ${pct}%`); mgc.handleInput('parent', pctToCanonical(mgc, pct)).catch(e => console.log(`call ${pct}% rejected: ${e.message}`)); await sleep(400); } printSnapshots('right after sequence ends', pumps); // Final demand was 100% — drain and verify pumps converged. await sleep(4000); printSnapshots('after 4s drain (demand was last set to 100%)', pumps); expectAllRunningAt100(pumps, 'Scenario 4'); });