machineGroupControl/test/integration/turnoff-deadlock.integration.test.js

// Regression: pump A in pumpingstation-complete-example demo got stuck
// running at minimum flow while basin level dropped past stopLevel and
// kept dropping all the way to dry-run threshold.
//
// Root cause (two parts):
//
// 1. rotatingMachine.executeSequence on shutdown went through an
//    interruptible-abort path that returned the FSM to 'operational',
//    triggering state.transitionToState's auto-pickup of the queued
//    delayedMove — re-engaging the pump before the shutdown sequence
//    could reach stopping/coolingdown/idle. Fix: clear delayedMove at
//    the top of shutdown/emergencystop sequences.
//
// 2. PS calls turnOffAllMachines on every tick (every 2 s) while
//    level < stopLevel. Each call interrupted the still-running prior
//    shutdown's transitions, resetting the FSM to 'accelerating'. The
//    pump bounced accelerating ↔ decelerating forever and the actual
//    shutdown sequence transitions never ran. Fix: serialize per-pump
//    shutdown calls in turnOffAllMachines so concurrent invocations
//    are no-ops while a shutdown is already in flight.
//
// This test exercises part 2 — the per-pump serialization at the MGC
// level — by hammering turnOffAllMachines from a tight loop, mirroring
// the live tick cadence.

const test = require('node:test');
const assert = require('node:assert/strict');

const MachineGroup = require('../../src/specificClass');
const Machine = require('../../../rotatingMachine/src/specificClass');

const logCfg = { enabled: false, logLevel: 'error' };

const stateConfig = {
  general: { logging: logCfg },
  state: { current: 'idle' },
  movement: { mode: 'staticspeed', speed: 50, maxSpeed: 100, interval: 10 },
  // Non-zero shutdown timing so a shutdown takes long enough that a
  // concurrent turnOff call lands mid-sequence — exactly the live race.
  time: { starting: 0, warmingup: 0, stopping: 1, coolingdown: 1 },
};

function machineConfig(id) {
  return {
    general: { logging: logCfg, name: id, id, unit: 'm3/h' },
    functionality: { softwareType: 'machine', role: 'rotationaldevicecontroller' },
    asset: { category: 'pump', type: 'centrifugal', model: 'hidrostal-H05K-S03R', supplier: 'hidrostal' },
    mode: {
      current: 'auto',
      allowedActions: { auto: ['execsequence', 'execmovement', 'flowmovement', 'statuscheck'] },
      allowedSources: { auto: ['parent', 'GUI'] },
    },
    sequences: {
      startup:        ['starting', 'warmingup', 'operational'],
      shutdown:       ['stopping', 'coolingdown', 'idle'],
      emergencystop:  ['emergencystop', 'off'],
    },
  };
}

function groupConfig() {
  return {
    general: { logging: logCfg, name: 'mgc', id: 'mgc' },
    functionality: { softwareType: 'machinegroup', role: 'groupcontroller', positionVsParent: 'atEquipment' },
    scaling: { current: 'normalized' },
    mode:    { current: 'optimalcontrol' },
  };
}

function buildGroup() {
  const mgc = new MachineGroup(groupConfig());
  const ids = ['pump_a', 'pump_b', 'pump_c'];
  const pumps = ids.map(id => new Machine(machineConfig(id), stateConfig));
  for (const m of pumps) {
    m.updateMeasuredPressure(0,    'upstream',   { timestamp: Date.now(), unit: 'mbar', childName: 'up', childId: `up-${m.config.general.id}` });
    m.updateMeasuredPressure(1100, 'downstream', { timestamp: Date.now(), unit: 'mbar', childName: 'dn', childId: `dn-${m.config.general.id}` });
    mgc.childRegistrationUtils.registerChild(m, 'downstream');
  }
  mgc.calcAbsoluteTotals();
  mgc.calcDynamicTotals();
  return { mgc, pumps };
}

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

test('repeated turnOffAllMachines reaches idle (serializes concurrent shutdowns)', async () => {
  const { mgc, pumps } = buildGroup();
  const pumpA = pumps[0];

  // Start pump A and queue a delayedMove the way MGC's optimalControl
  // would when PS sends a 1% dead-zone keep-alive.
  await pumpA.handleInput('parent', 'execsequence', 'startup');
  assert.equal(pumpA.state.getCurrentState(), 'operational');
  pumpA.setpoint(80);                  // start a slow move (not awaited)
  await sleep(50);
  assert.equal(pumpA.state.getCurrentState(), 'accelerating');
  pumpA.state.delayedMove = 75;

  // Mimic PS's tick loop: fire turnOffAllMachines on a tight cadence
  // without awaiting. Without the per-pump serialization in
  // turnOffAllMachines, each call hits the still-running prior shutdown
  // and bounces the pump back to accelerating — the live deadlock.
  const ticks = [];
  for (let i = 0; i < 6; i++) {
    ticks.push(mgc.turnOffAllMachines());
    await sleep(80);                   // half the realtime tick — tighter race
  }
  await Promise.all(ticks);
  // Allow the (single) in-flight shutdown to finish its 1+1 s timed
  // transitions through stopping → coolingdown → idle.
  await sleep(2500);

  assert.equal(pumpA.state.getCurrentState(), 'idle',
    `pump must reach idle under repeated turnOff calls; got ${pumpA.state.getCurrentState()} (delayedMove=${pumpA.state.delayedMove})`);
  assert.equal(pumpA.state.delayedMove, null,
    'delayedMove must be cleared after shutdown');
});

test('turnOffAllMachines clears MGC._delayedCall to cancel any deferred dispatch', async () => {
  // PS sends a 1% keep-alive while MGC is mid-dispatch. MGC parks it in
  // _delayedCall. PS then crosses stopLevel and calls turnOffAllMachines.
  // Without clearing _delayedCall, MGC's finally block fires the parked
  // 1% call AFTER the shutdown — re-engaging the pump.
  const { mgc } = buildGroup();
  mgc._delayedCall = { source: 'parent', demand: 1, powerCap: Infinity, priorityList: null };

  await mgc.turnOffAllMachines();

  assert.equal(mgc._delayedCall, null,
    'turnOff must cancel any deferred dispatch so it cannot re-engage pumps post-shutdown');
});