From c2132ea7eb3ce40668648ad03c8b0e979d5aafb1 Mon Sep 17 00:00:00 2001 From: Mikayla Fischler Date: Thu, 6 Apr 2023 12:52:25 -0400 Subject: [PATCH] #147 possible fix for MODBUS failures on server startup --- install_manifest.json | 2 +- rtu/dev/boilerv_rtu.lua | 11 ++++++++++- rtu/dev/envd_rtu.lua | 11 ++++++++++- rtu/dev/imatrix_rtu.lua | 11 ++++++++++- rtu/dev/redstone_rtu.lua | 3 ++- rtu/dev/sna_rtu.lua | 11 ++++++++++- rtu/dev/sps_rtu.lua | 11 ++++++++++- rtu/dev/turbinev_rtu.lua | 11 ++++++++++- rtu/startup.lua | 28 ++++++++++++++++++---------- rtu/threads.lua | 16 +++++++++++----- 10 files changed, 92 insertions(+), 23 deletions(-) diff --git a/install_manifest.json b/install_manifest.json index dc14922..bf2124b 100644 --- a/install_manifest.json +++ b/install_manifest.json @@ -1 +1 @@ -{"versions": {"installer": "v1.0", "bootloader": "0.2", "comms": "1.4.0", "reactor-plc": "v1.0.0", "rtu": "v0.13.0", "supervisor": "v0.14.2", "coordinator": "v0.12.3", "pocket": "alpha-v0.0.0"}, "files": {"system": ["initenv.lua", "startup.lua"], "common": ["scada-common/crypto.lua", "scada-common/ppm.lua", "scada-common/comms.lua", "scada-common/psil.lua", "scada-common/tcallbackdsp.lua", "scada-common/rsio.lua", "scada-common/constants.lua", "scada-common/mqueue.lua", "scada-common/crash.lua", "scada-common/log.lua", "scada-common/types.lua", "scada-common/util.lua"], "graphics": ["graphics/element.lua", "graphics/flasher.lua", "graphics/core.lua", "graphics/elements/textbox.lua", "graphics/elements/displaybox.lua", "graphics/elements/pipenet.lua", "graphics/elements/rectangle.lua", "graphics/elements/div.lua", "graphics/elements/tiling.lua", "graphics/elements/colormap.lua", "graphics/elements/indicators/alight.lua", "graphics/elements/indicators/icon.lua", "graphics/elements/indicators/power.lua", "graphics/elements/indicators/rad.lua", "graphics/elements/indicators/state.lua", "graphics/elements/indicators/light.lua", "graphics/elements/indicators/vbar.lua", "graphics/elements/indicators/coremap.lua", "graphics/elements/indicators/data.lua", "graphics/elements/indicators/hbar.lua", "graphics/elements/indicators/trilight.lua", "graphics/elements/controls/switch_button.lua", "graphics/elements/controls/spinbox_numeric.lua", "graphics/elements/controls/hazard_button.lua", "graphics/elements/controls/push_button.lua", "graphics/elements/controls/radio_button.lua", "graphics/elements/controls/multi_button.lua", "graphics/elements/animations/waiting.lua"], "lockbox": ["lockbox/init.lua", "lockbox/LICENSE", "lockbox/kdf/pbkdf2.lua", "lockbox/util/bit.lua", "lockbox/util/array.lua", "lockbox/util/stream.lua", "lockbox/util/queue.lua", "lockbox/digest/sha2_224.lua", "lockbox/digest/sha1.lua", "lockbox/digest/sha2_256.lua", "lockbox/cipher/aes128.lua", "lockbox/cipher/aes256.lua", "lockbox/cipher/aes192.lua", "lockbox/cipher/mode/ofb.lua", "lockbox/cipher/mode/cbc.lua", "lockbox/cipher/mode/ctr.lua", "lockbox/cipher/mode/cfb.lua", "lockbox/mac/hmac.lua", "lockbox/padding/ansix923.lua", "lockbox/padding/pkcs7.lua", "lockbox/padding/zero.lua", "lockbox/padding/isoiec7816.lua"], "reactor-plc": ["reactor-plc/threads.lua", "reactor-plc/plc.lua", "reactor-plc/config.lua", "reactor-plc/startup.lua"], "rtu": ["rtu/threads.lua", "rtu/rtu.lua", "rtu/modbus.lua", "rtu/config.lua", "rtu/startup.lua", "rtu/dev/sps_rtu.lua", "rtu/dev/envd_rtu.lua", "rtu/dev/boilerv_rtu.lua", "rtu/dev/redstone_rtu.lua", "rtu/dev/sna_rtu.lua", "rtu/dev/imatrix_rtu.lua", "rtu/dev/turbinev_rtu.lua"], "supervisor": ["supervisor/supervisor.lua", "supervisor/unit.lua", "supervisor/config.lua", "supervisor/startup.lua", "supervisor/unitlogic.lua", "supervisor/facility.lua", "supervisor/session/coordinator.lua", "supervisor/session/svqtypes.lua", "supervisor/session/svsessions.lua", "supervisor/session/rtu.lua", "supervisor/session/plc.lua", "supervisor/session/rsctl.lua", "supervisor/session/rtu/boilerv.lua", "supervisor/session/rtu/txnctrl.lua", "supervisor/session/rtu/unit_session.lua", "supervisor/session/rtu/turbinev.lua", "supervisor/session/rtu/envd.lua", "supervisor/session/rtu/imatrix.lua", "supervisor/session/rtu/sps.lua", "supervisor/session/rtu/qtypes.lua", "supervisor/session/rtu/sna.lua", "supervisor/session/rtu/redstone.lua"], "coordinator": ["coordinator/coordinator.lua", "coordinator/renderer.lua", "coordinator/iocontrol.lua", "coordinator/sounder.lua", "coordinator/config.lua", "coordinator/startup.lua", "coordinator/apisessions.lua", "coordinator/process.lua", "coordinator/ui/dialog.lua", "coordinator/ui/style.lua", "coordinator/ui/layout/main_view.lua", "coordinator/ui/layout/unit_view.lua", "coordinator/ui/components/reactor.lua", "coordinator/ui/components/processctl.lua", "coordinator/ui/components/unit_overview.lua", "coordinator/ui/components/boiler.lua", "coordinator/ui/components/unit_detail.lua", "coordinator/ui/components/imatrix.lua", "coordinator/ui/components/unit_waiting.lua", "coordinator/ui/components/turbine.lua"], "pocket": ["pocket/config.lua", "pocket/startup.lua"]}, "depends": {"reactor-plc": ["system", "common"], "rtu": ["system", "common"], "supervisor": ["system", "common"], "coordinator": ["system", "common", "graphics"], "pocket": ["system", "common", "graphics"]}, "sizes": {"manifest": 4646, "system": 1982, "common": 91084, "graphics": 99858, "lockbox": 100797, "reactor-plc": 75529, "rtu": 83729, "supervisor": 275020, "coordinator": 181148, "pocket": 335}} \ No newline at end of file +{"versions": {"installer": "v1.0", "bootloader": "0.2", "comms": "1.4.0", "reactor-plc": "v1.0.0", "rtu": "v0.13.1", "supervisor": "v0.14.2", "coordinator": "v0.12.3", "pocket": "alpha-v0.0.0"}, "files": {"system": ["initenv.lua", "startup.lua"], "common": ["scada-common/comms.lua", "scada-common/constants.lua", "scada-common/crash.lua", "scada-common/crypto.lua", "scada-common/log.lua", "scada-common/mqueue.lua", "scada-common/ppm.lua", "scada-common/psil.lua", "scada-common/rsio.lua", "scada-common/tcallbackdsp.lua", "scada-common/types.lua", "scada-common/util.lua"], "graphics": ["graphics/core.lua", "graphics/element.lua", "graphics/flasher.lua", "graphics/elements/colormap.lua", "graphics/elements/displaybox.lua", "graphics/elements/div.lua", "graphics/elements/pipenet.lua", "graphics/elements/rectangle.lua", "graphics/elements/textbox.lua", "graphics/elements/tiling.lua", "graphics/elements/animations/waiting.lua", "graphics/elements/controls/hazard_button.lua", "graphics/elements/controls/multi_button.lua", "graphics/elements/controls/push_button.lua", "graphics/elements/controls/radio_button.lua", "graphics/elements/controls/spinbox_numeric.lua", "graphics/elements/controls/switch_button.lua", "graphics/elements/indicators/alight.lua", "graphics/elements/indicators/coremap.lua", "graphics/elements/indicators/data.lua", "graphics/elements/indicators/hbar.lua", "graphics/elements/indicators/icon.lua", "graphics/elements/indicators/light.lua", "graphics/elements/indicators/power.lua", "graphics/elements/indicators/rad.lua", "graphics/elements/indicators/state.lua", "graphics/elements/indicators/trilight.lua", "graphics/elements/indicators/vbar.lua"], "lockbox": ["lockbox/init.lua", "lockbox/LICENSE", "lockbox/cipher/aes128.lua", "lockbox/cipher/aes192.lua", "lockbox/cipher/aes256.lua", "lockbox/cipher/mode/cbc.lua", "lockbox/cipher/mode/cfb.lua", "lockbox/cipher/mode/ctr.lua", "lockbox/cipher/mode/ofb.lua", "lockbox/digest/sha1.lua", "lockbox/digest/sha2_224.lua", "lockbox/digest/sha2_256.lua", "lockbox/kdf/pbkdf2.lua", "lockbox/mac/hmac.lua", "lockbox/padding/ansix923.lua", "lockbox/padding/isoiec7816.lua", "lockbox/padding/pkcs7.lua", "lockbox/padding/zero.lua", "lockbox/util/array.lua", "lockbox/util/bit.lua", "lockbox/util/queue.lua", "lockbox/util/stream.lua"], "reactor-plc": ["reactor-plc/config.lua", "reactor-plc/plc.lua", "reactor-plc/startup.lua", "reactor-plc/threads.lua"], "rtu": ["rtu/config.lua", "rtu/modbus.lua", "rtu/rtu.lua", "rtu/startup.lua", "rtu/threads.lua", "rtu/dev/boilerv_rtu.lua", "rtu/dev/envd_rtu.lua", "rtu/dev/imatrix_rtu.lua", "rtu/dev/redstone_rtu.lua", "rtu/dev/sna_rtu.lua", "rtu/dev/sps_rtu.lua", "rtu/dev/turbinev_rtu.lua"], "supervisor": ["supervisor/config.lua", "supervisor/facility.lua", "supervisor/startup.lua", "supervisor/supervisor.lua", "supervisor/unit.lua", "supervisor/unitlogic.lua", "supervisor/session/coordinator.lua", "supervisor/session/plc.lua", "supervisor/session/rsctl.lua", "supervisor/session/rtu.lua", "supervisor/session/svqtypes.lua", "supervisor/session/svsessions.lua", "supervisor/session/rtu/boilerv.lua", "supervisor/session/rtu/envd.lua", "supervisor/session/rtu/imatrix.lua", "supervisor/session/rtu/qtypes.lua", "supervisor/session/rtu/redstone.lua", "supervisor/session/rtu/sna.lua", "supervisor/session/rtu/sps.lua", "supervisor/session/rtu/turbinev.lua", "supervisor/session/rtu/txnctrl.lua", "supervisor/session/rtu/unit_session.lua"], "coordinator": ["coordinator/apisessions.lua", "coordinator/config.lua", "coordinator/coordinator.lua", "coordinator/iocontrol.lua", "coordinator/process.lua", "coordinator/renderer.lua", "coordinator/sounder.lua", "coordinator/startup.lua", "coordinator/ui/dialog.lua", "coordinator/ui/style.lua", "coordinator/ui/components/boiler.lua", "coordinator/ui/components/imatrix.lua", "coordinator/ui/components/processctl.lua", "coordinator/ui/components/reactor.lua", "coordinator/ui/components/turbine.lua", "coordinator/ui/components/unit_detail.lua", "coordinator/ui/components/unit_overview.lua", "coordinator/ui/components/unit_waiting.lua", "coordinator/ui/layout/main_view.lua", "coordinator/ui/layout/unit_view.lua"], "pocket": ["pocket/config.lua", "pocket/startup.lua"]}, "depends": {"reactor-plc": ["system", "common"], "rtu": ["system", "common"], "supervisor": ["system", "common"], "coordinator": ["system", "common", "graphics"], "pocket": ["system", "common", "graphics"]}, "sizes": {"manifest": 4647, "system": 2050, "common": 94425, "graphics": 103176, "lockbox": 104193, "reactor-plc": 77488, "rtu": 88644, "supervisor": 282356, "coordinator": 185569, "pocket": 351}} \ No newline at end of file diff --git a/rtu/dev/boilerv_rtu.lua b/rtu/dev/boilerv_rtu.lua index b93d412..46ac7c2 100644 --- a/rtu/dev/boilerv_rtu.lua +++ b/rtu/dev/boilerv_rtu.lua @@ -5,9 +5,13 @@ local boilerv_rtu = {} -- create new boiler (mek 10.1+) device ---@nodiscard ---@param boiler table +---@return rtu_device interface, boolean faulted function boilerv_rtu.new(boiler) local unit = rtu.init_unit() + -- disable auto fault clearing + boiler.__p_disable_afc() + -- discrete inputs -- unit.connect_di(boiler.isFormed) @@ -50,7 +54,12 @@ function boilerv_rtu.new(boiler) -- holding registers -- -- none - return unit.interface() + -- check if any calls faulted + local faulted = boiler.__p_is_faulted() + boiler.__p_clear_fault() + boiler.__p_enable_afc() + + return unit.interface(), faulted end return boilerv_rtu diff --git a/rtu/dev/envd_rtu.lua b/rtu/dev/envd_rtu.lua index ba4758a..2894e2c 100644 --- a/rtu/dev/envd_rtu.lua +++ b/rtu/dev/envd_rtu.lua @@ -5,9 +5,13 @@ local envd_rtu = {} -- create new environment detector device ---@nodiscard ---@param envd table +---@return rtu_device interface, boolean faulted function envd_rtu.new(envd) local unit = rtu.init_unit() + -- disable auto fault clearing + envd.__p_disable_afc() + -- discrete inputs -- -- none @@ -21,7 +25,12 @@ function envd_rtu.new(envd) -- holding registers -- -- none - return unit.interface() + -- check if any calls faulted + local faulted = envd.__p_is_faulted() + envd.__p_clear_fault() + envd.__p_enable_afc() + + return unit.interface(), faulted end return envd_rtu diff --git a/rtu/dev/imatrix_rtu.lua b/rtu/dev/imatrix_rtu.lua index 29405b8..3b72a12 100644 --- a/rtu/dev/imatrix_rtu.lua +++ b/rtu/dev/imatrix_rtu.lua @@ -5,9 +5,13 @@ local imatrix_rtu = {} -- create new induction matrix (mek 10.1+) device ---@nodiscard ---@param imatrix table +---@return rtu_device interface, boolean faulted function imatrix_rtu.new(imatrix) local unit = rtu.init_unit() + -- disable auto fault clearing + imatrix.__p_disable_afc() + -- discrete inputs -- unit.connect_di(imatrix.isFormed) @@ -37,7 +41,12 @@ function imatrix_rtu.new(imatrix) -- holding registers -- -- none - return unit.interface() + -- check if any calls faulted + local faulted = imatrix.__p_is_faulted() + imatrix.__p_clear_fault() + imatrix.__p_enable_afc() + + return unit.interface(), faulted end return imatrix_rtu diff --git a/rtu/dev/redstone_rtu.lua b/rtu/dev/redstone_rtu.lua index da7db6b..c482999 100644 --- a/rtu/dev/redstone_rtu.lua +++ b/rtu/dev/redstone_rtu.lua @@ -11,6 +11,7 @@ local digital_write = rsio.digital_write -- create new redstone device ---@nodiscard +---@return rtu_rs_device interface, boolean faulted function redstone_rtu.new() local unit = rtu.init_unit() @@ -111,7 +112,7 @@ function redstone_rtu.new() ) end - return public + return public, false end return redstone_rtu diff --git a/rtu/dev/sna_rtu.lua b/rtu/dev/sna_rtu.lua index 0339794..16c0cfd 100644 --- a/rtu/dev/sna_rtu.lua +++ b/rtu/dev/sna_rtu.lua @@ -5,9 +5,13 @@ local sna_rtu = {} -- create new solar neutron activator (SNA) device ---@nodiscard ---@param sna table +---@return rtu_device interface, boolean faulted function sna_rtu.new(sna) local unit = rtu.init_unit() + -- disable auto fault clearing + sna.__p_disable_afc() + -- discrete inputs -- -- none @@ -32,7 +36,12 @@ function sna_rtu.new(sna) -- holding registers -- -- none - return unit.interface() + -- check if any calls faulted + local faulted = sna.__p_is_faulted() + sna.__p_clear_fault() + sna.__p_enable_afc() + + return unit.interface(), faulted end return sna_rtu diff --git a/rtu/dev/sps_rtu.lua b/rtu/dev/sps_rtu.lua index ba0a18c..349550c 100644 --- a/rtu/dev/sps_rtu.lua +++ b/rtu/dev/sps_rtu.lua @@ -5,9 +5,13 @@ local sps_rtu = {} -- create new super-critical phase shifter (SPS) device ---@nodiscard ---@param sps table +---@return rtu_device interface, boolean faulted function sps_rtu.new(sps) local unit = rtu.init_unit() + -- disable auto fault clearing + sps.__p_disable_afc() + -- discrete inputs -- unit.connect_di(sps.isFormed) @@ -42,7 +46,12 @@ function sps_rtu.new(sps) -- holding registers -- -- none - return unit.interface() + -- check if any calls faulted + local faulted = sps.__p_is_faulted() + sps.__p_clear_fault() + sps.__p_enable_afc() + + return unit.interface(), faulted end return sps_rtu diff --git a/rtu/dev/turbinev_rtu.lua b/rtu/dev/turbinev_rtu.lua index 89b3ae0..4f2ee48 100644 --- a/rtu/dev/turbinev_rtu.lua +++ b/rtu/dev/turbinev_rtu.lua @@ -5,9 +5,13 @@ local turbinev_rtu = {} -- create new turbine (mek 10.1+) device ---@nodiscard ---@param turbine table +---@return rtu_device interface, boolean faulted function turbinev_rtu.new(turbine) local unit = rtu.init_unit() + -- disable auto fault clearing + turbine.__p_disable_afc() + -- discrete inputs -- unit.connect_di(turbine.isFormed) @@ -49,7 +53,12 @@ function turbinev_rtu.new(turbine) -- holding registers -- unit.connect_holding_reg(turbine.getDumpingMode, turbine.setDumpingMode) - return unit.interface() + -- check if any calls faulted + local faulted = turbine.__p_is_faulted() + turbine.__p_clear_fault() + turbine.__p_enable_afc() + + return unit.interface(), faulted end return turbinev_rtu diff --git a/rtu/startup.lua b/rtu/startup.lua index df79496..2f780cc 100644 --- a/rtu/startup.lua +++ b/rtu/startup.lua @@ -25,7 +25,7 @@ local sna_rtu = require("rtu.dev.sna_rtu") local sps_rtu = require("rtu.dev.sps_rtu") local turbinev_rtu = require("rtu.dev.turbinev_rtu") -local RTU_VERSION = "v0.13.0" +local RTU_VERSION = "v0.13.1" local RTU_UNIT_TYPE = types.RTU_UNIT_TYPE @@ -290,8 +290,9 @@ local function main() local type = nil ---@type string|nil local rtu_iface = nil ---@type rtu_device local rtu_type = nil ---@type RTU_UNIT_TYPE - local is_multiblock = false + local is_multiblock = false ---@type boolean local formed = nil ---@type boolean|nil + local faulted = nil ---@type boolean|nil if device == nil then local message = util.c("configure> '", name, "' not found, using placeholder") @@ -307,7 +308,7 @@ local function main() if type == "boilerValve" then -- boiler multiblock rtu_type = RTU_UNIT_TYPE.BOILER_VALVE - rtu_iface = boilerv_rtu.new(device) + rtu_iface, faulted = boilerv_rtu.new(device) is_multiblock = true formed = device.isFormed() @@ -319,7 +320,7 @@ local function main() elseif type == "turbineValve" then -- turbine multiblock rtu_type = RTU_UNIT_TYPE.TURBINE_VALVE - rtu_iface = turbinev_rtu.new(device) + rtu_iface, faulted = turbinev_rtu.new(device) is_multiblock = true formed = device.isFormed() @@ -331,7 +332,7 @@ local function main() elseif type == "inductionPort" then -- induction matrix multiblock rtu_type = RTU_UNIT_TYPE.IMATRIX - rtu_iface = imatrix_rtu.new(device) + rtu_iface, faulted = imatrix_rtu.new(device) is_multiblock = true formed = device.isFormed() @@ -343,7 +344,7 @@ local function main() elseif type == "spsPort" then -- SPS multiblock rtu_type = RTU_UNIT_TYPE.SPS - rtu_iface = sps_rtu.new(device) + rtu_iface, faulted = sps_rtu.new(device) is_multiblock = true formed = device.isFormed() @@ -355,11 +356,11 @@ local function main() elseif type == "solarNeutronActivator" then -- SNA rtu_type = RTU_UNIT_TYPE.SNA - rtu_iface = sna_rtu.new(device) + rtu_iface, _ = sna_rtu.new(device) elseif type == "environmentDetector" then -- advanced peripherals environment detector rtu_type = RTU_UNIT_TYPE.ENV_DETECTOR - rtu_iface = envd_rtu.new(device) + rtu_iface, _ = envd_rtu.new(device) elseif type == ppm.VIRTUAL_DEVICE_TYPE then -- placeholder device rtu_type = RTU_UNIT_TYPE.VIRTUAL @@ -391,8 +392,15 @@ local function main() table.insert(units, rtu_unit) - if is_multiblock and not formed then - log.info(util.c("configure> device '", name, "' is not formed")) + if is_multiblock then + if not formed then + log.info(util.c("configure> device '", name, "' is not formed")) + elseif faulted then + -- sometimes there is a race condition on server boot where it reports formed, but + -- the other functions are not yet defined (that's the theory at least). mark as unformed to attempt connection later + formed = false + log.warning(util.c("configure> device '", name, "' is formed, but initialization had one or more faults: marked as unformed")) + end end local for_message = "facility" diff --git a/rtu/threads.lua b/rtu/threads.lua index 6b06eb0..27ad68e 100644 --- a/rtu/threads.lua +++ b/rtu/threads.lua @@ -375,37 +375,43 @@ function threads.thread__unit_comms(smem, unit) ppm.unmount(unit.device) local type, device = ppm.mount(iface) + local faulted = false if device ~= nil then if type == "boilerValve" and unit.type == RTU_UNIT_TYPE.BOILER_VALVE then -- boiler multiblock unit.device = device - unit.rtu = boilerv_rtu.new(device) + unit.rtu, faulted = boilerv_rtu.new(device) unit.formed = device.isFormed() unit.modbus_io = modbus.new(unit.rtu, true) elseif type == "turbineValve" and unit.type == RTU_UNIT_TYPE.TURBINE_VALVE then -- turbine multiblock unit.device = device - unit.rtu = turbinev_rtu.new(device) + unit.rtu, faulted = turbinev_rtu.new(device) unit.formed = device.isFormed() unit.modbus_io = modbus.new(unit.rtu, true) elseif type == "inductionPort" and unit.type == RTU_UNIT_TYPE.IMATRIX then -- induction matrix multiblock unit.device = device - unit.rtu = imatrix_rtu.new(device) + unit.rtu, faulted = imatrix_rtu.new(device) unit.formed = device.isFormed() unit.modbus_io = modbus.new(unit.rtu, true) elseif type == "spsPort" and unit.type == RTU_UNIT_TYPE.SPS then -- SPS multiblock unit.device = device - unit.rtu = sps_rtu.new(device) + unit.rtu, faulted = sps_rtu.new(device) unit.formed = device.isFormed() unit.modbus_io = modbus.new(unit.rtu, true) else log.error("illegal remount of non-multiblock RTU attempted for " .. short_name, true) end - rtu_comms.send_remounted(unit.uid) + if unit.formed and faulted then + -- something is still wrong = can't mark as formed yet + unit.formed = false + else + rtu_comms.send_remounted(unit.uid) + end else -- fully lost the peripheral now :( log.error(util.c(unit.name, " lost (failed reconnect)"))