From 02763c9cb312c0104446e8605b55cbb004addb71 Mon Sep 17 00:00:00 2001 From: Mikayla Fischler Date: Sun, 3 Apr 2022 12:08:22 -0400 Subject: [PATCH] #4 PLC peripheral disconnect handling and small bugfixes/cleanup --- reactor-plc/plc.lua | 31 ++++++- reactor-plc/startup.lua | 193 ++++++++++++++++++++++++++++------------ scada-common/log.lua | 4 + scada-common/ppm.lua | 2 +- 4 files changed, 170 insertions(+), 60 deletions(-) diff --git a/reactor-plc/plc.lua b/reactor-plc/plc.lua index 5a5ce4a..9a168bb 100644 --- a/reactor-plc/plc.lua +++ b/reactor-plc/plc.lua @@ -11,6 +11,10 @@ function iss_init(reactor) trip_cause = "" } + local reconnect_reactor = function (reactor) + self.reactor = reactor + end + local check = function () local status = "ok" local was_tripped = self.tripped @@ -110,6 +114,7 @@ function iss_init(reactor) end return { + reconnect_reactor = reconnect_reactor, check = check, trip_timeout = trip_timeout, reset = reset, @@ -197,7 +202,7 @@ function rplc_packet() end -- reactor PLC communications -function rplc_comms(id, modem, local_port, server_port, reactor, iss) +function comms_init(id, modem, local_port, server_port, reactor, iss) local self = { id = id, seq_num = 0, @@ -211,6 +216,11 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss) linked = false } + -- open modem + if not self.modem.isOpen(self.l_port) then + self.modem.open(self.l_port) + end + -- PRIVATE FUNCTIONS -- local _send = function (msg) @@ -323,6 +333,21 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss) -- PUBLIC FUNCTIONS -- + -- reconnect a newly connected modem + local reconnect_modem = function (modem) + self.modem = modem + + -- open modem + if not self.modem.isOpen(self.l_port) then + self.modem.open(self.l_port) + end + end + + -- reconnect a newly connected reactor + local reconnect_reactor = function (reactor) + self.reactor = reactor + end + -- parse an RPLC packet local parse_packet = function(side, sender, reply_to, message, distance) local pkt = nil @@ -410,7 +435,7 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss) local max_burn_rate = self.reactor.getMaxBurnRate() local success = false - if max_burn_rate is not nil then + if max_burn_rate ~= nil then if burn_rate > 0 and burn_rate <= max_burn_rate then success = self.reactor.setBurnRate(burn_rate) end @@ -508,6 +533,8 @@ function rplc_comms(id, modem, local_port, server_port, reactor, iss) local unlink = function () self.linked = false end return { + reconnect_modem = reconnect_modem, + reconnect_reactor = reconnect_reactor, parse_packet = parse_packet, handle_packet = handle_packet, send_link_req = send_link_req, diff --git a/reactor-plc/startup.lua b/reactor-plc/startup.lua index a85d11f..29522e1 100644 --- a/reactor-plc/startup.lua +++ b/reactor-plc/startup.lua @@ -22,35 +22,50 @@ ppm.mount_all() local reactor = ppm.get_device("fissionReactor") local modem = ppm.get_device("modem") +local init_ok = true +local control_degraded = { degraded = false, no_reactor = false, no_modem = false } + -- we need a reactor and a modem if reactor == nil then - print("Fission reactor not found, exiting..."); - return -elseif modem == nil then - print("No modem found, disabling reactor and exiting...") + print_ts("Fission reactor not found. Running in a degraded state...\n"); + log._warning("no reactor on startup") + init_ok = false + control_degraded.degraded = true + control_degraded.no_reactor = true +end +if modem == nil then + print_ts("No modem found. Disabling reactor and running in a degraded state...\n") + log._warning("no modem on startup") reactor.scram() - return + init_ok = false + control_degraded.degraded = true + control_degraded.no_modem = true end --- just booting up, no fission allowed (neutrons stay put thanks) -reactor.scram() +::init:: +if init_ok then + -- just booting up, no fission allowed (neutrons stay put thanks) + reactor.scram() --- init internal safety system -local iss = plc.iss_init(reactor) + -- init internal safety system + local iss = plc.iss_init(reactor) + log._debug("iss init") --- start comms -if not modem.isOpen(config.LISTEN_PORT) then - modem.open(config.LISTEN_PORT) + -- start comms + local plc_comms = plc.comms_init(config.REACTOR_ID, modem, config.LISTEN_PORT, config.SERVER_PORT, reactor, iss) + log._debug("comms init") + + -- comms watchdog, 3 second timeout + local conn_watchdog = watchdog.new_watchdog(3) + log._debug("conn watchdog started") + + -- loop clock (10Hz, 2 ticks) + local loop_tick = os.startTimer(0.05) + log._debug("loop clock started") +else + log._warning("booted in a degraded state, awaiting peripheral connections...") end -local plc_comms = plc.rplc_comms(config.REACTOR_ID, modem, config.LISTEN_PORT, config.SERVER_PORT, reactor, iss) - --- comms watchdog, 3 second timeout -local conn_watchdog = watchdog.new_watchdog(3) - --- loop clock (10Hz, 2 ticks) -local loop_tick = os.startTimer(0.05) - -- send status updates at ~3.33Hz (every 6 server ticks) (every 3 loop ticks) -- send link requests at 0.5Hz (every 40 server ticks) (every 20 loop ticks) local UPDATE_TICKS = 3 @@ -67,51 +82,112 @@ local reactor_scram = true -- treated as latching e-stop while true do local event, param1, param2, param3, param4, param5 = os.pullEventRaw() - -- if we tried to SCRAM but failed, keep trying - -- if it disconnected, isPowered will return nil (and error logs will get spammed at 10Hz, so disable reporting) - -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check) - ppm.disable_reporting() - if reactor_scram and reactor.isPowered() then - reactor.scram() + if init_ok then + -- if we tried to SCRAM but failed, keep trying + -- if it disconnected, isPowered will return nil (and error logs will get spammed at 10Hz, so disable reporting) + -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check) + ppm.disable_reporting() + if reactor_scram and reactor.isPowered() then + reactor.scram() + end + ppm.enable_reporting() end - ppm.enable_reporting() + -- check for peripheral changes before ISS checks if event == "peripheral_detach" then - ppm.handle_unmount(param1) + local device = ppm.handle_unmount(param1) - -- try to scram reactor if it is still connected - reactor_scram = true - if reactor.scram() then - print_ts("[fatal] PLC lost a peripheral: successful SCRAM\n") - else - print_ts("[fatal] PLC lost a peripheral: failed SCRAM\n") + if device.type == "fissionReactor" then + print_ts("reactor disconnected!\n") + log._error("reactor disconnected!") + control_degraded.no_reactor = true + -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ? + elseif device.type == "modem" then + print_ts("modem disconnected!\n") + log._error("modem disconnected!") + control_degraded.no_modem = true + + if init_ok then + -- try to scram reactor if it is still connected + reactor_scram = true + if reactor.scram() then + print_ts("successful reactor SCRAM\n") + else + print_ts("failed reactor SCRAM\n") + end + end + + control_degraded.degraded = true + end + elseif event == "peripheral" then + local device = ppm.mount(param1) + + if device.type == "fissionReactor" then + -- reconnected reactor + reactor_scram = true + device.scram() + + print_ts("reactor reconnected.\n") + log._info("reactor reconnected.") + control_degraded.no_reactor = false + + if init_ok then + iss.reconnect_reactor(device) + plc_comms.reconnect_reactor(device) + end + + -- determine if we are still in a degraded state + if get_device("modem") not nil then + control_degraded.degraded = false + end + elseif device.type == "modem" then + -- reconnected modem + if init_ok then + plc_comms.reconnect_modem(device) + end + + print_ts("modem reconnected.\n") + log._info("modem reconnected.") + control_degraded.no_modem = false + + -- determine if we are still in a degraded state + if ppm.get_device("fissionReactor") not nil then + control_degraded.degraded = false + end end - -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_PERI_DC) ? + if not init_ok and not control_degraded.degraded then + init_ok = false + goto init + end end -- check safety (SCRAM occurs if tripped) - local iss_tripped, iss_status, iss_first = iss.check() - reactor_scram = reactor_scram or iss_tripped - if iss_first then - plc_comms.send_iss_alarm(iss_status) + if not control_degraded.degraded then + local iss_tripped, iss_status, iss_first = iss.check() + reactor_scram = reactor_scram or iss_tripped + if iss_first then + plc_comms.send_iss_alarm(iss_status) + end end -- handle event if event == "timer" and param1 == loop_tick then - -- basic event tick, send updated data if it is time (~3.33Hz) - -- iss was already checked (main reason for this tick rate) - ticks_to_update = ticks_to_update - 1 + if not control_degraded.no_modem then + -- basic event tick, send updated data if it is time (~3.33Hz) + -- iss was already checked (main reason for this tick rate) + ticks_to_update = ticks_to_update - 1 - if plc_comms.is_linked() then - if ticks_to_update <= 0 then - plc_comms.send_status(control_state, iss_tripped) - ticks_to_update = UPDATE_TICKS - end - else - if ticks_to_update <= 0 then - plc_comms.send_link_req() - ticks_to_update = LINK_TICKS + if plc_comms.is_linked() then + if ticks_to_update <= 0 then + plc_comms.send_status(control_state, iss_tripped) + ticks_to_update = UPDATE_TICKS + end + else + if ticks_to_update <= 0 then + plc_comms.send_link_req() + ticks_to_update = LINK_TICKS + end end end elseif event == "modem_message" then @@ -130,14 +206,17 @@ while true do print_ts("[alert] server timeout, reactor disabled\n") elseif event == "terminate" then -- safe exit - reactor_scram = true - if reactor.scram() then - print_ts("[alert] exiting, reactor disabled\n") - else - -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ? - print_ts("[alert] exiting, reactor failed to disable\n") + if init_ok then + reactor_scram = true + if reactor.scram() then + print_ts("[alert] exiting, reactor disabled\n") + else + -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_LOST_CONTROL) ? + print_ts("[alert] exiting, reactor failed to disable\n") + end end -- send an alarm: plc_comms.send_alarm(ALARMS.PLC_SHUTDOWN) ? + print_ts("[alert] exited") return end end diff --git a/scada-common/log.lua b/scada-common/log.lua index 3a56919..79296c6 100644 --- a/scada-common/log.lua +++ b/scada-common/log.lua @@ -34,6 +34,10 @@ function _debug(msg, trace) end end +function _info(msg) + _log("[INF] " .. msg .. "\n") +end + function _warning(msg) _log("[WRN] " .. msg .. "\n") end diff --git a/scada-common/ppm.lua b/scada-common/ppm.lua index 6f33117..9924007 100644 --- a/scada-common/ppm.lua +++ b/scada-common/ppm.lua @@ -90,7 +90,7 @@ function handle_unmount(iface) log._warning("PPM: lost device " .. type .. " mounted to " .. iface) - return self.mounts[iface] + return lost_dev end -- list all available peripherals