#44 RPS optimizations, manual trip, RPS handles all reactor state control

reactor-plc/plc.lua

@@ -22,49 +22,48 @@ local println_ts = util.println_ts
 -- identifies dangerous states and SCRAMs reactor if warranted
 -- autonomous from main SCADA supervisor/coordinator control
 plc.rps_init = function (reactor)
+    local state_keys = {
+        dmg_crit = 1,
+        high_temp = 2,
+        no_coolant = 3,
+        ex_waste = 4,
+        ex_hcoolant = 5,
+        no_fuel = 6,
+        fault = 7,
+        timeout = 8,
+        manual = 9
+    }
+
     local self = {
         reactor = reactor,
-        cache = { false, false, false, false, false, false, false },
+        state = { false, false, false, false, false, false, false, false, false },
-        timed_out = false,
+        reactor_enabled = false,
         tripped = false,
         trip_cause = ""
     }
 
     -- PRIVATE FUNCTIONS --
 
+    -- set reactor access fault flag
+    local _set_fault = function ()
+        self.state[state_keys.fault] = true
+    end
+
+    -- clear reactor access fault flag
+    local _clear_fault = function ()
+        self.state[state_keys.fault] = false
+    end
+
     -- check for critical damage
     local _damage_critical = function ()
         local damage_percent = self.reactor.getDamagePercent()
         if damage_percent == ppm.ACCESS_FAULT then
             -- lost the peripheral or terminated, handled later
             log.error("RPS: failed to check reactor damage")
-            return false
+            _set_fault()
+            self.state[state_keys.dmg_crit] = false
         else
-            return damage_percent >= 100
+            self.state[state_keys.dmg_crit] = damage_percent >= 100
-        end
-    end
-
-    -- check for heated coolant backup
-    local _excess_heated_coolant = function ()
-        local hc_needed = self.reactor.getHeatedCoolantNeeded()
-        if hc_needed == ppm.ACCESS_FAULT then
-            -- lost the peripheral or terminated, handled later
-            log.error("RPS: failed to check reactor heated coolant level")
-            return false
-        else
-            return hc_needed == 0
-        end
-    end
-
-    -- check for excess waste
-    local _excess_waste = function ()
-        local w_needed = self.reactor.getWasteNeeded()
-        if w_needed == ppm.ACCESS_FAULT then
-            -- lost the peripheral or terminated, handled later
-            log.error("RPS: failed to check reactor waste level")
-            return false
-        else
-            return w_needed == 0
         end
     end
 
@@ -75,9 +74,49 @@ plc.rps_init = function (reactor)
         if temp == ppm.ACCESS_FAULT then
             -- lost the peripheral or terminated, handled later
             log.error("RPS: failed to check reactor temperature")
-            return false
+            _set_fault()
+            self.state[state_keys.high_temp] = false
         else
-            return temp >= 1200
+            self.state[state_keys.high_temp] = temp >= 1200
+        end
+    end
+
+    -- check if there is no coolant (<2% filled)
+    local _no_coolant = function ()
+        local coolant_filled = self.reactor.getCoolantFilledPercentage()
+        if coolant_filled == ppm.ACCESS_FAULT then
+            -- lost the peripheral or terminated, handled later
+            log.error("RPS: failed to check reactor coolant level")
+            _set_fault()
+            self.state[state_keys.no_coolant] = false
+        else
+            self.state[state_keys.no_coolant] = coolant_filled < 0.02
+        end
+    end
+
+    -- check for excess waste (>80% filled)
+    local _excess_waste = function ()
+        local w_filled = self.reactor.getWasteFilledPercentage()
+        if w_filled == ppm.ACCESS_FAULT then
+            -- lost the peripheral or terminated, handled later
+            log.error("RPS: failed to check reactor waste level")
+            _set_fault()
+            self.state[state_keys.ex_waste] = false
+        else
+            self.state[state_keys.ex_waste] = w_filled > 0.8
+        end
+    end
+
+    -- check for heated coolant backup (>95% filled)
+    local _excess_heated_coolant = function ()
+        local hc_filled = self.reactor.getHeatedCoolantFilledPercentage()
+        if hc_filled == ppm.ACCESS_FAULT then
+            -- lost the peripheral or terminated, handled later
+            log.error("RPS: failed to check reactor heated coolant level")
+            _set_fault()
+            state[state_keys.ex_hcoolant] = false
+        else
+            state[state_keys.ex_hcoolant] = hc_filled > 0.95
         end
     end
 
@@ -86,22 +125,11 @@ plc.rps_init = function (reactor)
         local fuel = self.reactor.getFuel()
         if fuel == ppm.ACCESS_FAULT then
             -- lost the peripheral or terminated, handled later
-            log.error("RPS: failed to check reactor fuel level")
+            log.error("RPS: failed to check reactor fuel")
-            return false
+            _set_fault()
+            state[state_keys.no_fuel] = false
         else
-            return fuel == 0
+            state[state_keys.no_fuel] = fuel.amount == 0
-        end
-    end
-
-    -- check if there is no coolant
-    local _no_coolant = function ()
-        local coolant_filled = self.reactor.getCoolantFilledPercentage()
-        if coolant_filled == ppm.ACCESS_FAULT then
-            -- lost the peripheral or terminated, handled later
-            log.error("RPS: failed to check reactor coolant level")
-            return false
-        else
-            return coolant_filled < 0.02
         end
     end
 
@@ -114,78 +142,114 @@ plc.rps_init = function (reactor)
 
     -- report a PLC comms timeout
     local trip_timeout = function ()
-        self.timed_out = true
+        state[state_keys.timed_out] = true
+    end
+
+    -- manually SCRAM the reactor
+    local trip_manual = function ()
+        state[state_keys.manual]  = true
+    end
+
+    -- SCRAM the reactor now
+    local scram = function ()
+        log.info("RPS: reactor SCRAM")
+
+        self.reactor.scram()
+        if self.reactor.__p_is_faulted() then
+            log.error("RPS: failed reactor SCRAM")
+            return false
+        else
+            self.reactor_enabled = false
+            return true
+        end
+    end
+
+    -- start the reactor
+    local activate = function ()
+        if not self.tripped then
+            log.info("RPS: reactor start")
+
+            self.reactor.activate()
+            if self.reactor.__p_is_faulted() then
+                log.error("RPS: failed reactor start")
+            else
+                self.reactor_enabled = true
+                return true
+            end
+        end
+
+        return false
     end
 
     -- check all safety conditions
     local check = function ()
         local status = rps_status_t.ok
         local was_tripped = self.tripped
+        local first_trip = false
 
-        -- update cache
+        -- update state
-        self.cache = {
+        parallel.waitForAll(
-            _damage_critical(),
+            _damage_critical,
-            _excess_heated_coolant(),
+            _high_temp,
-            _excess_waste(),
+            _no_coolant,
-            _high_temp(),
+            _excess_waste,
-            _insufficient_fuel(),
+            _excess_heated_coolant,
-            _no_coolant(),
+            _insufficient_fuel
-            self.timed_out
+        )
-        }
 
         -- check system states in order of severity
         if self.tripped then
             status = self.trip_cause
-        elseif self.cache[1] then
+        elseif self.state[state_keys.dmg_crit] then
-            log.warning("RPS: damage critical!")
+            log.warning("RPS: damage critical")
             status = rps_status_t.dmg_crit
-        elseif self.cache[4] then
+        elseif self.state[state_keys.high_temp] then
-            log.warning("RPS: high temperature!")
+            log.warning("RPS: high temperature")
             status = rps_status_t.high_temp
-        elseif self.cache[2] then
+        elseif self.state[state_keys.no_coolant] then
-            log.warning("RPS: heated coolant backup!")
+            log.warning("RPS: no coolant")
-            status = rps_status_t.ex_hcoolant
-        elseif self.cache[6] then
-            log.warning("RPS: no coolant!")
             status = rps_status_t.no_coolant
-        elseif self.cache[3] then
+        elseif self.state[state_keys.ex_waste] then
-            log.warning("RPS: full waste!")
+            log.warning("RPS: full waste")
             status = rps_status_t.ex_waste
-        elseif self.cache[5] then
+        elseif self.state[state_keys.ex_hcoolant] then
-            log.warning("RPS: no fuel!")
+            log.warning("RPS: heated coolant backup")
+            status = rps_status_t.ex_hcoolant
+        elseif self.state[state_keys.no_fuel] then
+            log.warning("RPS: no fuel")
             status = rps_status_t.no_fuel
-        elseif self.cache[7] then
+        elseif self.state[state_keys.fault] then
-            log.warning("RPS: supervisor connection timeout!")
+            log.warning("RPS: reactor access fault")
+            status = rps_status_t.fault
+        elseif self.state[state_keys.timeout] then
+            log.warning("RPS: supervisor connection timeout")
             status = rps_status_t.timeout
+        elseif self.state[state_keys.manual] then
+            log.warning("RPS: manual SCRAM requested")
+            status = rps_status_t.manual
         else
             self.tripped = false
         end
 
         -- if a new trip occured...
-        local first_trip = false
+        if (not was_tripped) and (status ~= rps_status_t.ok) then
-        if not was_tripped and status ~= rps_status_t.ok then
-            log.warning("RPS: reactor SCRAM")
-
             first_trip = true
             self.tripped = true
             self.trip_cause = status
 
-            self.reactor.scram()
+            scram()
-            if self.reactor.__p_is_faulted() then
-                log.error("RPS: failed reactor SCRAM")
-            end
         end
 
         return self.tripped, status, first_trip
     end
 
     -- get the RPS status
-    local status = function () return self.cache end
+    local status = function () return self.state end
     local is_tripped = function () return self.tripped end
+    local is_active = function () return self.reactor_enabled end
 
     -- reset the RPS
     local reset = function ()
-        self.timed_out = false
         self.tripped = false
         self.trip_cause = rps_status_t.ok
     end
@@ -193,9 +257,13 @@ plc.rps_init = function (reactor)
     return {
         reconnect_reactor = reconnect_reactor,
         trip_timeout = trip_timeout,
+        trip_manual = trip_manual,
+        scram = scram,
+        activate = activate,
         check = check,
         status = status,
         is_tripped = is_tripped,
+        is_active = is_active,
         reset = reset
     }
 end
@@ -544,18 +612,6 @@ plc.comms = function (id, modem, local_port, server_port, reactor, rps)
                         -- request for physical structure
                         _send_struct()
                         log.debug("sent out structure again, did supervisor miss it?")
-                    elseif packet.type == RPLC_TYPES.MEK_SCRAM then
-                        -- disable the reactor
-                        self.scrammed = true
-                        plc_state.scram = true
-                        self.reactor.scram()
-                        _send_ack(packet.type, self.reactor.__p_is_ok())
-                    elseif packet.type == RPLC_TYPES.MEK_ENABLE then
-                        -- enable the reactor
-                        self.scrammed = false
-                        plc_state.scram = false
-                        self.reactor.activate()
-                        _send_ack(packet.type, self.reactor.__p_is_ok())
                     elseif packet.type == RPLC_TYPES.MEK_BURN_RATE then
                         -- set the burn rate
                         if packet.length == 1 then
@@ -581,6 +637,15 @@ plc.comms = function (id, modem, local_port, server_port, reactor, rps)
                         else
                             log.debug("RPLC set burn rate packet length mismatch")
                         end
+                    elseif packet.type == RPLC_TYPES.RPS_ENABLE then
+                        -- enable the reactor
+                        self.scrammed = false
+                        _send_ack(packet.type, self.rps.activate())
+                    elseif packet.type == RPLC_TYPES.RPS_SCRAM then
+                        -- disable the reactor
+                        self.scrammed = true
+                        self.rps.trip_manual()
+                        _send_ack(packet.type, true)
                     elseif packet.type == RPLC_TYPES.RPS_RESET then
                         -- reset the RPS status
                         rps.reset()

reactor-plc/startup.lua

@@ -11,7 +11,7 @@ local config = require("config")
 local plc = require("plc")
 local threads = require("threads")
 
-local R_PLC_VERSION = "alpha-v0.6.1"
+local R_PLC_VERSION = "alpha-v0.6.2"
 
 local print = util.print
 local println = util.println
@@ -37,7 +37,6 @@ local __shared_memory = {
     plc_state = {
         init_ok = true,
         shutdown = false,
-        scram = true,
         degraded = false,
         no_reactor = false,
         no_modem = false

reactor-plc/threads.lua

@@ -13,7 +13,7 @@ local println_ts = util.println_ts
 local psleep = util.psleep
 
 local MAIN_CLOCK    = 1   -- (1Hz, 20 ticks)
-local RPS_SLEEP     = 500 -- (500ms, 10 ticks)
+local RPS_SLEEP     = 250 -- (250ms, 5 ticks)
 local COMMS_SLEEP   = 150 -- (150ms, 3 ticks)
 local SP_CTRL_SLEEP = 250 -- (250ms, 5 ticks)
 
@@ -207,7 +207,7 @@ threads.thread__rps = function (smem)
             if plc_state.init_ok then
                 -- SCRAM if no open connection
                 if networked and not plc_comms.is_linked() then
-                    plc_state.scram = true
+                    rps.scram()
                     if was_linked then
                         was_linked = false
                         rps.trip_timeout()
@@ -219,21 +219,17 @@ threads.thread__rps = function (smem)
 
                 -- if we tried to SCRAM but failed, keep trying
                 -- in that case, SCRAM won't be called until it reconnects (this is the expected use of this check)
-                if not plc_state.no_reactor and plc_state.scram and reactor.getStatus() then
+                if not plc_state.no_reactor and rps.is_tripped() and reactor.getStatus() then
-                    reactor.scram()
+                    rps.scram()
                 end
 
                 -- if we are in standalone mode, continuously reset RPS
                 -- RPS will trip again if there are faults, but if it isn't cleared, the user can't re-enable
-                if not networked then
+                if not networked then rps.reset() end
-                    plc_state.scram = false
-                    rps.reset()
-                end
 
                 -- check safety (SCRAM occurs if tripped)
                 if not plc_state.no_reactor then
                     local rps_tripped, rps_status_string, rps_first = rps.check()
-                    plc_state.scram = plc_state.scram or rps_tripped
 
                     if rps_first then
                         println_ts("[RPS] SCRAM! safety trip: " .. rps_status_string)
@@ -250,27 +246,20 @@ threads.thread__rps = function (smem)
 
                 if msg.qtype == mqueue.TYPE.COMMAND then
                     -- received a command
+                    if plc_state.init_ok then
                         if msg.message == MQ__RPS_CMD.SCRAM then
-                        -- basic SCRAM
+                            -- SCRAM
-                        plc_state.scram = true
+                            rps.scram()
-                        reactor.scram()
                         elseif msg.message == MQ__RPS_CMD.DEGRADED_SCRAM then
-                        -- SCRAM with print
+                            -- lost peripheral(s)
-                        plc_state.scram = true
+                            rps.trip_degraded()
-                        if reactor.scram() then
-                            println_ts("successful reactor SCRAM")
-                            log.error("successful reactor SCRAM")
-                        else
-                            println_ts("failed reactor SCRAM")
-                            log.error("failed reactor SCRAM")
-                        end
                         elseif msg.message == MQ__RPS_CMD.TRIP_TIMEOUT then
                             -- watchdog tripped
-                        plc_state.scram = true
                             rps.trip_timeout()
                             println_ts("server timeout")
                             log.warning("server timeout")
                         end
+                    end
                 elseif msg.qtype == mqueue.TYPE.DATA then
                     -- received data
                 elseif msg.qtype == mqueue.TYPE.PACKET then
@@ -286,9 +275,7 @@ threads.thread__rps = function (smem)
                 -- safe exit
                 log.info("rps thread shutdown initiated")
                 if plc_state.init_ok then
-                    plc_state.scram = true
+                    if rps.scram() then
-                    reactor.scram()
-                    if reactor.__p_is_ok() then
                         println_ts("reactor disabled")
                         log.info("rps thread reactor SCRAM OK")
                     else
@@ -368,6 +355,8 @@ threads.thread__comms_rx = function (smem)
         -- load in from shared memory
         local plc_state     = smem.plc_state
         local setpoints     = smem.setpoints
+        local plc_dev       = smem.plc_dev
+        local rps           = smem.plc_sys.rps
         local plc_comms     = smem.plc_sys.plc_comms
         local conn_watchdog = smem.plc_sys.conn_watchdog
 
@@ -388,7 +377,7 @@ threads.thread__comms_rx = function (smem)
                 elseif msg.qtype == mqueue.TYPE.PACKET then
                     -- received a packet
                     -- handle the packet (setpoints passed to update burn rate setpoint)
-                    --                   (plc_state passed to allow clearing SCRAM flag and check if degraded)
+                    --                   (plc_state passed to check if degraded)
                     --                   (conn_watchdog passed to allow feeding the watchdog)
                     plc_comms.handle_packet(msg.message, setpoints, plc_state, conn_watchdog)
                 end
@@ -421,6 +410,7 @@ threads.thread__setpoint_control = function (smem)
         local plc_state     = smem.plc_state
         local setpoints     = smem.setpoints
         local plc_dev       = smem.plc_dev
+        local rps           = smem.plc_sys.rps
 
         local last_update   = util.time()
         local running       = false
@@ -433,7 +423,7 @@ threads.thread__setpoint_control = function (smem)
 
             -- check if we should start ramping
             if setpoints.burn_rate ~= last_sp_burn then
-                if not plc_state.scram then
+                if rps.is_active() then
                     if math.abs(setpoints.burn_rate - last_sp_burn) <= 5 then
                         -- update without ramp if <= 5 mB/t change
                         log.debug("setting burn rate directly to " .. setpoints.burn_rate .. "mB/t")
@@ -459,7 +449,7 @@ threads.thread__setpoint_control = function (smem)
                 running = false
 
                 -- adjust burn rate (setpoints.burn_rate)
-                if not plc_state.scram then
+                if rps.is_active() then
                     local current_burn_rate = reactor.getBurnRate()
                     if (current_burn_rate ~= ppm.ACCESS_FAULT) and (current_burn_rate ~= setpoints.burn_rate) then
                         -- calculate new burn rate

scada-common/comms.lua

@@ -17,9 +17,9 @@ local RPLC_TYPES = {
     LINK_REQ = 1,       -- linking requests
     STATUS = 2,         -- reactor/system status
     MEK_STRUCT = 3,     -- mekanism build structure
-    MEK_SCRAM = 4,      -- SCRAM reactor
+    MEK_BURN_RATE = 4,  -- set burn rate
-    MEK_ENABLE = 5,     -- enable reactor
+    RPS_ENABLE = 5,     -- enable reactor
-    MEK_BURN_RATE = 6,  -- set burn rate
+    RPS_SCRAM = 6,      -- SCRAM reactor
     RPS_STATUS = 7,     -- RPS status
     RPS_ALARM = 8,      -- RPS alarm broadcast
     RPS_RESET = 9       -- clear RPS trip (if in bad state, will trip immediately)
@@ -229,9 +229,9 @@ comms.rplc_packet = function ()
                 self.type == RPLC_TYPES.LINK_REQ or
                 self.type == RPLC_TYPES.STATUS or
                 self.type == RPLC_TYPES.MEK_STRUCT or
-                self.type == RPLC_TYPES.MEK_SCRAM or
-                self.type == RPLC_TYPES.MEK_ENABLE or
                 self.type == RPLC_TYPES.MEK_BURN_RATE or
+                self.type == RPLC_TYPES.RPS_ENABLE or
+                self.type == RPLC_TYPES.RPS_SCRAM or
                 self.type == RPLC_TYPES.RPS_ALARM or
                 self.type == RPLC_TYPES.RPS_STATUS or
                 self.type == RPLC_TYPES.RPS_RESET

scada-common/types.lua

@@ -17,12 +17,14 @@ types.rtu_t = {
 types.rps_status_t = {
     ok = "ok",
     dmg_crit = "dmg_crit",
-    ex_hcoolant = "heated_coolant_backup",
-    ex_waste = "full_waste",
     high_temp = "high_temp",
-    no_fuel = "no_fuel",
     no_coolant = "no_coolant",
-    timeout = "timeout"
+    ex_waste = "full_waste",
+    ex_hcoolant = "heated_coolant_backup",
+    no_fuel = "no_fuel",
+    fault = "fault",
+    timeout = "timeout",
+    manual = "manual"
 }
 
 -- MODBUS

supervisor/session/plc.lua

@@ -291,16 +291,15 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue)
                 else
                     log.debug(log_header .. "RPLC struct packet length mismatch")
                 end
-            elseif pkt.type == RPLC_TYPES.MEK_SCRAM then
+            elseif pkt.type == RPLC_TYPES.MEK_BURN_RATE then
-                -- SCRAM acknowledgement
+                -- burn rate acknowledgement
                 local ack = _get_ack(pkt)
                 if ack then
-                    self.acks.scram = true
+                    self.acks.burn_rate = true
-                    self.sDB.control_state = false
                 elseif ack == false then
-                    log.debug(log_header .. "SCRAM failed!")
+                    log.debug(log_header .. "burn rate update failed!")
                 end
-            elseif pkt.type == RPLC_TYPES.MEK_ENABLE then
+            elseif pkt.type == RPLC_TYPES.RPS_ENABLE then
                 -- enable acknowledgement
                 local ack = _get_ack(pkt)
                 if ack then
@@ -309,13 +308,14 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue)
                 elseif ack == false then
                     log.debug(log_header .. "enable failed!")
                 end
-            elseif pkt.type == RPLC_TYPES.MEK_BURN_RATE then
+            elseif pkt.type == RPLC_TYPES.RPS_SCRAM then
-                -- burn rate acknowledgement
+                -- SCRAM acknowledgement
                 local ack = _get_ack(pkt)
                 if ack then
-                    self.acks.burn_rate = true
+                    self.acks.scram = true
+                    self.sDB.control_state = false
                 elseif ack == false then
-                    log.debug(log_header .. "burn rate update failed!")
+                    log.debug(log_header .. "SCRAM failed!")
                 end
             elseif pkt.type == RPLC_TYPES.RPS_STATUS then
                 -- RPS status packet received, copy data
@@ -428,16 +428,16 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue)
                 elseif message.qtype == mqueue.TYPE.COMMAND then
                     -- handle instruction
                     local cmd = message.message
-                    if cmd == PLC_S_CMDS.SCRAM then
+                    if cmd == PLC_S_CMDS.ENABLE then
-                        -- SCRAM reactor
-                        self.acks.scram = false
-                        self.retry_times.scram_req = util.time() + INITIAL_WAIT
-                        _send(RPLC_TYPES.MEK_SCRAM, {})
-                    elseif cmd == PLC_S_CMDS.ENABLE then
                         -- enable reactor
                         self.acks.enable = false
                         self.retry_times.enable_req = util.time() + INITIAL_WAIT
-                        _send(RPLC_TYPES.MEK_ENABLE, {})
+                        _send(RPLC_TYPES.RPS_ENABLE, {})
+                    elseif cmd == PLC_S_CMDS.SCRAM then
+                        -- SCRAM reactor
+                        self.acks.scram = false
+                        self.retry_times.scram_req = util.time() + INITIAL_WAIT
+                        _send(RPLC_TYPES.RPS_SCRAM, {})
                     elseif cmd == PLC_S_CMDS.RPS_RESET then
                         -- reset RPS
                         self.acks.rps_reset = false
@@ -517,7 +517,7 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue)
 
             if not self.acks.scram then
                 if rtimes.scram_req - util.time() <= 0 then
-                    _send(RPLC_TYPES.MEK_SCRAM, {})
+                    _send(RPLC_TYPES.RPS_SCRAM, {})
                     rtimes.scram_req = util.time() + RETRY_PERIOD
                 end
             end
@@ -526,7 +526,7 @@ plc.new_session = function (id, for_reactor, in_queue, out_queue)
 
             if not self.acks.enable then
                 if rtimes.enable_req - util.time() <= 0 then
-                    _send(RPLC_TYPES.MEK_ENABLE, {})
+                    _send(RPLC_TYPES.RPS_ENABLE, {})
                     rtimes.enable_req = util.time() + RETRY_PERIOD
                 end
             end

supervisor/startup.lua

@@ -14,7 +14,7 @@ local svsessions = require("session.svsessions")
 local config = require("config")
 local supervisor = require("supervisor")
 
-local SUPERVISOR_VERSION = "alpha-v0.3.1"
+local SUPERVISOR_VERSION = "alpha-v0.3.2"
 
 local print = util.print
 local println = util.println