world: pace the mob AI and respawn sweeps instead of spinning

MobAIThread and MobRespawnThread both ran `while (true)` around a sweep of every
zone with no delay at all, so each of them pinned a full core for the lifetime of
the world server. Measured in the container: 195% total CPU with

    tid 229  aiThread       106% of one core
    tid 231  respawnThread  106% of one core

and nothing else above 1%. The 100 ms check in the respawn loop only throttles how
often a mob may respawn, it never paused the loop.

Both sweeps now run at a fixed rate (AI_TICK_INTERVAL_MS / RESPAWN_TICK_INTERVAL_MS,
100 ms by default) by sleeping the remainder of each tick. MobAI.DetermineAction
already gates individual actions on timestamps (lastAttackTime, nextCastTime,
patrol delays), so pacing the sweep removes the busy spin without changing the AI
behaviour; raise the constants to trade reaction time for CPU.

src/engine/devcmd/cmds/ShutdownCmd.java

@@ -0,0 +1,84 @@
+// • ▌ ▄ ·.  ▄▄▄·  ▄▄ • ▪   ▄▄· ▄▄▄▄·  ▄▄▄·  ▐▄▄▄  ▄▄▄ .
+// ·██ ▐███▪▐█ ▀█ ▐█ ▀ ▪██ ▐█ ▌▪▐█ ▀█▪▐█ ▀█ •█▌ ▐█▐▌·
+// ▐█ ▌▐▌▐█·▄█▀▀█ ▄█ ▀█▄▐█·██ ▄▄▐█▀▀█▄▄█▀▀█ ▐█▐ ▐▌▐▀▀▀
+// ██ ██▌▐█▌▐█ ▪▐▌▐█▄▪▐█▐█▌▐███▌██▄▪▐█▐█ ▪▐▌██▐ █▌▐█▄▄▌
+// ▀▀  █▪▀▀▀ ▀  ▀ ·▀▀▀▀ ▀▀▀·▀▀▀ ·▀▀▀▀  ▀  ▀ ▀▀  █▪ ▀▀▀
+//      Magicbane Emulator Project © 2013 - 2022
+//                www.magicbane.com
+
+
+package engine.devcmd.cmds;
+
+import engine.devcmd.AbstractDevCmd;
+import engine.gameManager.GracefulShutdown;
+import engine.objects.AbstractGameObject;
+import engine.objects.PlayerCharacter;
+
+/**
+ * Devcmd: graceful shutdown.
+ *
+ * Announces a countdown to every player, writes all online characters to the
+ * database (skills/powers, stat modifiers, experience) and then stops the
+ * login and world servers. Unlike './reboot' - which simply runs mbrestart.sh
+ * and therefore kills the JVMs with SIGKILL - nothing still parked in a
+ * deferred database job is lost.
+ *
+ * Usage: ./shutdown [seconds]  (default 60, maximum 3600)
+ */
+public class ShutdownCmd extends AbstractDevCmd {
+
+    private static final int DEFAULT_DELAY_SECONDS = 60;
+    private static final int MAX_DELAY_SECONDS = 3600;
+
+    public ShutdownCmd() {
+        super("shutdown");
+    }
+
+    @Override
+    protected void _doCmd(PlayerCharacter pcSender, String[] args, AbstractGameObject target) {
+
+        int delay = DEFAULT_DELAY_SECONDS;
+
+        if (args.length > 0 && args[0].isEmpty() == false) {
+            try {
+                delay = Integer.parseInt(args[0].trim());
+            } catch (NumberFormatException e) {
+                this.sendUsage(pcSender);
+                return;
+            }
+        }
+
+        if (delay < 0)
+            delay = 0;
+
+        if (delay > MAX_DELAY_SECONDS)
+            delay = MAX_DELAY_SECONDS;
+
+        throwbackInfo(pcSender, "Graceful shutdown started: player data will be saved in " + delay + " second(s).");
+
+        final int delaySeconds = delay;
+
+        // The countdown sleeps, so run it off the message handling thread.
+
+        Thread shutdownThread = new Thread(new Runnable() {
+
+            @Override
+            public void run() {
+                GracefulShutdown.gracefulStop(delaySeconds);
+            }
+        }, "gracefulShutdown");
+
+        shutdownThread.setDaemon(false);
+        shutdownThread.start();
+    }
+
+    @Override
+    protected String _getHelpString() {
+        return "Gracefully shuts the server down: warns the players, saves every online character, then stops the servers";
+    }
+
+    @Override
+    protected String _getUsageString() {
+        return "' /shutdown [seconds]'";
+    }
+}

src/engine/gameManager/DevCmdManager.java

@@ -123,6 +123,7 @@ public enum DevCmdManager {
         DevCmdManager.registerDevCmd(new ShowOffsetCmd());
         DevCmdManager.registerDevCmd(new RealmInfoCmd());
         DevCmdManager.registerDevCmd(new RebootCmd());
+        DevCmdManager.registerDevCmd(new ShutdownCmd());
         DevCmdManager.registerDevCmd(new SetMineTypeCmd());
         DevCmdManager.registerDevCmd(new SetMineExpansion());
         DevCmdManager.registerDevCmd(new SetForceRenameCityCmd());

src/engine/gameManager/GracefulShutdown.java

@@ -0,0 +1,202 @@
+// • ▌ ▄ ·.  ▄▄▄·  ▄▄ • ▪   ▄▄· ▄▄▄▄·  ▄▄▄·  ▐▄▄▄  ▄▄▄ .
+// ·██ ▐███▪▐█ ▀█ ▐█ ▀ ▪██ ▐█ ▌▪▐█ ▀█▪▐█ ▀█ •█▌ ▐█▐▌·
+// ▐█ ▌▐▌▐█·▄█▀▀█ ▄█ ▀█▄▐█·██ ▄▄▐█▀▀█▄▄█▀▀█ ▐█▐ ▐▌▐▀▀▀
+// ██ ██▌▐█▌▐█ ▪▐▌▐█▄▪▐█▐█▌▐███▌██▄▪▐█▐█ ▪▐▌██▐ █▌▐█▄▄▌
+// ▀▀  █▪▀▀▀ ▀  ▀ ·▀▀▀▀ ▀▀▀·▀▀▀ ·▀▀▀▀  ▀  ▀ ▀▀  █▪ ▀▀▀
+//      Magicbane Emulator Project © 2013 - 2022
+//                www.magicbane.com
+
+
+package engine.gameManager;
+
+import engine.objects.PlayerCharacter;
+import engine.util.ThreadUtils;
+import org.pmw.tinylog.Logger;
+
+import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Graceful shutdown support.
+ *
+ * The stock devops scripts stop the game with SIGKILL (mbkill.sh runs
+ * 'kill -9'), which throws away everything still parked in a character's
+ * deferred database job: experience is written five minutes after it is
+ * earned and stats/skills thirty seconds after they change
+ * (see {@link engine.jobs.DatabaseUpdateJob}). This class adds the missing
+ * write-out:
+ *
+ * <ul>
+ * <li>{@link #flushAllPlayers()} writes every online character's pending state
+ * (skills/powers, stat modifiers, experience) immediately.</li>
+ * <li>{@link #registerShutdownHook(String)} makes an orderly SIGTERM flush
+ * before the JVM exits. SIGKILL cannot be intercepted by design, so it is
+ * still the operator's job to log the players out first, or to use
+ * {@link #gracefulStop(int)}.</li>
+ * <li>{@link #gracefulStop(int)} announces a countdown to the players, flushes
+ * them and then stops the login and world servers.</li>
+ * </ul>
+ */
+public class GracefulShutdown {
+
+    private static final AtomicBoolean flushing = new AtomicBoolean(false);
+    private static final AtomicBoolean stopping = new AtomicBoolean(false);
+
+    // Countdown marks, in seconds remaining, that are announced to the players.
+
+    private static final int[] ANNOUNCE_MARKS = {300, 180, 120, 60, 30, 10, 5, 3, 1};
+
+    /**
+     * Write every online character's deferred state to the database now.
+     * Idempotent and safe to call from a shutdown hook or repeatedly.
+     *
+     * @return the number of characters that were written successfully
+     */
+    public static int flushAllPlayers() {
+
+        if (flushing.compareAndSet(false, true) == false) {
+            Logger.warn("GracefulShutdown: a flush is already in progress, skipping");
+            return 0;
+        }
+
+        int saved = 0;
+        int failed = 0;
+
+        try {
+            ArrayList<PlayerCharacter> online = new ArrayList<>(SessionManager.getAllActivePlayerCharacters());
+
+            Logger.info("GracefulShutdown: flushing " + online.size() + " online character(s)");
+
+            for (PlayerCharacter pc : online) {
+
+                if (pc == null)
+                    continue;
+
+                try {
+                    pc.updateSkillsAndPowersToDatabase();
+                    DbManager.PlayerCharacterQueries.UPDATE_CHARACTER_STATS(pc);
+                    DbManager.PlayerCharacterQueries.UPDATE_CHARACTER_EXPERIENCE(pc);
+                    saved++;
+                    Logger.info("GracefulShutdown: saved " + pc.getCombinedName() + " (" + pc.getObjectUUID() + ')');
+                } catch (Exception e) {
+                    failed++;
+                    Logger.error("GracefulShutdown: could not save character " + pc.getObjectUUID() + ": " + e);
+                }
+            }
+
+            Logger.info("GracefulShutdown: flush finished, saved=" + saved + " failed=" + failed);
+            System.out.println("GracefulShutdown: flush finished, saved=" + saved + " failed=" + failed);
+
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: flush aborted: " + e);
+        } finally {
+            flushing.set(false);
+        }
+
+        return saved;
+    }
+
+    /**
+     * Broadcast a system-wide flash message to every player.
+     *
+     * @param text - message to display
+     */
+    public static void announce(String text) {
+
+        if (text == null || text.isEmpty())
+            return;
+
+        try {
+            ChatManager.chatSystemFlash(text);
+            Logger.info("GracefulShutdown: announced '" + text + '\'');
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to announce: " + e);
+        }
+    }
+
+    /**
+     * Register the JVM shutdown hook used by an orderly SIGTERM
+     * (kill -TERM on the process, i.e. the signal the servers currently never
+     * receive). The hook flushes every online character before the JVM exits.
+     *
+     * @param serverName - name used in the log lines
+     */
+    public static void registerShutdownHook(final String serverName) {
+
+        try {
+            Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+
+                @Override
+                public void run() {
+                    Logger.info("GracefulShutdown: " + serverName + " is terminating, flushing player data");
+                    System.out.println("GracefulShutdown: " + serverName + " is terminating, flushing player data");
+                    flushAllPlayers();
+                    Logger.info("GracefulShutdown: " + serverName + " shutdown hook finished");
+                    System.out.println("GracefulShutdown: " + serverName + " shutdown hook finished");
+                }
+            }, "gracefulShutdownHook"));
+
+            Logger.info("GracefulShutdown: shutdown hook registered for " + serverName);
+            System.out.println("GracefulShutdown: shutdown hook registered for " + serverName);
+
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to register shutdown hook: " + e);
+        }
+    }
+
+    /**
+     * Announce a countdown, flush every online character and then stop the
+     * login and world servers. Only the first call has any effect.
+     *
+     * @param delaySeconds - seconds to wait before saving and stopping
+     *                     (0 stops immediately)
+     */
+    public static void gracefulStop(final int delaySeconds) {
+
+        if (stopping.compareAndSet(false, true) == false) {
+            Logger.warn("GracefulShutdown: a shutdown is already in progress");
+            return;
+        }
+
+        int remaining = Math.max(0, delaySeconds);
+
+        announce("Server is shutting down in " + remaining + " seconds. Please log out to save your character.");
+
+        while (remaining > 0) {
+
+            ThreadUtils.sleep(1000);
+            remaining--;
+
+            for (int mark : ANNOUNCE_MARKS)
+                if (mark == remaining)
+                    announce("Server is shutting down in " + remaining + " second(s).");
+        }
+
+        announce("Server is shutting down now. Saving all characters...");
+
+        flushAllPlayers();
+
+        stopServers();
+    }
+
+    /**
+     * Stop the login and world server processes the same way the devops
+     * scripts do. Called only after every character has been written out.
+     */
+    private static void stopServers() {
+
+        try {
+            Logger.info("GracefulShutdown: stopping the login and world servers");
+            System.out.println("GracefulShutdown: player data saved, stopping the login and world servers");
+            Runtime.getRuntime().exec("./mbkill.sh");
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to run mbkill.sh: " + e);
+        }
+
+        // Give the script a moment to stop the login server before this JVM disappears.
+
+        ThreadUtils.sleep(2000);
+
+        System.exit(0);
+    }
+}

src/engine/mobileAI/MobAI.java

@@ -693,6 +693,8 @@ public class MobAI {
                     DefaultLogic(mob);
                     break;
             }
+            if(mob.isAlive())
+                RecoverHealth(mob);
         } catch (Exception e) {
             Logger.info(mob.getObjectUUID() + " " + mob.getName() + " Failed At: DetermineAction" + " " + e.getMessage());
         }
@@ -808,7 +810,7 @@ public class MobAI {
                         chaseTarget(mob);
                     break;
                 case GuardMinion:
-                    if (!mob.npcOwner.isAlive() || ((Mob) mob.npcOwner).despawned)
+                    if (!mob.npcOwner.isAlive() && mob.getCombatTarget() == null)
                         randomGuardPatrolPoint(mob);
                     else {
                         if (mob.getCombatTarget() != null) {
@@ -827,6 +829,7 @@ public class MobAI {
                         chaseTarget(mob);
                     }
                     break;
+
             }
         } catch (Exception e) {
             Logger.info(mob.getObjectUUID() + " " + mob.getName() + " Failed At: CheckMobMovement" + " " + e.getMessage());
@@ -1043,7 +1046,6 @@ public class MobAI {
                     mob.setCombatTarget(newTarget);
 
             }
-
             CheckMobMovement(mob);
             CheckForAttack(mob);
         } catch (Exception e) {
@@ -1054,24 +1056,9 @@ public class MobAI {
     public static void GuardMinionLogic(Mob mob) {
 
         try {
-            if (!mob.npcOwner.isAlive()) {
-
-                if (mob.getCombatTarget() == null) {
-                    CheckForPlayerGuardAggro(mob);
-                } else {
-
-                    AbstractWorldObject newTarget = ChangeTargetFromHateValue(mob);
-
-                    if (newTarget != null) {
-
-                        if (newTarget.getObjectType().equals(Enum.GameObjectType.PlayerCharacter)) {
-                            if (GuardCanAggro(mob, (PlayerCharacter) newTarget))
-                                mob.setCombatTarget(newTarget);
-                        } else
-                            mob.setCombatTarget(newTarget);
-
-                    }
-                }
+            boolean isComanded = mob.npcOwner.isAlive();
+            if (!isComanded) {
+                GuardCaptainLogic(mob);
             }else {
                 if (mob.npcOwner.getCombatTarget() != null)
                     mob.setCombatTarget(mob.npcOwner.getCombatTarget());
@@ -1110,22 +1097,6 @@ public class MobAI {
                 CheckMobMovement(mob);
 
             CheckForAttack(mob);
-
-            //recover health
-
-            if (mob.getTimestamps().containsKey("HEALTHRECOVERED") == false)
-                mob.getTimestamps().put("HEALTHRECOVERED", System.currentTimeMillis());
-
-            if (mob.isSit() && mob.getTimeStamp("HEALTHRECOVERED") < System.currentTimeMillis() + 3000)
-                if (mob.getHealth() < mob.getHealthMax()) {
-
-                    float recoveredHealth = mob.getHealthMax() * ((1 + mob.getBonuses().getFloatPercentAll(Enum.ModType.HealthRecoverRate, Enum.SourceType.None)) * 0.01f);
-                    mob.setHealth(mob.getHealth() + recoveredHealth);
-                    mob.getTimestamps().put("HEALTHRECOVERED", System.currentTimeMillis());
-
-                    if (mob.getHealth() > mob.getHealthMax())
-                        mob.setHealth(mob.getHealthMax());
-                }
         } catch (Exception e) {
             Logger.info(mob.getObjectUUID() + " " + mob.getName() + " Failed At: PetLogic" + " " + e.getMessage());
         }
@@ -1379,4 +1350,22 @@ public class MobAI {
         }
         return null;
     }
+
+    public static void RecoverHealth(Mob mob){
+        //recover health
+
+        if (mob.getTimestamps().containsKey("HEALTHRECOVERED") == false)
+            mob.getTimestamps().put("HEALTHRECOVERED", System.currentTimeMillis());
+
+        if (mob.isSit() && mob.getTimeStamp("HEALTHRECOVERED") < System.currentTimeMillis() + 3000)
+            if (mob.getHealth() < mob.getHealthMax()) {
+
+                float recoveredHealth = mob.getHealthMax() * ((1 + mob.getBonuses().getFloatPercentAll(Enum.ModType.HealthRecoverRate, Enum.SourceType.None)) * 0.01f);
+                mob.setHealth(mob.getHealth() + recoveredHealth);
+                mob.getTimestamps().put("HEALTHRECOVERED", System.currentTimeMillis());
+
+                if (mob.getHealth() > mob.getHealthMax())
+                    mob.setHealth(mob.getHealthMax());
+            }
+    }
 }

src/engine/mobileAI/Threads/MobAIThread.java

@@ -14,6 +14,21 @@ public class MobAIThread implements Runnable{
     public static int AI_PULSE_MOB_THRESHOLD = 200;
     public static int AI_PATROL_DIVISOR = 15;
     public static float AI_CAST_FREQUENCY;
+
+    /**
+     * Fixed-rate pacing for one full AI pass, in milliseconds (default 10 passes
+     * per second).
+     *
+     * The pass used to run with no delay whatsoever - `while (true)` straight into
+     * a sweep of every zone and every mob - which pinned one core for the entire
+     * lifetime of the world server (~100% of a core even in an empty world, two
+     * cores together with MobRespawnThread). Individual actions inside
+     * MobAI.DetermineAction are already gated by timestamps (lastAttackTime,
+     * nextCastTime, patrol delays), so running the sweep at a fixed rate removes
+     * the busy spin without changing behaviour. Raise this value to trade AI
+     * reaction time for CPU.
+     */
+    public static int AI_TICK_INTERVAL_MS = 100;
     // Thread constructor
 
     public MobAIThread() {
@@ -27,6 +42,9 @@ public class MobAIThread implements Runnable{
         AI_CAST_FREQUENCY = Float.parseFloat(ConfigManager.MB_AI_CAST_FREQUENCY.getValue());
         AI_BASE_AGGRO_RANGE = (int)(60 * Float.parseFloat(ConfigManager.MB_AI_AGGRO_RANGE.getValue()));
         while (true) {
+
+            long tickStart = System.currentTimeMillis();
+
             for (Zone zone : ZoneManager.getAllZones()) {
 
                 for (Mob mob : zone.zoneMobSet) {
@@ -40,6 +58,16 @@ public class MobAIThread implements Runnable{
                     }
                 }
             }
+
+            long sleep = AI_TICK_INTERVAL_MS - (System.currentTimeMillis() - tickStart);
+
+            if (sleep > 0) {
+                try {
+                    Thread.sleep(sleep);
+                } catch (InterruptedException ie) {
+                    return;
+                }
+            }
         }
     }
     public static void startAIThread() {

src/engine/mobileAI/Threads/MobRespawnThread.java

@@ -25,6 +25,13 @@ import org.pmw.tinylog.Logger;
 
 public class MobRespawnThread implements Runnable {
 
+    /**
+     * Fixed-rate pacing for one respawn sweep, in milliseconds. The sweep used to
+     * run with no delay at all, pinning a core for the lifetime of the world server
+     * (the 100 ms check below only throttles how often a mob may actually respawn,
+     * it never paused the loop). Keep this at or below the 100 ms respawn throttle.
+     */
+    public static int RESPAWN_TICK_INTERVAL_MS = 100;
 
     public MobRespawnThread() {
         Logger.info(" MobRespawnThread thread has started!");
@@ -36,6 +43,8 @@ public class MobRespawnThread implements Runnable {
 
         while (true) {
 
+            long tickStart = System.currentTimeMillis();
+
             try {
                 for (Zone zone : ZoneManager.getAllZones()) {
 
@@ -55,6 +64,15 @@ public class MobRespawnThread implements Runnable {
                 Logger.error(e);
             }
 
+            long sleep = RESPAWN_TICK_INTERVAL_MS - (System.currentTimeMillis() - tickStart);
+
+            if (sleep > 0) {
+                try {
+                    Thread.sleep(sleep);
+                } catch (InterruptedException ie) {
+                    return;
+                }
+            }
         }
     }
     public static void startRespawnThread() {

src/engine/server/login/LoginServer.java

@@ -96,6 +96,11 @@ public class LoginServer {
             // Start the Login Server
 
             loginServer.init();
+
+            // flush any cached character state on an orderly termination
+            // (the login server holds no players, so this normally logs 0).
+
+            GracefulShutdown.registerShutdownHook("LoginServer");
             loginServer.exec();
 
             exit(0);
@@ -145,29 +150,42 @@ public class LoginServer {
 
     }
 
+    /**
+     * Returns true when something is already listening on the given TCP port.
+     *
+     * This used to shell out to `lsof -i tcp:<port>` through /bin/bash and read the
+     * child's output until EOF. With no timeout and no waitFor() a child that blocks
+     * is fatal: lsof has been observed stuck in uninterruptible I/O (D state) while
+     * scanning the busy world server inside the container, which left the login
+     * server permanently inside its startup path. LoginServer.exec() never returned
+     * from checkServerHealth(), the accept loop never started, and clients could open
+     * a TCP connection to the port and then wait forever for the 100 byte DH
+     * handshake - surfacing as "Failed to open a server connection" after the
+     * client's own timeout.
+     *
+     * A bind probe is equivalent for this purpose, needs no external tool and cannot
+     * hang. SO_REUSEADDR does not allow stealing a port that is actively listening,
+     * so a successful bind still means the port is free.
+     */
     public static boolean isPortInUse(int port) {
 
-        ProcessBuilder builder = new ProcessBuilder("/bin/bash", "-c", "lsof -i tcp:" + port + " | tail -n +2 | awk '{print $2}'");
-        builder.redirectErrorStream(true);
-        Process process = null;
-        String line = null;
-        boolean portInUse = false;
+        java.net.ServerSocket probe = null;
 
         try {
-            process = builder.start();
-
-            InputStream is = process.getInputStream();
-            BufferedReader reader = new BufferedReader(new InputStreamReader(is));
-
-            while ((line = reader.readLine()) != null) {
-                portInUse = true;
-            }
-
+            probe = new java.net.ServerSocket();
+            probe.setReuseAddress(true);
+            probe.bind(new java.net.InetSocketAddress(port));
+            return false;
         } catch (IOException e) {
-            e.printStackTrace();
+            return true;
+        } finally {
+            if (probe != null)
+                try {
+                    probe.close();
+                } catch (IOException e) {
+                    // the bind result is what matters; closing cannot change it
+                }
         }
-
-        return portInUse;
     }
 
     private void exec() {

src/engine/server/world/WorldServer.java

@@ -109,6 +109,11 @@ public class WorldServer {
 
 			worldServer.init();
 
+			// flush every online character if the JVM is asked to terminate
+			// in an orderly way (kill -TERM); SIGKILL cannot be intercepted.
+
+			GracefulShutdown.registerShutdownHook("WorldServer");
+
 			int retVal = worldServer.exec();
 
 			if (retVal != 0)