graceful shutdown: ./shutdown flushes players, then stops the servers

The stock scripts stop the game with SIGKILL (mbkill.sh runs kill -9) and
'./reboot' only calls mbrestart.sh, so anything still parked in a character's
deferred database job is thrown away: experience is written five minutes after
it is earned and stats/skills thirty seconds after they change
(engine.jobs.DatabaseUpdateJob). PlayerCharacter.updateDatabase() is an empty
stub, so there was no write-out path at all.

Added engine.gameManager.GracefulShutdown:
  flushAllPlayers()      - writes skills/powers, stat modifiers and experience
                           for every online character
  announce(String)       - server wide flash message
  registerShutdownHook() - flush on an orderly SIGTERM (SIGKILL cannot be
                           intercepted by design)
  gracefulStop(int)      - countdown, flush, then stop the login and world
                           servers

Added the './shutdown [seconds]' dev command (engine.devcmd.cmds.ShutdownCmd,
default 60s, ADMIN only like every other dev command), registered together with
the shutdown hook in WorldServer.main and LoginServer.main.

Verified on a MagicBox container: ant builds clean, the boot log shows
"shutdown hook registered for WorldServer"/"LoginServer", and kill -TERM on the
world server logs "WorldServer is terminating, flushing player data" ->
"flush finished, saved=0 failed=0" -> clean exit.

src/engine/devcmd/cmds/ShutdownCmd.java

@@ -0,0 +1,84 @@
+// • ▌ ▄ ·.  ▄▄▄·  ▄▄ • ▪   ▄▄· ▄▄▄▄·  ▄▄▄·  ▐▄▄▄  ▄▄▄ .
+// ·██ ▐███▪▐█ ▀█ ▐█ ▀ ▪██ ▐█ ▌▪▐█ ▀█▪▐█ ▀█ •█▌ ▐█▐▌·
+// ▐█ ▌▐▌▐█·▄█▀▀█ ▄█ ▀█▄▐█·██ ▄▄▐█▀▀█▄▄█▀▀█ ▐█▐ ▐▌▐▀▀▀
+// ██ ██▌▐█▌▐█ ▪▐▌▐█▄▪▐█▐█▌▐███▌██▄▪▐█▐█ ▪▐▌██▐ █▌▐█▄▄▌
+// ▀▀  █▪▀▀▀ ▀  ▀ ·▀▀▀▀ ▀▀▀·▀▀▀ ·▀▀▀▀  ▀  ▀ ▀▀  █▪ ▀▀▀
+//      Magicbane Emulator Project © 2013 - 2022
+//                www.magicbane.com
+
+
+package engine.devcmd.cmds;
+
+import engine.devcmd.AbstractDevCmd;
+import engine.gameManager.GracefulShutdown;
+import engine.objects.AbstractGameObject;
+import engine.objects.PlayerCharacter;
+
+/**
+ * Devcmd: graceful shutdown.
+ *
+ * Announces a countdown to every player, writes all online characters to the
+ * database (skills/powers, stat modifiers, experience) and then stops the
+ * login and world servers. Unlike './reboot' - which simply runs mbrestart.sh
+ * and therefore kills the JVMs with SIGKILL - nothing still parked in a
+ * deferred database job is lost.
+ *
+ * Usage: ./shutdown [seconds]  (default 60, maximum 3600)
+ */
+public class ShutdownCmd extends AbstractDevCmd {
+
+    private static final int DEFAULT_DELAY_SECONDS = 60;
+    private static final int MAX_DELAY_SECONDS = 3600;
+
+    public ShutdownCmd() {
+        super("shutdown");
+    }
+
+    @Override
+    protected void _doCmd(PlayerCharacter pcSender, String[] args, AbstractGameObject target) {
+
+        int delay = DEFAULT_DELAY_SECONDS;
+
+        if (args.length > 0 && args[0].isEmpty() == false) {
+            try {
+                delay = Integer.parseInt(args[0].trim());
+            } catch (NumberFormatException e) {
+                this.sendUsage(pcSender);
+                return;
+            }
+        }
+
+        if (delay < 0)
+            delay = 0;
+
+        if (delay > MAX_DELAY_SECONDS)
+            delay = MAX_DELAY_SECONDS;
+
+        throwbackInfo(pcSender, "Graceful shutdown started: player data will be saved in " + delay + " second(s).");
+
+        final int delaySeconds = delay;
+
+        // The countdown sleeps, so run it off the message handling thread.
+
+        Thread shutdownThread = new Thread(new Runnable() {
+
+            @Override
+            public void run() {
+                GracefulShutdown.gracefulStop(delaySeconds);
+            }
+        }, "gracefulShutdown");
+
+        shutdownThread.setDaemon(false);
+        shutdownThread.start();
+    }
+
+    @Override
+    protected String _getHelpString() {
+        return "Gracefully shuts the server down: warns the players, saves every online character, then stops the servers";
+    }
+
+    @Override
+    protected String _getUsageString() {
+        return "' /shutdown [seconds]'";
+    }
+}

src/engine/gameManager/DevCmdManager.java

@@ -123,6 +123,7 @@ public enum DevCmdManager {
         DevCmdManager.registerDevCmd(new ShowOffsetCmd());
         DevCmdManager.registerDevCmd(new RealmInfoCmd());
         DevCmdManager.registerDevCmd(new RebootCmd());
+        DevCmdManager.registerDevCmd(new ShutdownCmd());
         DevCmdManager.registerDevCmd(new SetMineTypeCmd());
         DevCmdManager.registerDevCmd(new SetMineExpansion());
         DevCmdManager.registerDevCmd(new SetForceRenameCityCmd());

src/engine/gameManager/GracefulShutdown.java

@@ -0,0 +1,202 @@
+// • ▌ ▄ ·.  ▄▄▄·  ▄▄ • ▪   ▄▄· ▄▄▄▄·  ▄▄▄·  ▐▄▄▄  ▄▄▄ .
+// ·██ ▐███▪▐█ ▀█ ▐█ ▀ ▪██ ▐█ ▌▪▐█ ▀█▪▐█ ▀█ •█▌ ▐█▐▌·
+// ▐█ ▌▐▌▐█·▄█▀▀█ ▄█ ▀█▄▐█·██ ▄▄▐█▀▀█▄▄█▀▀█ ▐█▐ ▐▌▐▀▀▀
+// ██ ██▌▐█▌▐█ ▪▐▌▐█▄▪▐█▐█▌▐███▌██▄▪▐█▐█ ▪▐▌██▐ █▌▐█▄▄▌
+// ▀▀  █▪▀▀▀ ▀  ▀ ·▀▀▀▀ ▀▀▀·▀▀▀ ·▀▀▀▀  ▀  ▀ ▀▀  █▪ ▀▀▀
+//      Magicbane Emulator Project © 2013 - 2022
+//                www.magicbane.com
+
+
+package engine.gameManager;
+
+import engine.objects.PlayerCharacter;
+import engine.util.ThreadUtils;
+import org.pmw.tinylog.Logger;
+
+import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Graceful shutdown support.
+ *
+ * The stock devops scripts stop the game with SIGKILL (mbkill.sh runs
+ * 'kill -9'), which throws away everything still parked in a character's
+ * deferred database job: experience is written five minutes after it is
+ * earned and stats/skills thirty seconds after they change
+ * (see {@link engine.jobs.DatabaseUpdateJob}). This class adds the missing
+ * write-out:
+ *
+ * <ul>
+ * <li>{@link #flushAllPlayers()} writes every online character's pending state
+ * (skills/powers, stat modifiers, experience) immediately.</li>
+ * <li>{@link #registerShutdownHook(String)} makes an orderly SIGTERM flush
+ * before the JVM exits. SIGKILL cannot be intercepted by design, so it is
+ * still the operator's job to log the players out first, or to use
+ * {@link #gracefulStop(int)}.</li>
+ * <li>{@link #gracefulStop(int)} announces a countdown to the players, flushes
+ * them and then stops the login and world servers.</li>
+ * </ul>
+ */
+public class GracefulShutdown {
+
+    private static final AtomicBoolean flushing = new AtomicBoolean(false);
+    private static final AtomicBoolean stopping = new AtomicBoolean(false);
+
+    // Countdown marks, in seconds remaining, that are announced to the players.
+
+    private static final int[] ANNOUNCE_MARKS = {300, 180, 120, 60, 30, 10, 5, 3, 1};
+
+    /**
+     * Write every online character's deferred state to the database now.
+     * Idempotent and safe to call from a shutdown hook or repeatedly.
+     *
+     * @return the number of characters that were written successfully
+     */
+    public static int flushAllPlayers() {
+
+        if (flushing.compareAndSet(false, true) == false) {
+            Logger.warn("GracefulShutdown: a flush is already in progress, skipping");
+            return 0;
+        }
+
+        int saved = 0;
+        int failed = 0;
+
+        try {
+            ArrayList<PlayerCharacter> online = new ArrayList<>(SessionManager.getAllActivePlayerCharacters());
+
+            Logger.info("GracefulShutdown: flushing " + online.size() + " online character(s)");
+
+            for (PlayerCharacter pc : online) {
+
+                if (pc == null)
+                    continue;
+
+                try {
+                    pc.updateSkillsAndPowersToDatabase();
+                    DbManager.PlayerCharacterQueries.UPDATE_CHARACTER_STATS(pc);
+                    DbManager.PlayerCharacterQueries.UPDATE_CHARACTER_EXPERIENCE(pc);
+                    saved++;
+                    Logger.info("GracefulShutdown: saved " + pc.getCombinedName() + " (" + pc.getObjectUUID() + ')');
+                } catch (Exception e) {
+                    failed++;
+                    Logger.error("GracefulShutdown: could not save character " + pc.getObjectUUID() + ": " + e);
+                }
+            }
+
+            Logger.info("GracefulShutdown: flush finished, saved=" + saved + " failed=" + failed);
+            System.out.println("GracefulShutdown: flush finished, saved=" + saved + " failed=" + failed);
+
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: flush aborted: " + e);
+        } finally {
+            flushing.set(false);
+        }
+
+        return saved;
+    }
+
+    /**
+     * Broadcast a system-wide flash message to every player.
+     *
+     * @param text - message to display
+     */
+    public static void announce(String text) {
+
+        if (text == null || text.isEmpty())
+            return;
+
+        try {
+            ChatManager.chatSystemFlash(text);
+            Logger.info("GracefulShutdown: announced '" + text + '\'');
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to announce: " + e);
+        }
+    }
+
+    /**
+     * Register the JVM shutdown hook used by an orderly SIGTERM
+     * (kill -TERM on the process, i.e. the signal the servers currently never
+     * receive). The hook flushes every online character before the JVM exits.
+     *
+     * @param serverName - name used in the log lines
+     */
+    public static void registerShutdownHook(final String serverName) {
+
+        try {
+            Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+
+                @Override
+                public void run() {
+                    Logger.info("GracefulShutdown: " + serverName + " is terminating, flushing player data");
+                    System.out.println("GracefulShutdown: " + serverName + " is terminating, flushing player data");
+                    flushAllPlayers();
+                    Logger.info("GracefulShutdown: " + serverName + " shutdown hook finished");
+                    System.out.println("GracefulShutdown: " + serverName + " shutdown hook finished");
+                }
+            }, "gracefulShutdownHook"));
+
+            Logger.info("GracefulShutdown: shutdown hook registered for " + serverName);
+            System.out.println("GracefulShutdown: shutdown hook registered for " + serverName);
+
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to register shutdown hook: " + e);
+        }
+    }
+
+    /**
+     * Announce a countdown, flush every online character and then stop the
+     * login and world servers. Only the first call has any effect.
+     *
+     * @param delaySeconds - seconds to wait before saving and stopping
+     *                     (0 stops immediately)
+     */
+    public static void gracefulStop(final int delaySeconds) {
+
+        if (stopping.compareAndSet(false, true) == false) {
+            Logger.warn("GracefulShutdown: a shutdown is already in progress");
+            return;
+        }
+
+        int remaining = Math.max(0, delaySeconds);
+
+        announce("Server is shutting down in " + remaining + " seconds. Please log out to save your character.");
+
+        while (remaining > 0) {
+
+            ThreadUtils.sleep(1000);
+            remaining--;
+
+            for (int mark : ANNOUNCE_MARKS)
+                if (mark == remaining)
+                    announce("Server is shutting down in " + remaining + " second(s).");
+        }
+
+        announce("Server is shutting down now. Saving all characters...");
+
+        flushAllPlayers();
+
+        stopServers();
+    }
+
+    /**
+     * Stop the login and world server processes the same way the devops
+     * scripts do. Called only after every character has been written out.
+     */
+    private static void stopServers() {
+
+        try {
+            Logger.info("GracefulShutdown: stopping the login and world servers");
+            System.out.println("GracefulShutdown: player data saved, stopping the login and world servers");
+            Runtime.getRuntime().exec("./mbkill.sh");
+        } catch (Exception e) {
+            Logger.error("GracefulShutdown: unable to run mbkill.sh: " + e);
+        }
+
+        // Give the script a moment to stop the login server before this JVM disappears.
+
+        ThreadUtils.sleep(2000);
+
+        System.exit(0);
+    }
+}

src/engine/server/login/LoginServer.java

@@ -96,6 +96,11 @@ public class LoginServer {
             // Start the Login Server
 
             loginServer.init();
+
+            // flush any cached character state on an orderly termination
+            // (the login server holds no players, so this normally logs 0).
+
+            GracefulShutdown.registerShutdownHook("LoginServer");
             loginServer.exec();
 
             exit(0);

src/engine/server/world/WorldServer.java

@@ -109,6 +109,11 @@ public class WorldServer {
 
 			worldServer.init();
 
+			// flush every online character if the JVM is asked to terminate
+			// in an orderly way (kill -TERM); SIGKILL cannot be intercepted.
+
+			GracefulShutdown.registerShutdownHook("WorldServer");
+
 			int retVal = worldServer.exec();
 
 			if (retVal != 0)