Core/Database: Fixes game crashes when loading into work due to hotfix_blob table having bad data. Now using mysqlcli to handle all file imports. Must fill out MySQLExecutable in config to use sql updater.

2022-09-24 13:30:15 -04:00
parent 431a06fe24
commit 44e9ff9793
5 changed files with 283 additions and 79 deletions
@@ -103,14 +103,23 @@ AllowLoggingIPAddressesInDatabase = 1
 # MYSQL SETTINGS
 #
 #    LoginDatabaseInfo
-#        Description: Database connection settings for the realm server.
-#        Example:     "hostname;port;username;password;database"
+#        Description: Database connection settings for the realm server.    
+#        Example:
+#					  DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets.
+#					  DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets)
+#					  DatabaseInfo.Username = "username"
+#					  DatabaseInfo.Password = "password"
+#					  DatabaseInfo.Database = "auth"
+#					  DatabaseInfo.SSL = false
+#
+#		 Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini.

 LoginDatabaseInfo.Host = "127.0.0.1"
 LoginDatabaseInfo.Port = "3306"
 LoginDatabaseInfo.Username = "username"
 LoginDatabaseInfo.Password = "password"
 LoginDatabaseInfo.Database = "auth"
+LoginDatabaseInfo.SSL = false

 #
 #    DBQueryWorkerInterval
@@ -119,6 +128,17 @@ LoginDatabaseInfo.Database = "auth"

 DBQueryWorkerInterval = 1000

+#
+#    MySQLExecutable
+#        Description: The path to your MySQL CLI binary.
+#                     If the path is left empty, built-in path from cmake is used.
+#        Example:     "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
+#                     "mysql.exe"
+#                     "/usr/bin/mysql"
+#        Default:     ""
+
+MySQLExecutable = ""
+
 #
 ###################################################################################################

@@ -19,6 +19,7 @@ using Framework.Configuration;
 using MySqlConnector;
 using System;
 using System.Collections.Generic;
+using System.IO;

 namespace Framework.Database
 {
@@ -38,10 +39,11 @@ namespace Framework.Database
                MySqlConnectionInfo connectionObject = new()
                {
                    Host = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Host", ""),
-                    Port = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""),
+                    PortOrSocket = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""),
                    Username = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Username", ""),
                    Password = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Password", ""),
-                    Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", "")
+                    Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", ""),
+                    UseSSL = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.SSL", false)
                };

                var error = database.Initialize(connectionObject);
@@ -50,17 +52,9 @@ namespace Framework.Database
                    // Database does not exist
                    if (error == MySqlErrorCode.UnknownDatabase && updatesEnabled && _autoSetup)
                    {
-                        Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: ");
-
-                        string answer = Console.ReadLine();
-                        if (string.IsNullOrEmpty(answer) || answer[0] != 'y')
-                            return false;
-
-                        Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"...");
-                        string sqlString = $"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci";
                        // Try to create the database and connect again if auto setup is enabled
-                        if (database.Apply(sqlString) && database.Initialize(connectionObject) == MySqlErrorCode.None)
-                            error = MySqlErrorCode.None;
+                        if (CreateDatabase(connectionObject, database))
+                            error = database.Initialize(connectionObject);
                    }

                    // If the error wasn't handled quit
@@ -69,9 +63,8 @@ namespace Framework.Database
                        Log.outError(LogFilter.ServerLoading, $"\nDatabase {connectionObject.Database} NOT opened. There were errors opening the MySQL connections. Check your SQLErrors for specific errors.");
                        return false;
                    }
-
-                    Log.outInfo(LogFilter.ServerLoading, "Done.");
                }
+
                return true;
            });

@@ -106,6 +99,47 @@ namespace Framework.Database
            });
        }

+        public bool CreateDatabase<T>(MySqlConnectionInfo connectionObject, MySqlBase<T> database)
+        {
+            Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: ");
+
+            string answer = Console.ReadLine();
+            if (!answer.IsEmpty() && answer[0] != 'y')
+                return false;
+
+            Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"...");
+
+            // Path of temp file
+            string temp = "create_table.sql";
+
+            // Create temporary query to use external MySQL CLi
+            try
+            {
+                using BinaryWriter binaryWriter = new(File.Open(temp, FileMode.Create, FileAccess.Write));
+                binaryWriter.Write($"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci");
+            }
+            catch (Exception)
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Failed to create temporary query file \"{temp}\"!");
+                return false;
+            }
+
+            try
+            {
+                database.ApplyFile(temp, false);
+            }
+            catch (Exception)
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Failed to create database {database.GetDatabaseName()}! Does the user (named in *.conf) have `CREATE`, `ALTER`, `DROP`, `INSERT` and `DELETE` privileges on the MySQL server?");
+                File.Delete(temp);
+                return false;
+            }
+
+            Log.outInfo(LogFilter.SqlUpdates, "Done.");
+            File.Delete(temp);
+            return true;
+        }
+
        public bool Load()
        {
            if (_updateFlags == 0)
@@ -38,6 +38,9 @@ namespace Framework.Database
            if (!result.IsEmpty() && !result.IsEmpty())
                return true;

+            if (!DBUpdaterUtil.CheckExecutable())
+                return false;
+
            Log.outInfo(LogFilter.SqlUpdates, $"Database {_database.GetDatabaseName()} is empty, auto populating it...");

            string path = GetSourceDirectory();
@@ -67,7 +70,14 @@ namespace Framework.Database

            // Update database
            Log.outInfo(LogFilter.SqlUpdates, $"Applying \'{fileName}\'...");
-            _database.ApplyFile(path + fileName);
+            try
+            {
+                ApplyFile(path + fileName);
+            }
+            catch (Exception)
+            {
+                return false;
+            }

            Log.outInfo(LogFilter.SqlUpdates, $"Done Applying \'{fileName}\'");
            return true;
@@ -75,13 +85,16 @@ namespace Framework.Database

        public bool Update()
        {
+            if (!DBUpdaterUtil.CheckExecutable())
+                return false;
+
            Log.outInfo(LogFilter.SqlUpdates, $"Updating {_database.GetDatabaseName()} database...");

            string sourceDirectory = GetSourceDirectory();

            if (!Directory.Exists(sourceDirectory))
            {
-                Log.outError(LogFilter.SqlUpdates, $"DBUpdater: Given source directory {sourceDirectory} does not exist, skipped!");
+                Log.outError(LogFilter.SqlUpdates, $"DBUpdater: The given source directory {sourceDirectory} does not exist, change the path to the directory where your sql directory exists (for example c:\\source\\cyphercore). Shutting down.");
                return false;
            }

@@ -255,19 +268,28 @@ namespace Framework.Database
            uint oldMSTime = Time.GetMSTime();

            // Update database
-            if (!_database.ApplyFile(path))
-                Log.outError(LogFilter.Sql, $"Update: {path} Failed. You need to apply it manually");
+            ApplyFile(path);

            // Return time the query took to apply
            return Time.GetMSTimeDiffToNow(oldMSTime);
        }

+        void ApplyFile(string path)
+        {
+            _database.ApplyFile(path);
+        }
+        
+        void Apply(string query)
+        {
+            _database.Execute(query);
+        }
+
        void UpdateEntry(AppliedFileEntry entry, uint speed)
        {
            string update = $"REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"{entry.Name}\", \"{entry.Hash}\", \'{entry.State}\', {speed})";

            // Update database
-            _database.Execute(update);
+            Apply(update);
        }

        void RenameEntry(string from, string to)
@@ -277,7 +299,7 @@ namespace Framework.Database
                string update = $"DELETE FROM `updates` WHERE `name`=\"{to}\"";

                // Update database
-                _database.Execute(update);
+                Apply(update);
            }

            // Rename
@@ -285,7 +307,7 @@ namespace Framework.Database
                string update = $"UPDATE `updates` SET `name`=\"{to}\" WHERE `name`=\"{from}\"";

                // Update database
-                _database.Execute(update);
+                Apply(update);
            }
        }

@@ -307,7 +329,7 @@ namespace Framework.Database
            update += ")";

            // Update database
-            _database.Execute(update);
+            Apply(update);
        }

        void UpdateState(string name, State state)
@@ -315,7 +337,7 @@ namespace Framework.Database
            string update = $"UPDATE `updates` SET `state`=\'{state}\' WHERE `name`=\"{name}\"";

            // Update database
-            _database.Execute(update);
+            Apply(update);
        }

        List<FileEntry> GetFileList()
@@ -425,7 +447,7 @@ namespace Framework.Database
    {
        public FileEntry(string _path, State _state)
        {
-            path = _path;
+            path = _path.Replace(@"\", @"/");
            state = _state;
        }

@@ -456,4 +478,28 @@ namespace Framework.Database
        Apply,
        Rehash
    }
+
+    static class DBUpdaterUtil
+    {
+        static string mysqlExecutablePath;
+
+        public static string GetMySQLExecutable()
+        {
+            return mysqlExecutablePath;
+        }
+
+        public static bool CheckExecutable()
+        {
+            string mysqlExePath = ConfigMgr.GetDefaultValue("MySQLExecutable", "");
+            if (mysqlExePath.IsEmpty() || !File.Exists(mysqlExePath))
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Didn't find any executable MySQL binary at \'{mysqlExePath}\' or in path, correct the path in the *.conf (\"MySQLExecutable\").");
+                return false;
+            }
+
+            // Correct the path to the cli
+            mysqlExecutablePath = mysqlExePath;
+            return true;
+        }
+    }
 }
@@ -19,7 +19,7 @@ using Framework.Threading;
 using MySqlConnector;
 using System;
 using System.Collections.Generic;
-using System.IO;
+using System.Diagnostics;
 using System.Text;
 using System.Threading.Tasks;
 using System.Transactions;
@@ -30,22 +30,66 @@ namespace Framework.Database
    {
        public MySqlConnection GetConnection()
        {
-            return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0");
-        }
-
-        public MySqlConnection GetConnectionNoDatabase()
-        {
-            return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Allow User Variables=True;Pooling=true;");
+            return new MySqlConnection($"Server={Host};Port={PortOrSocket};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0");
        }

        public string Host;
-        public string Port;
+        public string PortOrSocket;
+        public bool UseSSL;
        public string Username;
        public string Password;
        public string Database;
        public int Poolsize;
    }

+    public struct DBVersion
+    {
+        public int Major { get; }
+        public int Minor { get; }
+        public int Build { get; }
+        public bool IsMariaDB { get; }
+
+        public DBVersion(int major, int minor, int build, bool isMariaDB)
+        {
+            Major = major;
+            Minor = minor;
+            Build = build;
+            IsMariaDB = isMariaDB;
+        }
+
+        public static DBVersion Parse(string versionString)
+        {
+            int start = 0;
+            int index = versionString.IndexOf('.', start);
+
+            string val = versionString.Substring(start, index - start).Trim();
+            int major = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            start = index + 1;
+            index = versionString.IndexOf('.', start);
+
+            val = versionString.Substring(start, index - start).Trim();
+            int minor = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            start = index + 1;
+            int i = start;
+            while (i < versionString.Length && Char.IsDigit(versionString, i))
+                i++;
+            val = versionString.Substring(start, i - start).Trim();
+            int build = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            return new DBVersion(major, minor, build, versionString.Contains("Maria"));
+        }
+
+        public bool IsAtLeast(int majorNum, int minorNum, int buildNum)
+        {
+            if (Major > majorNum) return true;
+            if (Major == majorNum && Minor > minorNum) return true;
+            if (Major == majorNum && Minor == minorNum && Build >= buildNum) return true;
+            return false;
+        }
+    }
+
    public abstract class MySqlBase<T>
    {
        Dictionary<T, string> _preparedQueries = new();
@@ -55,6 +99,8 @@ namespace Framework.Database
        DatabaseUpdater<T> _updater;
        DatabaseWorker<T> _worker;

+        DBVersion version;
+
        public MySqlErrorCode Initialize(MySqlConnectionInfo connectionInfo)
        {
            _connectionInfo = connectionInfo;
@@ -66,9 +112,9 @@ namespace Framework.Database
                using (var connection = _connectionInfo.GetConnection())
                {
                    connection.Open();
-                    Log.outInfo(LogFilter.SqlDriver, $"Connected to MySQL(ver: {connection.ServerVersion}) Database: {_connectionInfo.Database}");
-                    //Connection is good lets set some default values to help with updates.
-                    Apply($"SET GLOBAL max_allowed_packet=1073741824;");
+
+                    version = DBVersion.Parse(connection.ServerVersion);
+                    Log.outInfo(LogFilter.SqlDriver, $"Connected to DB: {_connectionInfo.Database} Server: {(version.IsMariaDB ? "MariaDB" : "MySQL")} Ver: {connection.ServerVersion}");
                    return MySqlErrorCode.None;
                }
            }
@@ -196,51 +242,85 @@ namespace Framework.Database
            return new PreparedStatement(_preparedQueries[statement]);
        }

-        public bool Apply(string sql)
+        public void ApplyFile(string path, bool useDatabase = true)
        {
-            try
-            {
-                using (var Connection = _connectionInfo.GetConnectionNoDatabase())
-                {
-                    Connection.Open();
-                    using (MySqlCommand cmd = Connection.CreateCommand())
-                    {
-                        cmd.CommandText = sql;
-                        cmd.ExecuteNonQuery();
-                        return true;
-                    }
-                }
-            }
-            catch (MySqlException ex)
-            {
-                HandleMySQLException(ex, sql);
-                return false;
-            }
-        }
+            // CLI Client connection info
+            string args = $"-h{_connectionInfo.Host} ";
+            args += $"-u{_connectionInfo.Username} ";

-        public bool ApplyFile(string path)
-        {
-            try
-            {
-                string query = File.ReadAllText(path);
-                if (query.IsEmpty())
-                    return false;
+            if (!_connectionInfo.Password.IsEmpty())
+                args += $"-p{_connectionInfo.Password} ";

-                using (var connection = _connectionInfo.GetConnection())
-                {
-                    connection.Open();
-                    using (MySqlCommand cmd = connection.CreateCommand())
-                    {
-                        cmd.CommandText = query;
-                        cmd.ExecuteNonQuery();
-                        return true;
-                    }
-                }
-            }
-            catch (MySqlException ex)
+            // Check if we want to connect through ip or socket (Unix only)
+            if (OperatingSystem.IsWindows())
            {
-                HandleMySQLException(ex, path);
-                return false;
+                if (_connectionInfo.Host == ".")
+                    args += "--protocol=PIPE ";
+                else
+                    args += $"-P{_connectionInfo.PortOrSocket} ";
+            }
+            else
+            {
+                if (!char.IsDigit(_connectionInfo.PortOrSocket[0]))
+                {
+                    // We can't check if host == "." here, because it is named localhost if socket option is enabled
+                    args += "-P0 ";
+                    args += "--protocol=SOCKET ";
+                    args += $"-S{_connectionInfo.PortOrSocket} ";
+                }
+                else
+                    // generic case
+                    args += $"-P{_connectionInfo.PortOrSocket} ";
+            }
+
+            // Set the default charset to utf8
+            args += "--default-character-set=utf8 ";
+
+            // Set max allowed packet to 1 GB
+            args += "--max-allowed-packet=1GB ";
+
+            if (!version.IsMariaDB && version.IsAtLeast(8, 0, 0))
+            {
+                if (_connectionInfo.UseSSL)
+                    args += "--ssl-mode=REQUIRED ";
+            }
+            else
+            {
+                if (_connectionInfo.UseSSL)
+                    args += "--ssl ";
+            }
+
+            // Execute sql file
+            args += "-e ";
+            args += "\"BEGIN; SOURCE \"" + path + "\"; COMMIT;\" ";
+
+            // Database
+            if (useDatabase && !_connectionInfo.Database.IsEmpty())
+                args += _connectionInfo.Database;
+
+            // Invokes a mysql process which doesn't leak credentials to logs
+            Process process = new();
+            process.StartInfo = new(DBUpdaterUtil.GetMySQLExecutable());
+            process.StartInfo.UseShellExecute = false;
+            process.StartInfo.RedirectStandardOutput = true;
+            process.StartInfo.RedirectStandardError = true;
+            process.StartInfo.CreateNoWindow = true;
+            process.StartInfo.Arguments = args;
+
+            process.Start();
+            Log.outInfo(LogFilter.SqlUpdates, process.StandardOutput.ReadToEnd());
+            Log.outError(LogFilter.SqlUpdates, process.StandardError.ReadToEnd());
+            process.WaitForExit();
+
+            if (process.ExitCode != 0)
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Applying of file \'{path}\' to database \'{GetDatabaseName()}\' failed!" +
+                    " If you are a user, please pull the latest revision from the repository. " +
+                    "Also make sure you have not applied any of the databases with your sql client. " +
+                    "You cannot use auto-update system and import sql files from CYpherCore repository with your sql client. " +
+                    "If you are a developer, please fix your sql query.");
+
+                throw new Exception("update failed");
            }
        }

@@ -34,31 +34,45 @@ LogsDir = "Logs"
 #    LoginDatabaseInfo
 #    WorldDatabaseInfo
 #    CharacterDatabaseInfo
-#        Description: Database connection settings for the world server.
+#	 HotfixDatabaseInfo
+#        Description: Database connection settings for the world server.     
+#        Example:
+#					  DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets.
+#					  DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets)
+#					  DatabaseInfo.Username = "username"
+#					  DatabaseInfo.Password = "password"
+#					  DatabaseInfo.Database = "auth"
+#					  DatabaseInfo.SSL = false
+#
+#		 Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini.

 LoginDatabaseInfo.Host = "127.0.0.1"
 LoginDatabaseInfo.Port = "3306"
 LoginDatabaseInfo.Username = "username"
 LoginDatabaseInfo.Password = "password"
 LoginDatabaseInfo.Database = "auth"
+LoginDatabaseInfo.SSL = false

 CharacterDatabaseInfo.Host = "127.0.0.1"
 CharacterDatabaseInfo.Port = "3306"
 CharacterDatabaseInfo.Username = "username"
 CharacterDatabaseInfo.Password = "password"
 CharacterDatabaseInfo.Database = "characters"
+CharacterDatabaseInfo.SSL = false

 WorldDatabaseInfo.Host = "127.0.0.1"
 WorldDatabaseInfo.Port = "3306"
 WorldDatabaseInfo.Username = "username"
 WorldDatabaseInfo.Password = "password"
 WorldDatabaseInfo.Database = "world"
+WorldDatabaseInfo.SSL = false

 HotfixDatabaseInfo.Host = "127.0.0.1"
 HotfixDatabaseInfo.Port = "3306"
 HotfixDatabaseInfo.Username = "username"
 HotfixDatabaseInfo.Password = "password"
 HotfixDatabaseInfo.Database = "hotfixes"
+HotfixDatabaseInfo.SSL = false

 #
 #    WorldServerPort
@@ -93,6 +107,16 @@ BindIP = "0.0.0.0"

 ThreadPool = 2

+#
+#    MySQLExecutable
+#        Description: The path to your MySQL CLI binary.
+#        Example:     "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
+#                     "mysql.exe"
+#                     "/usr/bin/mysql"
+#        Default:     ""
+
+MySQLExecutable = ""
+
 #
 ###################################################################################################