Core/Database: Fixes game crashes when loading into work due to hotfix_blob table having bad data. Now using mysqlcli to handle all file imports. Must fill out MySQLExecutable in config to use sql updater.

Source/BNetServer/BNetServer.conf.dist

@@ -104,13 +104,22 @@ AllowLoggingIPAddressesInDatabase = 1
 #
 #    LoginDatabaseInfo
 #        Description: Database connection settings for the realm server.    
-#        Example:     "hostname;port;username;password;database"
+#        Example:
+#					  DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets.
+#					  DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets)
+#					  DatabaseInfo.Username = "username"
+#					  DatabaseInfo.Password = "password"
+#					  DatabaseInfo.Database = "auth"
+#					  DatabaseInfo.SSL = false
+#
+#		 Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini.
 
 LoginDatabaseInfo.Host = "127.0.0.1"
 LoginDatabaseInfo.Port = "3306"
 LoginDatabaseInfo.Username = "username"
 LoginDatabaseInfo.Password = "password"
 LoginDatabaseInfo.Database = "auth"
+LoginDatabaseInfo.SSL = false
 
 #
 #    DBQueryWorkerInterval
@@ -119,6 +128,17 @@ LoginDatabaseInfo.Database = "auth"
 
 DBQueryWorkerInterval = 1000
 
+#
+#    MySQLExecutable
+#        Description: The path to your MySQL CLI binary.
+#                     If the path is left empty, built-in path from cmake is used.
+#        Example:     "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
+#                     "mysql.exe"
+#                     "/usr/bin/mysql"
+#        Default:     ""
+
+MySQLExecutable = ""
+
 #
 ###################################################################################################
 

Source/Framework/Database/DatabaseLoader.cs

@@ -19,6 +19,7 @@ using Framework.Configuration;
 using MySqlConnector;
 using System;
 using System.Collections.Generic;
+using System.IO;
 
 namespace Framework.Database
 {
@@ -38,10 +39,11 @@ namespace Framework.Database
                 MySqlConnectionInfo connectionObject = new()
                 {
                     Host = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Host", ""),
-                    Port = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""),
+                    PortOrSocket = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""),
                     Username = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Username", ""),
                     Password = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Password", ""),
-                    Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", "")
+                    Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", ""),
+                    UseSSL = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.SSL", false)
                 };
 
                 var error = database.Initialize(connectionObject);
@@ -50,17 +52,9 @@ namespace Framework.Database
                     // Database does not exist
                     if (error == MySqlErrorCode.UnknownDatabase && updatesEnabled && _autoSetup)
                     {
-                        Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: ");
-
-                        string answer = Console.ReadLine();
-                        if (string.IsNullOrEmpty(answer) || answer[0] != 'y')
-                            return false;
-
-                        Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"...");
-                        string sqlString = $"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci";
                         // Try to create the database and connect again if auto setup is enabled
-                        if (database.Apply(sqlString) && database.Initialize(connectionObject) == MySqlErrorCode.None)
+                        if (CreateDatabase(connectionObject, database))
-                            error = MySqlErrorCode.None;
+                            error = database.Initialize(connectionObject);
                     }
 
                     // If the error wasn't handled quit
@@ -69,9 +63,8 @@ namespace Framework.Database
                         Log.outError(LogFilter.ServerLoading, $"\nDatabase {connectionObject.Database} NOT opened. There were errors opening the MySQL connections. Check your SQLErrors for specific errors.");
                         return false;
                     }
-
-                    Log.outInfo(LogFilter.ServerLoading, "Done.");
                 }
+
                 return true;
             });
 
@@ -106,6 +99,47 @@ namespace Framework.Database
             });
         }
 
+        public bool CreateDatabase<T>(MySqlConnectionInfo connectionObject, MySqlBase<T> database)
+        {
+            Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: ");
+
+            string answer = Console.ReadLine();
+            if (!answer.IsEmpty() && answer[0] != 'y')
+                return false;
+
+            Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"...");
+
+            // Path of temp file
+            string temp = "create_table.sql";
+
+            // Create temporary query to use external MySQL CLi
+            try
+            {
+                using BinaryWriter binaryWriter = new(File.Open(temp, FileMode.Create, FileAccess.Write));
+                binaryWriter.Write($"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci");
+            }
+            catch (Exception)
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Failed to create temporary query file \"{temp}\"!");
+                return false;
+            }
+
+            try
+            {
+                database.ApplyFile(temp, false);
+            }
+            catch (Exception)
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Failed to create database {database.GetDatabaseName()}! Does the user (named in *.conf) have `CREATE`, `ALTER`, `DROP`, `INSERT` and `DELETE` privileges on the MySQL server?");
+                File.Delete(temp);
+                return false;
+            }
+
+            Log.outInfo(LogFilter.SqlUpdates, "Done.");
+            File.Delete(temp);
+            return true;
+        }
+
         public bool Load()
         {
             if (_updateFlags == 0)

Source/Framework/Database/DatabaseUpdater.cs

@@ -38,6 +38,9 @@ namespace Framework.Database
             if (!result.IsEmpty() && !result.IsEmpty())
                 return true;
 
+            if (!DBUpdaterUtil.CheckExecutable())
+                return false;
+
             Log.outInfo(LogFilter.SqlUpdates, $"Database {_database.GetDatabaseName()} is empty, auto populating it...");
 
             string path = GetSourceDirectory();
@@ -67,7 +70,14 @@ namespace Framework.Database
 
             // Update database
             Log.outInfo(LogFilter.SqlUpdates, $"Applying \'{fileName}\'...");
-            _database.ApplyFile(path + fileName);
+            try
+            {
+                ApplyFile(path + fileName);
+            }
+            catch (Exception)
+            {
+                return false;
+            }
 
             Log.outInfo(LogFilter.SqlUpdates, $"Done Applying \'{fileName}\'");
             return true;
@@ -75,13 +85,16 @@ namespace Framework.Database
 
         public bool Update()
         {
+            if (!DBUpdaterUtil.CheckExecutable())
+                return false;
+
             Log.outInfo(LogFilter.SqlUpdates, $"Updating {_database.GetDatabaseName()} database...");
 
             string sourceDirectory = GetSourceDirectory();
 
             if (!Directory.Exists(sourceDirectory))
             {
-                Log.outError(LogFilter.SqlUpdates, $"DBUpdater: Given source directory {sourceDirectory} does not exist, skipped!");
+                Log.outError(LogFilter.SqlUpdates, $"DBUpdater: The given source directory {sourceDirectory} does not exist, change the path to the directory where your sql directory exists (for example c:\\source\\cyphercore). Shutting down.");
                 return false;
             }
 
@@ -255,19 +268,28 @@ namespace Framework.Database
             uint oldMSTime = Time.GetMSTime();
 
             // Update database
-            if (!_database.ApplyFile(path))
+            ApplyFile(path);
-                Log.outError(LogFilter.Sql, $"Update: {path} Failed. You need to apply it manually");
 
             // Return time the query took to apply
             return Time.GetMSTimeDiffToNow(oldMSTime);
         }
 
+        void ApplyFile(string path)
+        {
+            _database.ApplyFile(path);
+        }
+        
+        void Apply(string query)
+        {
+            _database.Execute(query);
+        }
+
         void UpdateEntry(AppliedFileEntry entry, uint speed)
         {
             string update = $"REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"{entry.Name}\", \"{entry.Hash}\", \'{entry.State}\', {speed})";
 
             // Update database
-            _database.Execute(update);
+            Apply(update);
         }
 
         void RenameEntry(string from, string to)
@@ -277,7 +299,7 @@ namespace Framework.Database
                 string update = $"DELETE FROM `updates` WHERE `name`=\"{to}\"";
 
                 // Update database
-                _database.Execute(update);
+                Apply(update);
             }
 
             // Rename
@@ -285,7 +307,7 @@ namespace Framework.Database
                 string update = $"UPDATE `updates` SET `name`=\"{to}\" WHERE `name`=\"{from}\"";
 
                 // Update database
-                _database.Execute(update);
+                Apply(update);
             }
         }
 
@@ -307,7 +329,7 @@ namespace Framework.Database
             update += ")";
 
             // Update database
-            _database.Execute(update);
+            Apply(update);
         }
 
         void UpdateState(string name, State state)
@@ -315,7 +337,7 @@ namespace Framework.Database
             string update = $"UPDATE `updates` SET `state`=\'{state}\' WHERE `name`=\"{name}\"";
 
             // Update database
-            _database.Execute(update);
+            Apply(update);
         }
 
         List<FileEntry> GetFileList()
@@ -425,7 +447,7 @@ namespace Framework.Database
     {
         public FileEntry(string _path, State _state)
         {
-            path = _path;
+            path = _path.Replace(@"\", @"/");
             state = _state;
         }
 
@@ -456,4 +478,28 @@ namespace Framework.Database
         Apply,
         Rehash
     }
+
+    static class DBUpdaterUtil
+    {
+        static string mysqlExecutablePath;
+
+        public static string GetMySQLExecutable()
+        {
+            return mysqlExecutablePath;
+        }
+
+        public static bool CheckExecutable()
+        {
+            string mysqlExePath = ConfigMgr.GetDefaultValue("MySQLExecutable", "");
+            if (mysqlExePath.IsEmpty() || !File.Exists(mysqlExePath))
+            {
+                Log.outFatal(LogFilter.SqlUpdates, $"Didn't find any executable MySQL binary at \'{mysqlExePath}\' or in path, correct the path in the *.conf (\"MySQLExecutable\").");
+                return false;
+            }
+
+            // Correct the path to the cli
+            mysqlExecutablePath = mysqlExePath;
+            return true;
+        }
+    }
 }

Source/Framework/Database/MySqlBase.cs

@@ -19,7 +19,7 @@ using Framework.Threading;
 using MySqlConnector;
 using System;
 using System.Collections.Generic;
-using System.IO;
+using System.Diagnostics;
 using System.Text;
 using System.Threading.Tasks;
 using System.Transactions;
@@ -30,22 +30,66 @@ namespace Framework.Database
     {
         public MySqlConnection GetConnection()
         {
-            return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0");
+            return new MySqlConnection($"Server={Host};Port={PortOrSocket};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0");
-        }
-
-        public MySqlConnection GetConnectionNoDatabase()
-        {
-            return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Allow User Variables=True;Pooling=true;");
         }
 
         public string Host;
-        public string Port;
+        public string PortOrSocket;
+        public bool UseSSL;
         public string Username;
         public string Password;
         public string Database;
         public int Poolsize;
     }
 
+    public struct DBVersion
+    {
+        public int Major { get; }
+        public int Minor { get; }
+        public int Build { get; }
+        public bool IsMariaDB { get; }
+
+        public DBVersion(int major, int minor, int build, bool isMariaDB)
+        {
+            Major = major;
+            Minor = minor;
+            Build = build;
+            IsMariaDB = isMariaDB;
+        }
+
+        public static DBVersion Parse(string versionString)
+        {
+            int start = 0;
+            int index = versionString.IndexOf('.', start);
+
+            string val = versionString.Substring(start, index - start).Trim();
+            int major = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            start = index + 1;
+            index = versionString.IndexOf('.', start);
+
+            val = versionString.Substring(start, index - start).Trim();
+            int minor = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            start = index + 1;
+            int i = start;
+            while (i < versionString.Length && Char.IsDigit(versionString, i))
+                i++;
+            val = versionString.Substring(start, i - start).Trim();
+            int build = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo);
+
+            return new DBVersion(major, minor, build, versionString.Contains("Maria"));
+        }
+
+        public bool IsAtLeast(int majorNum, int minorNum, int buildNum)
+        {
+            if (Major > majorNum) return true;
+            if (Major == majorNum && Minor > minorNum) return true;
+            if (Major == majorNum && Minor == minorNum && Build >= buildNum) return true;
+            return false;
+        }
+    }
+
     public abstract class MySqlBase<T>
     {
         Dictionary<T, string> _preparedQueries = new();
@@ -55,6 +99,8 @@ namespace Framework.Database
         DatabaseUpdater<T> _updater;
         DatabaseWorker<T> _worker;
 
+        DBVersion version;
+
         public MySqlErrorCode Initialize(MySqlConnectionInfo connectionInfo)
         {
             _connectionInfo = connectionInfo;
@@ -66,9 +112,9 @@ namespace Framework.Database
                 using (var connection = _connectionInfo.GetConnection())
                 {
                     connection.Open();
-                    Log.outInfo(LogFilter.SqlDriver, $"Connected to MySQL(ver: {connection.ServerVersion}) Database: {_connectionInfo.Database}");
+
-                    //Connection is good lets set some default values to help with updates.
+                    version = DBVersion.Parse(connection.ServerVersion);
-                    Apply($"SET GLOBAL max_allowed_packet=1073741824;");
+                    Log.outInfo(LogFilter.SqlDriver, $"Connected to DB: {_connectionInfo.Database} Server: {(version.IsMariaDB ? "MariaDB" : "MySQL")} Ver: {connection.ServerVersion}");
                     return MySqlErrorCode.None;
                 }
             }
@@ -196,51 +242,85 @@ namespace Framework.Database
             return new PreparedStatement(_preparedQueries[statement]);
         }
 
-        public bool Apply(string sql)
+        public void ApplyFile(string path, bool useDatabase = true)
         {
-            try
+            // CLI Client connection info
+            string args = $"-h{_connectionInfo.Host} ";
+            args += $"-u{_connectionInfo.Username} ";
+
+            if (!_connectionInfo.Password.IsEmpty())
+                args += $"-p{_connectionInfo.Password} ";
+
+            // Check if we want to connect through ip or socket (Unix only)
+            if (OperatingSystem.IsWindows())
             {
-                using (var Connection = _connectionInfo.GetConnectionNoDatabase())
+                if (_connectionInfo.Host == ".")
-                {
+                    args += "--protocol=PIPE ";
-                    Connection.Open();
+                else
-                    using (MySqlCommand cmd = Connection.CreateCommand())
+                    args += $"-P{_connectionInfo.PortOrSocket} ";
-                    {
-                        cmd.CommandText = sql;
-                        cmd.ExecuteNonQuery();
-                        return true;
             }
-                }
+            else
-            }
-            catch (MySqlException ex)
             {
-                HandleMySQLException(ex, sql);
+                if (!char.IsDigit(_connectionInfo.PortOrSocket[0]))
-                return false;
+                {
+                    // We can't check if host == "." here, because it is named localhost if socket option is enabled
+                    args += "-P0 ";
+                    args += "--protocol=SOCKET ";
+                    args += $"-S{_connectionInfo.PortOrSocket} ";
                 }
+                else
+                    // generic case
+                    args += $"-P{_connectionInfo.PortOrSocket} ";
             }
 
-        public bool ApplyFile(string path)
+            // Set the default charset to utf8
-        {
+            args += "--default-character-set=utf8 ";
-            try
-            {
-                string query = File.ReadAllText(path);
-                if (query.IsEmpty())
-                    return false;
 
-                using (var connection = _connectionInfo.GetConnection())
+            // Set max allowed packet to 1 GB
+            args += "--max-allowed-packet=1GB ";
+
+            if (!version.IsMariaDB && version.IsAtLeast(8, 0, 0))
             {
-                    connection.Open();
+                if (_connectionInfo.UseSSL)
-                    using (MySqlCommand cmd = connection.CreateCommand())
+                    args += "--ssl-mode=REQUIRED ";
+            }
+            else
             {
-                        cmd.CommandText = query;
+                if (_connectionInfo.UseSSL)
-                        cmd.ExecuteNonQuery();
+                    args += "--ssl ";
-                        return true;
             }
-                }
+
-            }
+            // Execute sql file
-            catch (MySqlException ex)
+            args += "-e ";
+            args += "\"BEGIN; SOURCE \"" + path + "\"; COMMIT;\" ";
+
+            // Database
+            if (useDatabase && !_connectionInfo.Database.IsEmpty())
+                args += _connectionInfo.Database;
+
+            // Invokes a mysql process which doesn't leak credentials to logs
+            Process process = new();
+            process.StartInfo = new(DBUpdaterUtil.GetMySQLExecutable());
+            process.StartInfo.UseShellExecute = false;
+            process.StartInfo.RedirectStandardOutput = true;
+            process.StartInfo.RedirectStandardError = true;
+            process.StartInfo.CreateNoWindow = true;
+            process.StartInfo.Arguments = args;
+
+            process.Start();
+            Log.outInfo(LogFilter.SqlUpdates, process.StandardOutput.ReadToEnd());
+            Log.outError(LogFilter.SqlUpdates, process.StandardError.ReadToEnd());
+            process.WaitForExit();
+
+            if (process.ExitCode != 0)
             {
-                HandleMySQLException(ex, path);
+                Log.outFatal(LogFilter.SqlUpdates, $"Applying of file \'{path}\' to database \'{GetDatabaseName()}\' failed!" +
-                return false;
+                    " If you are a user, please pull the latest revision from the repository. " +
+                    "Also make sure you have not applied any of the databases with your sql client. " +
+                    "You cannot use auto-update system and import sql files from CYpherCore repository with your sql client. " +
+                    "If you are a developer, please fix your sql query.");
+
+                throw new Exception("update failed");
             }
         }
 

Source/WorldServer/WorldServer.conf.dist

@@ -34,31 +34,45 @@ LogsDir = "Logs"
 #    LoginDatabaseInfo
 #    WorldDatabaseInfo
 #    CharacterDatabaseInfo
+#	 HotfixDatabaseInfo
 #        Description: Database connection settings for the world server.     
+#        Example:
+#					  DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets.
+#					  DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets)
+#					  DatabaseInfo.Username = "username"
+#					  DatabaseInfo.Password = "password"
+#					  DatabaseInfo.Database = "auth"
+#					  DatabaseInfo.SSL = false
+#
+#		 Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini.
 
 LoginDatabaseInfo.Host = "127.0.0.1"
 LoginDatabaseInfo.Port = "3306"
 LoginDatabaseInfo.Username = "username"
 LoginDatabaseInfo.Password = "password"
 LoginDatabaseInfo.Database = "auth"
+LoginDatabaseInfo.SSL = false
 
 CharacterDatabaseInfo.Host = "127.0.0.1"
 CharacterDatabaseInfo.Port = "3306"
 CharacterDatabaseInfo.Username = "username"
 CharacterDatabaseInfo.Password = "password"
 CharacterDatabaseInfo.Database = "characters"
+CharacterDatabaseInfo.SSL = false
 
 WorldDatabaseInfo.Host = "127.0.0.1"
 WorldDatabaseInfo.Port = "3306"
 WorldDatabaseInfo.Username = "username"
 WorldDatabaseInfo.Password = "password"
 WorldDatabaseInfo.Database = "world"
+WorldDatabaseInfo.SSL = false
 
 HotfixDatabaseInfo.Host = "127.0.0.1"
 HotfixDatabaseInfo.Port = "3306"
 HotfixDatabaseInfo.Username = "username"
 HotfixDatabaseInfo.Password = "password"
 HotfixDatabaseInfo.Database = "hotfixes"
+HotfixDatabaseInfo.SSL = false
 
 #
 #    WorldServerPort
@@ -93,6 +107,16 @@ BindIP = "0.0.0.0"
 
 ThreadPool = 2
 
+#
+#    MySQLExecutable
+#        Description: The path to your MySQL CLI binary.
+#        Example:     "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe"
+#                     "mysql.exe"
+#                     "/usr/bin/mysql"
+#        Default:     ""
+
+MySQLExecutable = ""
+
 #
 ###################################################################################################