From 44e9ff979324820e2f3986223906bfce25976112 Mon Sep 17 00:00:00 2001 From: hondacrx Date: Sat, 24 Sep 2022 13:30:15 -0400 Subject: [PATCH] Core/Database: Fixes game crashes when loading into work due to hotfix_blob table having bad data. Now using mysqlcli to handle all file imports. Must fill out MySQLExecutable in config to use sql updater. --- Source/BNetServer/BNetServer.conf.dist | 24 ++- Source/Framework/Database/DatabaseLoader.cs | 62 +++++-- Source/Framework/Database/DatabaseUpdater.cs | 66 ++++++- Source/Framework/Database/MySqlBase.cs | 184 +++++++++++++------ Source/WorldServer/WorldServer.conf.dist | 26 ++- 5 files changed, 283 insertions(+), 79 deletions(-) diff --git a/Source/BNetServer/BNetServer.conf.dist b/Source/BNetServer/BNetServer.conf.dist index ea6e10b49..3be4d2e22 100644 --- a/Source/BNetServer/BNetServer.conf.dist +++ b/Source/BNetServer/BNetServer.conf.dist @@ -103,14 +103,23 @@ AllowLoggingIPAddressesInDatabase = 1 # MYSQL SETTINGS # # LoginDatabaseInfo -# Description: Database connection settings for the realm server. -# Example: "hostname;port;username;password;database" +# Description: Database connection settings for the realm server. +# Example: +# DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets. +# DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets) +# DatabaseInfo.Username = "username" +# DatabaseInfo.Password = "password" +# DatabaseInfo.Database = "auth" +# DatabaseInfo.SSL = false +# +# Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini. LoginDatabaseInfo.Host = "127.0.0.1" LoginDatabaseInfo.Port = "3306" LoginDatabaseInfo.Username = "username" LoginDatabaseInfo.Password = "password" LoginDatabaseInfo.Database = "auth" +LoginDatabaseInfo.SSL = false # # DBQueryWorkerInterval @@ -119,6 +128,17 @@ LoginDatabaseInfo.Database = "auth" DBQueryWorkerInterval = 1000 +# +# MySQLExecutable +# Description: The path to your MySQL CLI binary. +# If the path is left empty, built-in path from cmake is used. +# Example: "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe" +# "mysql.exe" +# "/usr/bin/mysql" +# Default: "" + +MySQLExecutable = "" + # ################################################################################################### diff --git a/Source/Framework/Database/DatabaseLoader.cs b/Source/Framework/Database/DatabaseLoader.cs index 01910dbff..f048e1f4d 100644 --- a/Source/Framework/Database/DatabaseLoader.cs +++ b/Source/Framework/Database/DatabaseLoader.cs @@ -19,6 +19,7 @@ using Framework.Configuration; using MySqlConnector; using System; using System.Collections.Generic; +using System.IO; namespace Framework.Database { @@ -38,10 +39,11 @@ namespace Framework.Database MySqlConnectionInfo connectionObject = new() { Host = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Host", ""), - Port = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""), + PortOrSocket = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Port", ""), Username = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Username", ""), Password = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Password", ""), - Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", "") + Database = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.Database", ""), + UseSSL = ConfigMgr.GetDefaultValue(baseDBName + "DatabaseInfo.SSL", false) }; var error = database.Initialize(connectionObject); @@ -50,17 +52,9 @@ namespace Framework.Database // Database does not exist if (error == MySqlErrorCode.UnknownDatabase && updatesEnabled && _autoSetup) { - Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: "); - - string answer = Console.ReadLine(); - if (string.IsNullOrEmpty(answer) || answer[0] != 'y') - return false; - - Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"..."); - string sqlString = $"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"; // Try to create the database and connect again if auto setup is enabled - if (database.Apply(sqlString) && database.Initialize(connectionObject) == MySqlErrorCode.None) - error = MySqlErrorCode.None; + if (CreateDatabase(connectionObject, database)) + error = database.Initialize(connectionObject); } // If the error wasn't handled quit @@ -69,9 +63,8 @@ namespace Framework.Database Log.outError(LogFilter.ServerLoading, $"\nDatabase {connectionObject.Database} NOT opened. There were errors opening the MySQL connections. Check your SQLErrors for specific errors."); return false; } - - Log.outInfo(LogFilter.ServerLoading, "Done."); } + return true; }); @@ -106,6 +99,47 @@ namespace Framework.Database }); } + public bool CreateDatabase(MySqlConnectionInfo connectionObject, MySqlBase database) + { + Log.outInfo(LogFilter.ServerLoading, $"Database \"{connectionObject.Database}\" does not exist, do you want to create it? [yes (default) / no]: "); + + string answer = Console.ReadLine(); + if (!answer.IsEmpty() && answer[0] != 'y') + return false; + + Log.outInfo(LogFilter.ServerLoading, $"Creating database \"{connectionObject.Database}\"..."); + + // Path of temp file + string temp = "create_table.sql"; + + // Create temporary query to use external MySQL CLi + try + { + using BinaryWriter binaryWriter = new(File.Open(temp, FileMode.Create, FileAccess.Write)); + binaryWriter.Write($"CREATE DATABASE `{connectionObject.Database}` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci"); + } + catch (Exception) + { + Log.outFatal(LogFilter.SqlUpdates, $"Failed to create temporary query file \"{temp}\"!"); + return false; + } + + try + { + database.ApplyFile(temp, false); + } + catch (Exception) + { + Log.outFatal(LogFilter.SqlUpdates, $"Failed to create database {database.GetDatabaseName()}! Does the user (named in *.conf) have `CREATE`, `ALTER`, `DROP`, `INSERT` and `DELETE` privileges on the MySQL server?"); + File.Delete(temp); + return false; + } + + Log.outInfo(LogFilter.SqlUpdates, "Done."); + File.Delete(temp); + return true; + } + public bool Load() { if (_updateFlags == 0) diff --git a/Source/Framework/Database/DatabaseUpdater.cs b/Source/Framework/Database/DatabaseUpdater.cs index 2d3cdb197..c1c3efd4b 100644 --- a/Source/Framework/Database/DatabaseUpdater.cs +++ b/Source/Framework/Database/DatabaseUpdater.cs @@ -38,6 +38,9 @@ namespace Framework.Database if (!result.IsEmpty() && !result.IsEmpty()) return true; + if (!DBUpdaterUtil.CheckExecutable()) + return false; + Log.outInfo(LogFilter.SqlUpdates, $"Database {_database.GetDatabaseName()} is empty, auto populating it..."); string path = GetSourceDirectory(); @@ -67,7 +70,14 @@ namespace Framework.Database // Update database Log.outInfo(LogFilter.SqlUpdates, $"Applying \'{fileName}\'..."); - _database.ApplyFile(path + fileName); + try + { + ApplyFile(path + fileName); + } + catch (Exception) + { + return false; + } Log.outInfo(LogFilter.SqlUpdates, $"Done Applying \'{fileName}\'"); return true; @@ -75,13 +85,16 @@ namespace Framework.Database public bool Update() { + if (!DBUpdaterUtil.CheckExecutable()) + return false; + Log.outInfo(LogFilter.SqlUpdates, $"Updating {_database.GetDatabaseName()} database..."); string sourceDirectory = GetSourceDirectory(); if (!Directory.Exists(sourceDirectory)) { - Log.outError(LogFilter.SqlUpdates, $"DBUpdater: Given source directory {sourceDirectory} does not exist, skipped!"); + Log.outError(LogFilter.SqlUpdates, $"DBUpdater: The given source directory {sourceDirectory} does not exist, change the path to the directory where your sql directory exists (for example c:\\source\\cyphercore). Shutting down."); return false; } @@ -255,19 +268,28 @@ namespace Framework.Database uint oldMSTime = Time.GetMSTime(); // Update database - if (!_database.ApplyFile(path)) - Log.outError(LogFilter.Sql, $"Update: {path} Failed. You need to apply it manually"); + ApplyFile(path); // Return time the query took to apply return Time.GetMSTimeDiffToNow(oldMSTime); } + void ApplyFile(string path) + { + _database.ApplyFile(path); + } + + void Apply(string query) + { + _database.Execute(query); + } + void UpdateEntry(AppliedFileEntry entry, uint speed) { string update = $"REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"{entry.Name}\", \"{entry.Hash}\", \'{entry.State}\', {speed})"; // Update database - _database.Execute(update); + Apply(update); } void RenameEntry(string from, string to) @@ -277,7 +299,7 @@ namespace Framework.Database string update = $"DELETE FROM `updates` WHERE `name`=\"{to}\""; // Update database - _database.Execute(update); + Apply(update); } // Rename @@ -285,7 +307,7 @@ namespace Framework.Database string update = $"UPDATE `updates` SET `name`=\"{to}\" WHERE `name`=\"{from}\""; // Update database - _database.Execute(update); + Apply(update); } } @@ -307,7 +329,7 @@ namespace Framework.Database update += ")"; // Update database - _database.Execute(update); + Apply(update); } void UpdateState(string name, State state) @@ -315,7 +337,7 @@ namespace Framework.Database string update = $"UPDATE `updates` SET `state`=\'{state}\' WHERE `name`=\"{name}\""; // Update database - _database.Execute(update); + Apply(update); } List GetFileList() @@ -425,7 +447,7 @@ namespace Framework.Database { public FileEntry(string _path, State _state) { - path = _path; + path = _path.Replace(@"\", @"/"); state = _state; } @@ -456,4 +478,28 @@ namespace Framework.Database Apply, Rehash } + + static class DBUpdaterUtil + { + static string mysqlExecutablePath; + + public static string GetMySQLExecutable() + { + return mysqlExecutablePath; + } + + public static bool CheckExecutable() + { + string mysqlExePath = ConfigMgr.GetDefaultValue("MySQLExecutable", ""); + if (mysqlExePath.IsEmpty() || !File.Exists(mysqlExePath)) + { + Log.outFatal(LogFilter.SqlUpdates, $"Didn't find any executable MySQL binary at \'{mysqlExePath}\' or in path, correct the path in the *.conf (\"MySQLExecutable\")."); + return false; + } + + // Correct the path to the cli + mysqlExecutablePath = mysqlExePath; + return true; + } + } } diff --git a/Source/Framework/Database/MySqlBase.cs b/Source/Framework/Database/MySqlBase.cs index 68289450a..0468c1249 100644 --- a/Source/Framework/Database/MySqlBase.cs +++ b/Source/Framework/Database/MySqlBase.cs @@ -19,7 +19,7 @@ using Framework.Threading; using MySqlConnector; using System; using System.Collections.Generic; -using System.IO; +using System.Diagnostics; using System.Text; using System.Threading.Tasks; using System.Transactions; @@ -30,22 +30,66 @@ namespace Framework.Database { public MySqlConnection GetConnection() { - return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0"); - } - - public MySqlConnection GetConnectionNoDatabase() - { - return new MySqlConnection($"Server={Host};Port={Port};User Id={Username};Password={Password};Allow User Variables=True;Pooling=true;"); + return new MySqlConnection($"Server={Host};Port={PortOrSocket};User Id={Username};Password={Password};Database={Database};Allow User Variables=True;Pooling=true;ConnectionIdleTimeout=1800;Command Timeout=0"); } public string Host; - public string Port; + public string PortOrSocket; + public bool UseSSL; public string Username; public string Password; public string Database; public int Poolsize; } + public struct DBVersion + { + public int Major { get; } + public int Minor { get; } + public int Build { get; } + public bool IsMariaDB { get; } + + public DBVersion(int major, int minor, int build, bool isMariaDB) + { + Major = major; + Minor = minor; + Build = build; + IsMariaDB = isMariaDB; + } + + public static DBVersion Parse(string versionString) + { + int start = 0; + int index = versionString.IndexOf('.', start); + + string val = versionString.Substring(start, index - start).Trim(); + int major = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo); + + start = index + 1; + index = versionString.IndexOf('.', start); + + val = versionString.Substring(start, index - start).Trim(); + int minor = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo); + + start = index + 1; + int i = start; + while (i < versionString.Length && Char.IsDigit(versionString, i)) + i++; + val = versionString.Substring(start, i - start).Trim(); + int build = Convert.ToInt32(val, System.Globalization.NumberFormatInfo.InvariantInfo); + + return new DBVersion(major, minor, build, versionString.Contains("Maria")); + } + + public bool IsAtLeast(int majorNum, int minorNum, int buildNum) + { + if (Major > majorNum) return true; + if (Major == majorNum && Minor > minorNum) return true; + if (Major == majorNum && Minor == minorNum && Build >= buildNum) return true; + return false; + } + } + public abstract class MySqlBase { Dictionary _preparedQueries = new(); @@ -55,6 +99,8 @@ namespace Framework.Database DatabaseUpdater _updater; DatabaseWorker _worker; + DBVersion version; + public MySqlErrorCode Initialize(MySqlConnectionInfo connectionInfo) { _connectionInfo = connectionInfo; @@ -66,9 +112,9 @@ namespace Framework.Database using (var connection = _connectionInfo.GetConnection()) { connection.Open(); - Log.outInfo(LogFilter.SqlDriver, $"Connected to MySQL(ver: {connection.ServerVersion}) Database: {_connectionInfo.Database}"); - //Connection is good lets set some default values to help with updates. - Apply($"SET GLOBAL max_allowed_packet=1073741824;"); + + version = DBVersion.Parse(connection.ServerVersion); + Log.outInfo(LogFilter.SqlDriver, $"Connected to DB: {_connectionInfo.Database} Server: {(version.IsMariaDB ? "MariaDB" : "MySQL")} Ver: {connection.ServerVersion}"); return MySqlErrorCode.None; } } @@ -196,51 +242,85 @@ namespace Framework.Database return new PreparedStatement(_preparedQueries[statement]); } - public bool Apply(string sql) + public void ApplyFile(string path, bool useDatabase = true) { - try - { - using (var Connection = _connectionInfo.GetConnectionNoDatabase()) - { - Connection.Open(); - using (MySqlCommand cmd = Connection.CreateCommand()) - { - cmd.CommandText = sql; - cmd.ExecuteNonQuery(); - return true; - } - } - } - catch (MySqlException ex) - { - HandleMySQLException(ex, sql); - return false; - } - } + // CLI Client connection info + string args = $"-h{_connectionInfo.Host} "; + args += $"-u{_connectionInfo.Username} "; - public bool ApplyFile(string path) - { - try - { - string query = File.ReadAllText(path); - if (query.IsEmpty()) - return false; + if (!_connectionInfo.Password.IsEmpty()) + args += $"-p{_connectionInfo.Password} "; - using (var connection = _connectionInfo.GetConnection()) - { - connection.Open(); - using (MySqlCommand cmd = connection.CreateCommand()) - { - cmd.CommandText = query; - cmd.ExecuteNonQuery(); - return true; - } - } - } - catch (MySqlException ex) + // Check if we want to connect through ip or socket (Unix only) + if (OperatingSystem.IsWindows()) { - HandleMySQLException(ex, path); - return false; + if (_connectionInfo.Host == ".") + args += "--protocol=PIPE "; + else + args += $"-P{_connectionInfo.PortOrSocket} "; + } + else + { + if (!char.IsDigit(_connectionInfo.PortOrSocket[0])) + { + // We can't check if host == "." here, because it is named localhost if socket option is enabled + args += "-P0 "; + args += "--protocol=SOCKET "; + args += $"-S{_connectionInfo.PortOrSocket} "; + } + else + // generic case + args += $"-P{_connectionInfo.PortOrSocket} "; + } + + // Set the default charset to utf8 + args += "--default-character-set=utf8 "; + + // Set max allowed packet to 1 GB + args += "--max-allowed-packet=1GB "; + + if (!version.IsMariaDB && version.IsAtLeast(8, 0, 0)) + { + if (_connectionInfo.UseSSL) + args += "--ssl-mode=REQUIRED "; + } + else + { + if (_connectionInfo.UseSSL) + args += "--ssl "; + } + + // Execute sql file + args += "-e "; + args += "\"BEGIN; SOURCE \"" + path + "\"; COMMIT;\" "; + + // Database + if (useDatabase && !_connectionInfo.Database.IsEmpty()) + args += _connectionInfo.Database; + + // Invokes a mysql process which doesn't leak credentials to logs + Process process = new(); + process.StartInfo = new(DBUpdaterUtil.GetMySQLExecutable()); + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = true; + process.StartInfo.Arguments = args; + + process.Start(); + Log.outInfo(LogFilter.SqlUpdates, process.StandardOutput.ReadToEnd()); + Log.outError(LogFilter.SqlUpdates, process.StandardError.ReadToEnd()); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + Log.outFatal(LogFilter.SqlUpdates, $"Applying of file \'{path}\' to database \'{GetDatabaseName()}\' failed!" + + " If you are a user, please pull the latest revision from the repository. " + + "Also make sure you have not applied any of the databases with your sql client. " + + "You cannot use auto-update system and import sql files from CYpherCore repository with your sql client. " + + "If you are a developer, please fix your sql query."); + + throw new Exception("update failed"); } } diff --git a/Source/WorldServer/WorldServer.conf.dist b/Source/WorldServer/WorldServer.conf.dist index 04f455a9f..6d9c54fa4 100644 --- a/Source/WorldServer/WorldServer.conf.dist +++ b/Source/WorldServer/WorldServer.conf.dist @@ -34,31 +34,45 @@ LogsDir = "Logs" # LoginDatabaseInfo # WorldDatabaseInfo # CharacterDatabaseInfo -# Description: Database connection settings for the world server. +# HotfixDatabaseInfo +# Description: Database connection settings for the world server. +# Example: +# DatabaseInfo.Host = hostname OR . for named-pipes/unit_sockets. +# DatabaseInfo.Port = port_number OR some_number (named_pipes) OR path to unix socket (unix_sockets) +# DatabaseInfo.Username = "username" +# DatabaseInfo.Password = "password" +# DatabaseInfo.Database = "auth" +# DatabaseInfo.SSL = false +# +# Note: When using NamedPipes on windows you must set "enable-named-pipe" to [mysqld] section my.ini. LoginDatabaseInfo.Host = "127.0.0.1" LoginDatabaseInfo.Port = "3306" LoginDatabaseInfo.Username = "username" LoginDatabaseInfo.Password = "password" LoginDatabaseInfo.Database = "auth" +LoginDatabaseInfo.SSL = false CharacterDatabaseInfo.Host = "127.0.0.1" CharacterDatabaseInfo.Port = "3306" CharacterDatabaseInfo.Username = "username" CharacterDatabaseInfo.Password = "password" CharacterDatabaseInfo.Database = "characters" +CharacterDatabaseInfo.SSL = false WorldDatabaseInfo.Host = "127.0.0.1" WorldDatabaseInfo.Port = "3306" WorldDatabaseInfo.Username = "username" WorldDatabaseInfo.Password = "password" WorldDatabaseInfo.Database = "world" +WorldDatabaseInfo.SSL = false HotfixDatabaseInfo.Host = "127.0.0.1" HotfixDatabaseInfo.Port = "3306" HotfixDatabaseInfo.Username = "username" HotfixDatabaseInfo.Password = "password" HotfixDatabaseInfo.Database = "hotfixes" +HotfixDatabaseInfo.SSL = false # # WorldServerPort @@ -93,6 +107,16 @@ BindIP = "0.0.0.0" ThreadPool = 2 +# +# MySQLExecutable +# Description: The path to your MySQL CLI binary. +# Example: "C:/Program Files/MySQL/MySQL Server 8.0/bin/mysql.exe" +# "mysql.exe" +# "/usr/bin/mysql" +# Default: "" + +MySQLExecutable = "" + # ###################################################################################################