Core/Chat: Load chat language data from db2 and implement serverside chat translations

Port From (https://github.com/TrinityCore/TrinityCore/commit/3b578fc6b95bfc812962fcef7930d8646016d184)

Source/Game/Chat/Commands/LearnCommands.cs

@@ -95,8 +95,11 @@ namespace Game.Chat.Commands
             static bool HandleLearnAllLangCommand(StringArguments args, CommandHandler handler)
             {
                 // skipping UNIVERSAL language (0)
-                for (byte i = 1; i < Enum.GetValues(typeof(Language)).Length; ++i)
+                Global.LanguageMgr.ForEachLanguage((lang, languageDesc) =>
-                    handler.GetSession().GetPlayer().LearnSpell(ObjectManager.lang_description[i].spell_id, false);
+                {
+                    handler.GetSession().GetPlayer().LearnSpell(languageDesc.SpellId, false);
+                    return true;
+                });
 
                 handler.SendSysMessage(CypherStrings.CommandLearnAllLang);
                 return true;

Source/Game/Chat/LanguageManager.cs

@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2012-2020 CypherCore <http://github.com/CypherCore>
+ * 
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Game.DataStorage;
+using Framework.Constants;
+using Game.Spells;
+using Framework.Dynamic;
+using Framework.Collections;
+
+namespace Game.Chat
+{
+    public class LanguageManager : Singleton<LanguageManager>
+    {
+        Dictionary<uint, LanguageDesc> _langsMap = new();
+        MultiMap<Tuple<uint, byte>, string> _wordsMap = new();
+
+        LanguageManager() { }
+
+        public void LoadSpellEffectLanguage(SpellEffectRecord spellEffect)
+        {
+            Cypher.Assert(spellEffect != null && spellEffect.Effect == (uint)SpellEffectName.Language);
+
+            uint languageId = (uint)spellEffect.EffectMiscValue[0];
+            if (!_langsMap.TryGetValue(languageId, out LanguageDesc desc))
+            {
+                Log.outWarn(LogFilter.Spells, $"LoadSpellEffectLanguage called on Spell {spellEffect.SpellID} with language {languageId} which does not exist in Language.db2!");
+                return;
+            }
+
+            desc.SpellId = spellEffect.SpellID;
+        }
+
+        uint GetSpellLanguage(uint spellId)
+        {
+            SpellInfo spellInfo = Global.SpellMgr.GetSpellInfo(spellId, Difficulty.None);
+            if (spellInfo != null)
+            {
+                var effects = spellInfo.GetEffects();
+                if (effects.Length != 1 || effects[0].Effect != SpellEffectName.Language)
+                    Log.outWarn(LogFilter.Spells, $"Invalid language spell {spellId}. Expected 1 effect with SPELL_EFFECT_LANGUAGE");
+                else
+                    return (uint)effects[0].MiscValue;
+            }
+            return 0;
+        }
+
+        bool IsRelevantLanguageSkill(SkillLineRecord skillLineEntry)
+        {
+            if (skillLineEntry == null)
+                return false;
+
+            SkillRaceClassInfoRecord entry = Global.DB2Mgr.GetAvailableSkillRaceClassInfo(skillLineEntry.Id);
+            return entry != null;
+        }
+
+        public void LoadLanguages()
+        {
+            uint oldMSTime = Time.GetMSTime();
+
+            // Load languages from Languages.db2. Just the id, we don't need the name
+            foreach (LanguagesRecord langEntry in CliDB.LanguagesStorage.Values)
+                _langsMap.Add(langEntry.Id, new LanguageDesc());
+
+            // Add the languages used in code in case they don't exist
+            _langsMap.Add((uint)Language.Universal, new LanguageDesc());
+            _langsMap.Add((uint)Language.Addon, new LanguageDesc());
+            _langsMap.Add((uint)Language.AddonLogged, new LanguageDesc());
+
+            // Log load time
+            Log.outInfo(LogFilter.ServerLoading, $"Loaded {_langsMap.Count} languages in {Time.GetMSTimeDiffToNow(oldMSTime)} ms");
+        }
+
+        public void LoadLanguagesSkills()
+        {
+            uint oldMSTime = Time.GetMSTime();
+
+            uint count = 0;
+            foreach (SkillLineRecord skillLineEntry in CliDB.SkillLineStorage.Values)
+            {
+                if (skillLineEntry.CategoryID != SkillCategory.Languages)
+                    continue;
+
+                if (!IsRelevantLanguageSkill(skillLineEntry))
+                    continue;
+
+                var skills = Global.DB2Mgr.GetSkillLineAbilitiesBySkill(skillLineEntry.Id);
+
+                // We're expecting only 1 skill
+                if (skills.Count != 1)
+                    Log.outWarn(LogFilter.ServerLoading, $"Found language skill line with {skills.Count} spells. Expected 1. Will use 1st if available");
+
+                SkillLineAbilityRecord ability = skills.Empty() ? null : skills[0];
+                if (ability != null)
+                {
+                    uint languageId = GetSpellLanguage(ability.Spell);
+                    if (languageId != 0)
+                    {
+                        if (!_langsMap.TryGetValue(languageId, out LanguageDesc desc))
+                            Log.outWarn(LogFilter.ServerLoading, $"Spell {ability.Spell} has language {languageId}, which doesn't exist in Languages.db2");
+                        else
+                        {
+                            desc.SpellId = ability.Spell;
+                            desc.SkillId = skillLineEntry.Id;
+                            ++count;
+                        }
+                    }
+                }
+            }
+
+            // Languages that don't have skills will be added in SpellMgr::LoadSpellInfoStore() (e.g. LANG_ZOMBIE, LANG_SHATH_YAR)
+
+            // Log load time
+            Log.outInfo(LogFilter.ServerLoading, $"Loaded {count} languages skills in {Time.GetMSTimeDiffToNow(oldMSTime)} ms");
+        }
+
+        public void LoadLanguagesWords()
+        {
+            uint oldMSTime = Time.GetMSTime();
+
+            uint wordsNum = 0;
+            foreach (LanguageWordsRecord wordEntry in CliDB.LanguageWordsStorage.Values)
+            {
+                byte length = (byte)Math.Min(18, wordEntry.Word.Length);
+
+                var key = Tuple.Create(wordEntry.LanguageID, length);
+
+                _wordsMap.Add(key, wordEntry.Word);
+                ++wordsNum;
+            }
+
+            // log load time
+            Log.outInfo(LogFilter.ServerLoading, $"Loaded {_wordsMap.Count} word groups from {wordsNum} words in {Time.GetMSTimeDiffToNow(oldMSTime)} ms");
+        }
+
+        List<string> FindWordGroup(uint language, uint wordLen)
+        {
+            return _wordsMap.LookupByKey(Tuple.Create(language, wordLen));
+        }
+
+        public string Translate(string msg, uint sourcePlayerLanguage)
+        {
+            StringBuilder result = new StringBuilder();
+            StringArray tokens = new(msg, ' ');
+
+            bool first = true;
+            foreach (string str in tokens)
+            {
+                string nextPart = str;
+                uint wordLen = (uint)Math.Min(18U, str.Length);
+                var wordGroup = FindWordGroup(sourcePlayerLanguage, wordLen);
+                if (wordGroup.Empty())
+                    nextPart = "";
+                else
+                {
+                    uint wordHash = SStrHash(str, true);
+                    byte idxInsideGroup = (byte)(wordHash % wordGroup.Count);
+                    nextPart = wordGroup[idxInsideGroup];
+                }
+
+                if (first)
+                    first = false;
+                else
+                    result.Append(" ");
+
+                result.Append(nextPart);
+            }
+            return result.ToString();
+        }
+
+        static char upper_backslash(char c) { return c == '/' ? '\\' : char.ToUpper(c); }
+
+        static uint[] s_hashtable =
+        {
+            0x486E26EE, 0xDCAA16B3, 0xE1918EEF, 0x202DAFDB,
+            0x341C7DC7, 0x1C365303, 0x40EF2D37, 0x65FD5E49,
+            0xD6057177, 0x904ECE93, 0x1C38024F, 0x98FD323B,
+            0xE3061AE7, 0xA39B0FA1, 0x9797F25F, 0xE4444563,
+        };
+
+        uint SStrHash(string str, bool caseInsensitive, uint seed = 0x7FED7FED)
+        {
+            uint shift = 0xEEEEEEEE;
+            for (var i = 0; i < str.Length; ++i)
+            {
+                var c = str[i];
+                if (caseInsensitive)
+                    c = upper_backslash(c);
+
+                seed = (s_hashtable[c >> 4] - s_hashtable[c & 0xF]) ^ (shift + seed);
+                shift = c + seed + 33 * shift + 3;
+            }
+
+            return seed != 0 ? seed : 1;
+        }
+
+        public bool IsLanguageExist(Language languageId)
+        {
+            return _langsMap.ContainsKey((uint)languageId);
+        }
+
+        public LanguageDesc GetLanguageDescById(Language languageId)
+        {
+            return _langsMap.LookupByKey((uint)languageId);
+        }
+
+        public bool ForEachLanguage(Func<uint, LanguageDesc, bool> callback)
+        {
+            foreach (var pair in _langsMap)
+                if (!callback(pair.Key, pair.Value))
+                    return false;
+            return true;
+        }
+    }
+}

Source/Game/Conditions/ConditionManager.cs

@@ -1807,10 +1807,10 @@ namespace Game
 
             if (condition.LanguageID != 0)
             {
-                LanguageDesc lang = ObjectManager.GetLanguageDescByID((Language)condition.LanguageID);
+                LanguageDesc langDesc = Global.LanguageMgr.GetLanguageDescById((Language)condition.LanguageID);
-                if (lang != null)
+                if (langDesc != null)
                 {
-                    uint languageSkill = player.GetSkillValue((SkillType)lang.skill_id);
+                    uint languageSkill = player.GetSkillValue((SkillType)langDesc.SkillId);
                     if (languageSkill == 0 && player.HasAuraTypeWithMiscvalue(AuraType.ComprehendLanguage, condition.LanguageID))
                         languageSkill = 300;
 

Source/Game/DataStorage/DB2Manager.cs

@@ -1851,6 +1851,16 @@ namespace Game.DataStorage
             return _skillLineAbilitiesBySkillupSkill.LookupByKey(skillId);
         }
 
+        public SkillRaceClassInfoRecord GetAvailableSkillRaceClassInfo(uint skill)
+        {
+            var bounds = _skillRaceClassInfoBySkill.LookupByKey(skill);
+            foreach (var record in bounds)
+                if (record.Availability == 1)
+                    return record;
+
+            return null;
+        }
+        
         public SkillRaceClassInfoRecord GetSkillRaceClassInfo(uint skill, Race race, Class class_)
         {
             var bounds = _skillRaceClassInfoBySkill.LookupByKey(skill);

Source/Game/Entities/Player/Player.cs

@@ -6228,10 +6228,21 @@ namespace Game.Entities
         {
             Global.ScriptMgr.OnPlayerChat(this, ChatMsg.Say, language, text);
 
-            ChatPkt data = new();
+            SendChatMessageToSetInRange(ChatMsg.Say, language, text, WorldConfig.GetFloatValue(WorldCfg.ListenRangeSay));
-            data.Initialize(ChatMsg.Say, language, this, this, text);
-            SendMessageToSetInRange(data, WorldConfig.GetFloatValue(WorldCfg.ListenRangeSay), true);
         }
+
+        void SendChatMessageToSetInRange(ChatMsg chatMsg, Language language, string text, float range)
+        {
+            ChatMessageDistDelivererCustomizer customizer = new(chatMsg, language, this, this, text);
+
+            // Send to self
+            SendPacket(customizer.UntranslatedPacket);
+
+            // Send to players
+            MessageDistDeliverer<ChatMessageDistDelivererCustomizer> notifier = new(this, customizer, range);
+            Cell.VisitWorldObjects(this, notifier, range);
+        }
+
         public override void Say(uint textId, WorldObject target = null)
         {
             Talk(textId, ChatMsg.Say, WorldConfig.GetFloatValue(WorldCfg.ListenRangeSay), target);
@@ -6323,6 +6334,10 @@ namespace Game.Entities
             packet.Initialize(ChatMsg.Whisper, Language.Universal, this, target, Global.DB2Mgr.GetBroadcastTextValue(bct, locale, GetGender()));
             target.SendPacket(packet);
         }
+        public bool CanUnderstandLanguageSkillId(uint langSkillId)
+        {
+            return IsGameMaster() || (langSkillId != 0 && HasSkill((SkillType)langSkillId));
+        }
         #endregion
         
         public void ClearWhisperWhiteList() { WhisperList.Clear(); }
@@ -7581,4 +7596,53 @@ namespace Game.Entities
         //Clears the Menu
         public void ClearGossipMenu() { PlayerTalkClass.ClearMenus(); }
     }
+
+    class ChatMessageDistDelivererCustomizer : IDoWork<Player>
+    {
+        // params
+        ChatMsg Type;
+        Language Language;
+        WorldObject Sender;
+        WorldObject Receiver;
+        string Text;
+
+        // caches
+        public ChatPkt UntranslatedPacket = new();
+        public Optional<ChatPkt> TranslatedPacket;
+        public uint LanguageSkillId;
+
+        public ChatMessageDistDelivererCustomizer(ChatMsg chatType, Language language, WorldObject sender, WorldObject receiver, string message)
+        {
+            Type = chatType;
+            Language = language;
+            Sender = sender;
+            Receiver = receiver;
+            Text = message;
+
+            UntranslatedPacket.Initialize(Type, Language, Sender, Receiver, Text);
+            UntranslatedPacket.Write();
+
+            LanguageDesc languageDesc = Global.LanguageMgr.GetLanguageDescById(language);
+            if (languageDesc != null)
+                LanguageSkillId = languageDesc.SkillId;
+        }
+
+        public void Invoke(Player player)
+        {
+            if (player.CanUnderstandLanguageSkillId(LanguageSkillId))
+            {
+                player.SendPacket(UntranslatedPacket);
+                return;
+            }
+
+            if (!TranslatedPacket.HasValue)
+            {
+                TranslatedPacket.HasValue = true;
+                TranslatedPacket.Value.Initialize(Type, Language, Sender, Receiver, Global.LanguageMgr.Translate(Text, (uint)Language));
+                TranslatedPacket.Value.Write();
+            }
+
+            player.SendPacket(TranslatedPacket.Value);
+        }
+    }
 }

Source/Game/Globals/Global.cs

@@ -22,6 +22,8 @@ using Game.Arenas;
 using Game.BattleFields;
 using Game.BattleGrounds;
 using Game.BlackMarket;
+using Game.Cache;
+using Game.Chat;
 using Game.Collision;
 using Game.DataStorage;
 using Game.DungeonFinding;
@@ -29,14 +31,13 @@ using Game.Entities;
 using Game.Garrisons;
 using Game.Groups;
 using Game.Guilds;
+using Game.Loots;
 using Game.Maps;
 using Game.PvP;
 using Game.Scenarios;
 using Game.Scripting;
-using Game.SupportSystem;
 using Game.Services;
-using Game.Cache;
+using Game.SupportSystem;
-using Game.Loots;
 
 public static class Global
 {
@@ -107,6 +108,7 @@ public static class Global
     public static WeatherManager WeatherMgr { get { return WeatherManager.Instance; } }
 
     public static GameEventManager GameEventMgr { get { return GameEventManager.Instance; } }
+    public static LanguageManager LanguageMgr { get { return LanguageManager.Instance; } }
     public static CreatureTextManager CreatureTextMgr { get { return CreatureTextManager.Instance; } }
     public static AuctionManager AuctionHouseMgr { get { return AuctionManager.Instance; } }
 

Source/Game/Globals/ObjectManager.cs

@@ -45,41 +45,6 @@ namespace Game
                 _difficultyEntries[i] = new List<uint>();
                 _hasDifficultyEntries[i] = new List<uint>();
             }
-
-            lang_description = new LanguageDesc[]
-            {
-                new LanguageDesc(Language.Addon,            0,      0                               ),
-                new LanguageDesc(Language.AddonLogged,      0,      0                               ),
-                new LanguageDesc(Language.Universal,        0,      0                               ),
-                new LanguageDesc(Language.Orcish,           669,    SkillType.LanguageOrcish            ),
-                new LanguageDesc(Language.Darnassian,       671,    SkillType.LanguageDarnassian        ),
-                new LanguageDesc(Language.Taurahe,          670,    SkillType.LanguageTaurahe           ),
-                new LanguageDesc(Language.Dwarvish,         672,    SkillType.LanguageDwarven           ),
-                new LanguageDesc(Language.Common,           668,    SkillType.LanguageCommon            ),
-                new LanguageDesc(Language.Demonic,          815,    SkillType.LanguageDemonTongue       ),
-                new LanguageDesc(Language.Titan,            816,    SkillType.LanguageTitan             ),
-                new LanguageDesc(Language.Thalassian,       813,    SkillType.LanguageThalassian        ),
-                new LanguageDesc(Language.Draconic,         814,    SkillType.LanguageDraconic          ),
-                new LanguageDesc(Language.Kalimag,          265462, SkillType.LanguageOldTongue         ),
-                new LanguageDesc(Language.Gnomish,          7340,   SkillType.LanguageGnomish           ),
-                new LanguageDesc(Language.Troll,            7341,   SkillType.LanguageTroll             ),
-                new LanguageDesc(Language.Gutterspeak,      17737,  SkillType.LanguageForsaken          ),
-                new LanguageDesc(Language.Draenei,          29932,  SkillType.LanguageDraenei           ),
-                new LanguageDesc(Language.Zombie,           265467, 0                                   ),
-                new LanguageDesc(Language.GnomishBinary,    265460, 0                                   ),
-                new LanguageDesc(Language.GoblinBinary,     265461, 0                                   ),
-                new LanguageDesc(Language.Worgen,           69270,  SkillType.LanguageGilnean           ),
-                new LanguageDesc(Language.Goblin,           69269,  SkillType.LanguageGoblin            ),
-                new LanguageDesc(Language.PandarenNeutral,  108127, SkillType.LanguagePandarenNeutral   ),
-                new LanguageDesc(Language.PandarenAlliance, 108130, 0                                   ),
-                new LanguageDesc(Language.PandarenHorde,    108131, 0                                   ),
-                new LanguageDesc(Language.Sprite,           265466, 0                                   ),
-                new LanguageDesc(Language.ShathYar,         265465, 0                                   ),
-                new LanguageDesc(Language.Nerglish,         265464, 0                                   ),
-                new LanguageDesc(Language.Moonkin,          265463, 0                                   ),
-                new LanguageDesc(Language.Shalassian,       262439, SkillType.LanguageShalassian        ),
-                new LanguageDesc(Language.Thalassian2,      262454, SkillType.LanguageThalassian2       )
-            };
         }
 
         //Static Methods
@@ -109,16 +74,6 @@ namespace Game
             else
                 return new ExtendedPlayerName(name, "");
         }
-        public static LanguageDesc GetLanguageDescByID(Language lang)
-        {
-            for (byte i = 0; i < lang_description.Length; ++i)
-            {
-                if (lang_description[i].lang_id == lang)
-                    return lang_description[i];
-            }
-
-            return null;
-        }
         static LanguageType GetRealmLanguageType(bool create)
         {
             switch ((RealmZones)WorldConfig.GetIntValue(WorldCfg.RealmZone))
@@ -10238,7 +10193,6 @@ namespace Game
         }
 
         #region Fields
-        public static LanguageDesc[] lang_description;
         //General
         Dictionary<uint, StringArray> CypherStringStorage = new();
         Dictionary<uint, RepRewardRate> _repRewardRateStorage = new();
@@ -10931,16 +10885,8 @@ namespace Game
 
     public class LanguageDesc
     {
-        public LanguageDesc(Language langid, uint spellid, object skillid)
+        public uint SpellId;
-        {
+        public uint SkillId;
-            lang_id = langid;
-            spell_id = spellid;
-            skill_id = Convert.ToUInt32(skillid);
-        }
-
-        public Language lang_id;
-        public uint spell_id;
-        public uint skill_id;
     }
 
     class ItemSpecStats

Source/Game/Handlers/ChatHandler.cs

@@ -106,14 +106,14 @@ namespace Game
             }
 
             // prevent talking at unknown language (cheating)
-            LanguageDesc langDesc = ObjectManager.GetLanguageDescByID(lang);
+            LanguageDesc langDesc = Global.LanguageMgr.GetLanguageDescById(lang);
             if (langDesc == null)
             {
                 SendNotification(CypherStrings.UnknownLanguage);
                 return;
             }
 
-            if (langDesc.skill_id != 0 && !sender.HasSkill((SkillType)langDesc.skill_id))
+            if (langDesc.SkillId != 0 && !sender.HasSkill((SkillType)langDesc.SkillId))
             {
                 // also check SPELL_AURA_COMPREHEND_LANGUAGE (client offers option to speak in that language)
                 var langAuras = sender.GetAuraEffectsByType(AuraType.ComprehendLanguage);

Source/Game/Server/WorldManager.cs

@@ -439,6 +439,12 @@ namespace Game
             Global.VMapMgr.Initialize(mapData);
             Global.MMapMgr.Initialize(mapData);
 
+            Log.outInfo(LogFilter.ServerLoading, "Loading languages...");
+            Global.LanguageMgr.LoadLanguages();
+
+            Log.outInfo(LogFilter.ServerLoading, "Loading languages words...");
+            Global.LanguageMgr.LoadLanguagesWords();
+
             Log.outInfo(LogFilter.ServerLoading, "Loading SpellInfo Storage...");
             Global.SpellMgr.LoadSpellInfoStore();
 
@@ -460,6 +466,9 @@ namespace Game
             Log.outInfo(LogFilter.ServerLoading, "Loading SpellInfo immunity infos...");
             Global.SpellMgr.LoadSpellInfoImmunities();
 
+            Log.outInfo(LogFilter.ServerLoading, "Loading languages skills...");
+            Global.LanguageMgr.LoadLanguagesSkills();
+
             Log.outInfo(LogFilter.ServerLoading, "Loading PetFamilySpellsStore Data...");
             Global.SpellMgr.LoadPetFamilySpellsStore();
 

Source/Game/Spells/SpellManager.cs

@@ -2188,6 +2188,9 @@ namespace Game.Entities
                         }
                     }
                 }
+
+                if (effect.Effect == (int)SpellEffectName.Language)
+                    Global.LanguageMgr.LoadSpellEffectLanguage(effect);
             }
 
             foreach (SpellAuraOptionsRecord auraOptions in CliDB.SpellAuraOptionsStorage.Values)

Source/Game/Text/CreatureTextManager.cs

@@ -78,7 +78,7 @@ namespace Game
                         temp.sound = 0;
                     }
                 }
-                if (ObjectManager.GetLanguageDescByID(temp.lang) == null)
+                if (!Global.LanguageMgr.IsLanguageExist(temp.lang))
                 {
                     Log.outError(LogFilter.Sql, "GossipManager:  Entry {0}, Group {1} in table `creature_texts` using Language {2} but Language does not exist.", temp.creatureId, temp.groupId, temp.lang);
                     temp.lang = Language.Universal;