using System.Text; using System.Text.RegularExpressions; namespace Campus_live_rewrite.api.server.Classes { public class Sanitize { public string Context { get; set; } = "RoomChat"; public int Intent { get; set; } = 1; public bool PreRemoveBlockedCharacters { get; set; } = true; public string ReplacementChar { get; set; } = "*"; public string Value { get; set; } = ""; public int ruleset { get; set; } = 0; private const char MaskChar = '*'; // Words that should never be censored private static readonly HashSet Whitelist = new(StringComparer.OrdinalIgnoreCase) { "password", "pass", "class", "assistant", "account" }; // Bad words in NORMALIZED form (see Normalize): lowercase letters only, leet // characters folded, runs of 3+ identical letters collapsed to 2. Both sides of // the comparison go through the same normaliser, so a plain set lookup catches // "F.U.C.K" and "b1tch" with no pattern matching at all. // // This replaced 1749 compiled regexes of the form ^f.{0,1}u.{0,1}c.{0,1}k$, run // against every token of every message. The .{0,1} gaps were meant to jump // separator characters - but Normalize already strips every non-letter before the // match, so all they did was let a 3-letter entry match a 4-or-5-letter innocent // word: "fag" matched FLAG, "gay" matched GRAY, "cum" matched CHUM, "hoe" matched // HOSE. Exact comparison on the normalised token is both correct and far cheaper. private static readonly HashSet Swears = new(StringComparer.Ordinal); static Sanitize() { // Absolute path. This runs in a static constructor, so a miss here throws // TypeInitializationException on EVERY caller - room chat, DM threads, room // names, usernames, display names, bios. A missing word list must degrade to // "nothing is filtered", never take those endpoints down. string path = Path.Combine(Environment.CurrentDirectory, "Data", "badwords.txt"); try { if (!File.Exists(path)) { Console.WriteLine($"[Sanitize] word list not found at {path} - nothing will be filtered"); return; } int masked = 0; foreach (var line in File.ReadAllLines(path)) { // A masked entry ("f***", "****..just", "h*ll") is somebody's already-censored // OUTPUT, not a word: normalised it becomes "just", "hll", "f" - which is how // "just" got censored in chat. Skipped, never learnt. if (line.Contains('*')) { masked++; continue; } var normalized = Normalize(line); if (normalized.Length >= 3) Swears.Add(normalized); } if (masked > 0) Console.WriteLine($"[Sanitize] skipped {masked} masked (*) entries in the word list"); Console.WriteLine($"[Sanitize] loaded {Swears.Count} words from {path}"); } catch (Exception ex) { Console.WriteLine($"[Sanitize] could not load word list ({ex.Message}) - nothing will be filtered"); } } /* ---------------------------------------------------- * PUBLIC API * --------------------------------------------------*/ public static bool ContainsSwears(string message) { if (string.IsNullOrWhiteSpace(message)) return false; if (ConfigCampus.AllowSwears) return false; if (SplitKeepingWhitespace(message).Any(chunk => !IsWhitespace(chunk) && ChunkIsSwear(chunk))) return true; return FindConcealed(message).Count > 0; } public static string SanitizeMessage(string message, char mask = MaskChar) { if (string.IsNullOrWhiteSpace(message)) return message; var sb = new StringBuilder(message.Length); // Two passes, because they catch different evasions. // // 1. The whole whitespace-delimited chunk, punctuation stripped. This is what // catches "f.u.c.k", "f-u-c-k", "f*ck" and zero-width characters wedged // between letters - splitting on \W+ first would turn those into // single-letter tokens that are too short to test, which is exactly how // they used to get through. // // 2. Failing that, the individual words inside the chunk, so "hey,bitch" // (one chunk, normalising to the harmless "heybitch") still gets caught on // the word "bitch". // 3. The CONCEALED forms neither of those see, run first over the whole message so // the masked characters are already in place when the token passes look: // letters spread over several tokens ("s h i t", "sh it") and a swear // repeated or lightly suffixed inside one token ("shitshit", "shitty"). // See FindConcealed for the rules that keep "this hit" and "cockpit" alone. var concealed = FindConcealed(message); if (concealed.Count > 0) { var chars = message.ToCharArray(); foreach (var (start, end) in concealed) for (int i = start; i < end; i++) if (char.IsLetterOrDigit(chars[i])) chars[i] = mask; Console.WriteLine($"[Sanitize] Censored concealed word(s) in message: '{message}'"); message = new string(chars); } foreach (var chunk in SplitKeepingWhitespace(message)) { if (IsWhitespace(chunk)) { sb.Append(chunk); continue; } if (ChunkIsSwear(chunk)) { sb.Append(new string(mask, chunk.Length)); Console.WriteLine($"[Sanitize] Censored word in message: '{chunk}'"); continue; } foreach (var token in Regex.Split(chunk, @"(\W+)")) { if (IsWord(token) && ChunkIsSwear(token)) { sb.Append(new string(mask, token.Length)); Console.WriteLine($"[Sanitize] Censored word in message: '{token}'"); } else { sb.Append(token); } } } return sb.ToString(); } /* ---------------------------------------------------- * INTERNAL HELPERS * --------------------------------------------------*/ private static string[] SplitKeepingWhitespace(string input) => Regex.Split(input, @"(\s+)"); // Swears of 4+ letters, the only ones worth hunting inside other text: the 3-letter // entries ("ass", "cum") are exactly the ones that live inside innocent words. private static readonly Lazy LongSwears = new(() => Swears.Where(w => w.Length >= 4).OrderByDescending(w => w.Length).ToArray()); /// /// Character spans [start, end) of swears that the token passes cannot see: /// /// spread - the letters of one swear spread across several whitespace-separated /// tokens ("s h i t", "sh it", "s.h.i t"). A match counts only when it /// starts at the start of a token and ends at the end of one, covering /// whole tokens - so "this hit" (thi|shit spans two partial tokens) is /// not a match, "s h i t" is. /// compound - one token that IS a swear repeated or lightly suffixed: "shitshit", /// "shitty", "fucked". The rest of the token after the swear(s) must be /// at most 2 letters, and the swear must start the token - so "cockpit" /// (3 letters after) and "scunthorpe" (swear in the middle) are left alone. /// /// Whole-token exact matches are the caller's job (ChunkIsSwear); this only adds the /// forms it misses. /// private static List<(int start, int end)> FindConcealed(string message) { var spans = new List<(int, int)>(); if (string.IsNullOrWhiteSpace(message) || ConfigCampus.AllowSwears || Swears.Count == 0) return spans; // Normalised letter stream with, for every letter, its index in the original. var letters = new StringBuilder(message.Length); var origIdx = new List(message.Length); var tokenStarts = new HashSet(); // stream positions that begin a token var tokenEnds = new HashSet(); // stream positions just past a token's last letter bool inToken = false; for (int i = 0; i < message.Length; i++) { char c = message[i]; if (char.IsWhiteSpace(c)) { if (inToken) { tokenEnds.Add(letters.Length); inToken = false; } continue; } string n = Normalize(c.ToString()); if (n.Length == 0) continue; // punctuation / digit that folds to nothing if (!inToken) { tokenStarts.Add(letters.Length); inToken = true; } letters.Append(n[0]); origIdx.Add(i); } if (inToken) tokenEnds.Add(letters.Length); string stream = letters.ToString(); if (stream.Length < 4) return spans; // Per-token bounds in stream coordinates, for the compound rule. var tokens = tokenStarts.OrderBy(x => x).Zip(tokenEnds.OrderBy(x => x), (a, b) => (a, b)).ToList(); foreach (var swear in LongSwears.Value) { int from = 0; while ((from = stream.IndexOf(swear, from, StringComparison.Ordinal)) >= 0) { int end = from + swear.Length; bool spread = tokenStarts.Contains(from) && tokenEnds.Contains(end) && tokens.Count(t => t.a >= from && t.b <= end) >= 2; bool compound = false; if (!spread) { var tok = tokens.FirstOrDefault(t => t.a <= from && end <= t.b); if (tok != default && tok.a == from) { // Strip every leading repeat of the swear, then allow a tiny tail. int p = from; while (p + swear.Length <= tok.b && string.CompareOrdinal(stream, p, swear, 0, swear.Length) == 0) p += swear.Length; int tail = tok.b - p; // The tail has to be an actual suffix ("s", "y", "ed", "er"...). Any // two letters used to count, so "hello" was hell+o and censored. string tailText = tail > 0 ? stream.Substring(p, tail) : ""; bool suffix = tail == 0 || CompoundSuffixes.Contains(tailText); if (tail <= 2 && suffix && (p > from + swear.Length || tail > 0)) // repeated, or one swear + short suffix { compound = true; end = tok.b; string whole = stream.Substring(tok.a, tok.b - tok.a); if (Whitelist.Contains(whole) || Swears.Contains(whole)) compound = false; // exact forms are the token pass's } } } if (spread || compound) { if (!spans.Any(x => x.Item1 <= origIdx[from] && origIdx[end - 1] < x.Item2)) spans.Add((origIdx[from], origIdx[end - 1] + 1)); from = end; } else from++; } } return spans; } private static readonly HashSet CompoundSuffixes = new(StringComparer.Ordinal) { "s", "y", "z", "ed", "er", "es", "in", "ie", "ty", "rs", "ah" }; private static bool IsWhitespace(string s) => s.Length > 0 && char.IsWhiteSpace(s[0]); private static bool IsWord(string input) => input.Any(char.IsLetterOrDigit); private static bool ChunkIsSwear(string raw) { var normalized = Normalize(raw); if (normalized.Length < 3) return false; if (Whitelist.Contains(normalized)) return false; if (Swears.Contains(normalized)) return true; // "fuuuuck". Normalize leaves runs at 2 ("fuuck") so that "ass" does not // become "as", but that means a stretched word no longer matches its entry. // Squeezing every run to a single letter gives "fuck", which does. Only the // TOKEN is squeezed - the word list keeps its 2-letter runs, so this cannot // make the innocent "as" match the entry "ass". var squeezed = Regex.Replace(normalized, @"(.)\1+", "$1"); return squeezed.Length >= 3 && !Whitelist.Contains(squeezed) && Swears.Contains(squeezed); } private static string Normalize(string input) { input = input.ToLowerInvariant(); input = Regex.Replace(input, @"[0@]", "o"); input = Regex.Replace(input, @"[1!|]", "i"); input = Regex.Replace(input, @"[$5]", "s"); input = Regex.Replace(input, @"3", "e"); input = Regex.Replace(input, @"7", "t"); // Remove non letters input = Regex.Replace(input, @"[^a-z]", ""); // Collapse repeats (fuuuuuck → fuuck) input = Regex.Replace(input, @"(.)\1{2,}", "$1$1"); return input; } } }