That’s the flow – hopefully, the above makes sense. Feel free to ask.
using System.Globalization;
using System.Text;
namespace NormalisationEngine;
/// <summary>
/// Normalisation engine for input text.
/// </summary>
public static class NormalisationEngine
{
public static string NormaliseText(ref string input)
{
string normalisedInput = SanitiseAndNormaliseText(input.Trim());
input = "@@always-use-the-normalised-text@@"; // nuke it to stop it accidentally being used.
return normalisedInput;
}
private const int VariationSelectorStart = 0xFE00;
private const int VariationSelectorEnd = 0xFE0F;
private const int VariationSelectorSupplementStart = 0xE0100;
private const int VariationSelectorSupplementEnd = 0xE01EF;
private const int TagBlockStart = 0xE0000;
private const int TagBlockEnd = 0xE007F;
private static bool IsUnicodeSmugglingCodepoint(int cp) =>
(cp >= VariationSelectorStart && cp <= VariationSelectorEnd) ||
(cp >= VariationSelectorSupplementStart && cp <= VariationSelectorSupplementEnd) ||
(cp >= TagBlockStart && cp <= TagBlockEnd);
public static string StripUnicodeSmuggling(string? input)
{
if (string.IsNullOrEmpty(input)) return input ?? string.Empty;
StringBuilder sb = new(input.Length);
foreach (Rune rune in input.EnumerateRunes())
{
if (IsUnicodeSmugglingCodepoint(rune.Value)) continue;
if (CharUnicodeInfo.GetUnicodeCategory(rune.Value) == UnicodeCategory.Format) continue;
sb.Append(rune.ToString());
}
return sb.ToString();
}
private static string SanitiseAndNormaliseText(string? input)
{
if (string.IsNullOrEmpty(input)) return input ?? string.Empty;
// unescape literal "\n" before the collapse loop runs, so newly-created real newlines
// are subject to the same run-of-any-length collapse as newlines already in the input
string unescaped = input.Replace("\\n", "\n");
// normalize first to canonical form
string normalized = unescaped.Normalize(NormalizationForm.FormC);
StringBuilder sb = new(normalized.Length);
// Enumerate by full code point (Rune), not char: supplementary-plane characters (the Tag
// block and the variation-selector supplement) are surrogate pairs, and
// CharUnicodeInfo.GetUnicodeCategory(char) reports each surrogate half as
// UnicodeCategory.Surrogate rather than the composed code point's real category - a
// per-char loop silently lets them through instead of catching them via the Format check.
foreach (Rune rune in normalized.EnumerateRunes())
{
int cp = rune.Value;
// explicit smuggling ranges, dropped regardless of assigned Unicode category
if (IsUnicodeSmugglingCodepoint(cp))
{
continue;
}
var cat = CharUnicodeInfo.GetUnicodeCategory(cp);
// drop all Unicode format characters (includes U+FEFF BOM, ZWSP/ZWJ, LRM/RLM, etc.)
// includes: 1) U+2061–U+2064 (Invisible operators) 2) U+200C / U+200D (ZWNJ / ZWJ) 3) U+00AD (soft hyphen) 4) U+2060 (Word Joiner)
if (cat == UnicodeCategory.Format)
continue;
// convert all space separators (NBSP, NNBSP, IDEOGRAPHIC SPACE, etc.) to ' '
if (cat == UnicodeCategory.SpaceSeparator)
{
sb.Append(' ');
continue;
}
// normalize some common punctuation variants per GPT guidance
switch (cp)
{
case '\t': // TAB
sb.Append(' ');
continue;
case '\r': // CR
continue; // remove carriage returns
case '…': // HORIZONTAL ELLIPSIS
sb.Append("..."); // normalize ellipsis to three dots
continue;
// APOSTROPHES: normalize to ASCII "'"
case '’': // curly apostrophe
case '‘':
sb.Append('\''); // normalize fancy apostrophes to straight
continue;
// QUOTES: normalize to ASCII '"'
case '“':
case '”':
sb.Append('"'); // normalize fancy quotes to straight
continue;
// DASHES: normalize to ASCII '-'
case '—': // em dash
case '–': // en dash
case '‑': // non-breaking hyphen
sb.Append('-'); // normalize dashes to hyphen
continue;
case '─': // remove box-drawing characters that can sometimes appear in web-scraped text and cause issues
continue;
// QUESTION MARKS: normalize to ASCII '?'
case '？': // fullwidth ?
case '❓': // heavy ?
case '❔':
case '❕':
case '¿': // inverted ?
sb.Append('?');
continue;
}
// drop most control characters except common whitespace; normalize them to a space as well
if (cat == UnicodeCategory.Control && cp != '\n')
{
// else skip
continue;
}
sb.Append(rune.ToString());
}
string result = CollapseMultipleSpacesIntoSingleSpace(sb);
// ensure all types of brackets/pipes map to a standard
return FoldDelimiterConfusables(result);
}
private static readonly Dictionary<char, char> s_delimiterConfusables = new()
{
// Angle brackets
['⟨'] = '<', // MATHEMATICAL LEFT ANGLE BRACKET ⟨
['⟩'] = '>', // MATHEMATICAL RIGHT ANGLE BRACKET ⟩
['〈'] = '<', // LEFT-POINTING ANGLE BRACKET (deprecated) 〈
['〉'] = '>', // RIGHT-POINTING ANGLE BRACKET (deprecated) 〉
['〈'] = '<', // LEFT ANGLE BRACKET (CJK) 〈
['〉'] = '>', // RIGHT ANGLE BRACKET (CJK) 〉
['＜'] = '<', // FULLWIDTH LESS-THAN SIGN ＜
['＞'] = '>', // FULLWIDTH GREATER-THAN SIGN ＞
['﹤'] = '<', // SMALL LESS-THAN SIGN ﹤
['﹥'] = '>', // SMALL GREATER-THAN SIGN ﹥
// Square brackets
['［'] = '[', // FULLWIDTH LEFT SQUARE BRACKET ［
['］'] = ']', // FULLWIDTH RIGHT SQUARE BRACKET ］
['﹇'] = '[', // PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET ﹇
['﹈'] = ']', // PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET ﹈
// Curly brackets
['⦃'] = '{', // LEFT WHITE CURLY BRACKET ⦃
['⦄'] = '}', // RIGHT WHITE CURLY BRACKET ⦄
['｛'] = '{', // FULLWIDTH LEFT CURLY BRACKET ｛
['｝'] = '}', // FULLWIDTH RIGHT CURLY BRACKET ｝
// Pipes (special-token delimiter shape "<|...|>")
['｜'] = '|', // FULLWIDTH VERTICAL LINE ｜
['¦'] = '|', // BROKEN BAR ¦
// Colons (role-marker shape "system:"/"assistant:"/etc.)
['：'] = ':', // FULLWIDTH COLON ：
['﹕'] = ':', // SMALL COLON ﹕
['︓'] = ':', // PRESENTATION FORM FOR VERTICAL COLON ︓
['꞉'] = ':', // MODIFIER LETTER COLON ꞉
};
public static string FoldDelimiterConfusables(string? input)
{
if (string.IsNullOrEmpty(input)) return input ?? string.Empty;
StringBuilder? sb = null;
for (int i = 0; i < input.Length; i++)
{
char c = input[i];
if (s_delimiterConfusables.TryGetValue(c, out char ascii))
{
sb ??= new StringBuilder(input, 0, i, input.Length);
sb.Append(ascii);
}
else
{
sb?.Append(c);
}
}
return sb?.ToString() ?? input;
}
public static string CollapseNewlineRuns(string? input)
{
if (string.IsNullOrEmpty(input)) return input ?? string.Empty;
StringBuilder? sb = null;
bool lastWasNewLine = false;
for (int i = 0; i < input.Length; i++)
{
char c = input[i];
if (c == '\n')
{
if (lastWasNewLine)
{
sb ??= new StringBuilder(input, 0, i, input.Length);
continue; // drop this newline, run is already represented
}
lastWasNewLine = true;
sb?.Append(c);
continue;
}
lastWasNewLine = false;
sb?.Append(c);
}
return sb?.ToString() ?? input;
}
private static string CollapseMultipleSpacesIntoSingleSpace(StringBuilder sbContainingTextToRemoveSpacesFrom)
{
string stringWithoutSurplusSpaces = sbContainingTextToRemoveSpacesFrom.ToString();
bool lastWasSpace = false;
bool lastWasNewLine = false;
sbContainingTextToRemoveSpacesFrom.Clear(); // we're going to reinsert the cleaned chars
foreach (var ch in stringWithoutSurplusSpaces)
{
char thisChar = ch;
if (thisChar == '\t') // replace tabs with spaces
{
thisChar = ' ';
}
// multiple spaces in a row get collapsed to single space
if (thisChar == ' ')
{
if (lastWasSpace) continue; // this removes space following spaces
sbContainingTextToRemoveSpacesFrom.Append(' ');
lastWasSpace = true;
continue;
}
// multiple new lines in a row get collapsed to single new line
if (thisChar == '\n')
{
if (lastWasNewLine) continue; // remove new line following new line
sbContainingTextToRemoveSpacesFrom.Append('\n');
lastWasNewLine = true;
continue;
}
sbContainingTextToRemoveSpacesFrom.Append(thisChar);
lastWasSpace = false;
lastWasNewLine = false;
}
return sbContainingTextToRemoveSpacesFrom.ToString().Trim(); // even after collapsing, we have to trim leading/trailing spaces (avoids logic elsewhere)
}
}
