Radicade/Tools/CommentStripper/Program.cs

using System.Text;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;

namespace CommentStripper;

/// <summary>
/// Removes every comment from the given C# files.
///
/// This is deliberately not a text scan. A line-based strip corrupts anything that merely looks
/// like a comment - this codebase is full of URLs and paths inside string and interpolated
/// literals, verbatim strings, char literals and C# 11 raw strings. The file is parsed with the
/// real C# parser instead, comment trivia is dropped from each token, and the result is written
/// back. Before a file is written, the tool re-parses its own output and checks that the sequence
/// of token kinds and token text is unchanged: if only trivia moved, the rewrite touched no code.
///
/// Usage: CommentStripper [--dry-run] &lt;path&gt; [path...]
/// </summary>
internal static class Program
{
    private static readonly HashSet<string> SkippedDirectories = new(StringComparer.OrdinalIgnoreCase)
    {
        "bin", "obj", ".git", ".vs", "node_modules", "artifacts"
    };

    private static int Main(string[] args)
    {
        var dryRun = args.Contains("--dry-run");
        var roots = args.Where(argument => !argument.StartsWith("--", StringComparison.Ordinal)).ToArray();
        if (roots.Length == 0)
        {
            Console.Error.WriteLine("usage: CommentStripper [--dry-run] <path> [path...]");
            return 2;
        }

        var filesChanged = 0;
        var commentsRemoved = 0;
        var linesRemoved = 0;
        var filesSeen = 0;
        var failures = 0;

        foreach (var file in roots.SelectMany(EnumerateSources).Distinct(StringComparer.OrdinalIgnoreCase))
        {
            filesSeen++;
            try
            {
                var result = Strip(file, dryRun);
                if (result == null) continue;

                filesChanged++;
                commentsRemoved += result.Value.Comments;
                linesRemoved += result.Value.Lines;
            }
            catch (Exception exception)
            {
                failures++;
                Console.Error.WriteLine($"failed: {file}: {exception.Message}");
            }
        }

        Console.WriteLine($"{(dryRun ? "would change" : "changed")} {filesChanged} of {filesSeen} files; "
            + $"{commentsRemoved} comments, {linesRemoved} lines");
        return failures == 0 ? 0 : 1;
    }

    private static IEnumerable<string> EnumerateSources(string root)
    {
        if (File.Exists(root))
        {
            if (root.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)) yield return root;
            yield break;
        }

        if (!Directory.Exists(root)) yield break;

        foreach (var file in Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories))
        {
            var segments = file.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
            if (segments.Any(segment => SkippedDirectories.Contains(segment))) continue;
            yield return file;
        }
    }

    private static (int Comments, int Lines)? Strip(string path, bool dryRun)
    {
        var bytes = File.ReadAllBytes(path);
        var hasByteOrderMark = bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF;
        var original = new UTF8Encoding(false).GetString(bytes, hasByteOrderMark ? 3 : 0, bytes.Length - (hasByteOrderMark ? 3 : 0));

        var tree = CSharpSyntaxTree.ParseText(original, new CSharpParseOptions(LanguageVersion.Latest));
        var root = tree.GetRoot();

        var stripper = new CommentStripper();
        var rewritten = stripper.Visit(root) ?? root;
        var stripped = rewritten.ToFullString();

        if (stripper.Removed == 0 && stripped == original) return null;

        // Blank out the whitespace that propped up a removed comment so the file does not fill with
        // trailing-space-only lines. The split keeps each line's own terminator, because a verbatim
        // string in this codebase carries CRLF of its own and rewriting it to LF would change the
        // literal - which the token check below would then (correctly) refuse.
        var cleaned = string.Concat(LinePattern.Split(stripped).Select(chunk =>
        {
            var body = chunk.TrimEnd('\r', '\n');
            return body.Trim().Length == 0 ? chunk[body.Length..] : chunk;
        }));

        // Refuse to write anything that changed a token. Parsing the output and comparing the token
        // stream is the check that this was trivia surgery and not code surgery.
        var reparsed = CSharpSyntaxTree.ParseText(cleaned, new CSharpParseOptions(LanguageVersion.Latest));
        var errors = reparsed.GetDiagnostics()
            .Where(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error)
            .ToList();
        if (errors.Count > 0)
            throw new InvalidOperationException("output does not parse: " + errors[0].GetMessage());

        var before = Tokens(root);
        var after = Tokens(reparsed.GetRoot());
        if (before.Count != after.Count)
            throw new InvalidOperationException($"token count changed: {before.Count} -> {after.Count}");
        for (var index = 0; index < before.Count; index++)
        {
            if (before[index] != after[index])
                throw new InvalidOperationException($"token {index} changed: {before[index]} -> {after[index]}");
        }

        var removedLines = CountLines(original) - CountLines(cleaned);
        if (!dryRun)
            File.WriteAllText(path, cleaned, new UTF8Encoding(hasByteOrderMark));

        return (stripper.Removed, removedLines);
    }

    private static List<string> Tokens(SyntaxNode root) =>
        root.DescendantTokens(descendIntoTrivia: false)
            .Select(token => token.RawKind + ":" + token.Text)
            .ToList();

    private static int CountLines(string text) => text.Length == 0 ? 0 : text.Split('\n').Length;

    /// <summary>Splits into chunks that each keep their own line terminator.</summary>
    private static readonly System.Text.RegularExpressions.Regex LinePattern = new("(?<=\\n)", System.Text.RegularExpressions.RegexOptions.Compiled);

    private sealed class CommentStripper : CSharpSyntaxRewriter
    {
        public int Removed { get; private set; }

        public override SyntaxToken VisitToken(SyntaxToken token)
        {
            if (token.LeadingTrivia.Count == 0 && token.TrailingTrivia.Count == 0) return token;

            return token
                .WithLeadingTrivia(Clean(token.LeadingTrivia))
                .WithTrailingTrivia(Clean(token.TrailingTrivia));
        }

        private SyntaxTriviaList Clean(SyntaxTriviaList trivia)
        {
            if (trivia.Count == 0) return trivia;

            var kept = new List<SyntaxTrivia>(trivia.Count);
            foreach (var item in trivia)
            {
                if (IsComment(item))
                {
                    Removed++;
                    continue;
                }

                kept.Add(item);
            }

            return kept.Count == trivia.Count ? trivia : SyntaxFactory.TriviaList(kept);
        }

        private static bool IsComment(SyntaxTrivia trivia) => trivia.Kind() switch
        {
            SyntaxKind.SingleLineCommentTrivia => true,
            SyntaxKind.MultiLineCommentTrivia => true,
            SyntaxKind.SingleLineDocumentationCommentTrivia => true,
            SyntaxKind.MultiLineDocumentationCommentTrivia => true,
            _ => false
        };
    }
}