Managed ilasm: comprehensive parity fixes with native ilasm

eng/pipelines/coreclr/ilasm.yml

@@ -54,4 +54,6 @@ extends:
                 nameSuffix: ILAsm
                 buildArgs: -s tools.ilasm -c $(_BuildConfig) -test
                 timeoutInMinutes: 120
+                enablePublishTestResults: true
+                testResultsFormat: 'xunit'
         condition: and(succeeded(), ne(variables['Build.Reason'], 'Schedule'))

src/tools/ilasm/KNOWN-ISSUES.md

@@ -0,0 +1,26 @@
+# Managed IL Assembler - Known Issues
+
+## TLS RVA statics
+
+Thread-local storage (TLS) RVA static fields (`.data tls`) are not
+supported by the managed ilasm. The native ilasm emits a TLS directory
+entry in the PE header for these, which the managed ilasm's PE builder
+does not currently implement.
+
+## Win32 resources
+
+Embedding Win32 resources (in either `.obj` or `.res` format) is not
+supported by managed ilasm.
+
+## -MSV is not supported
+
+Overriding the metadata stream version with `-MSV` is not supported by
+managed ilasm.
+
+## -OPTIMIZE is a no-op
+
+Currently managed ilasm does not do any optimizations of the IL written by the user.
+
+## -FOLD is a no-op
+
+Currently managed ilasm does not fold identical IL bodies from different methods into the same blob.

src/tools/ilasm/src/ILAssembler/BlobBuilderExtensions.cs

@@ -1,12 +1,9 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System;
-using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Reflection.Metadata;
 using System.Reflection.Metadata.Ecma335;
-using System.Text;
 
 namespace ILAssembler
 {
@@ -85,7 +82,11 @@ public static void WriteSerializedValue<T>(this BlobBuilder writer, T value)
 
         public static void WriteTypeEntity(this BlobBuilder builder, EntityRegistry.TypeEntity entity)
         {
-            if (entity is EntityRegistry.FakeTypeEntity fakeEntity)
+            if (entity is EntityRegistry.TypeReferenceEntity typeRef)
+            {
+                builder.WriteCompressedInteger(CodedIndex.TypeDefOrRefOrSpec(typeRef.PseudoHandle));
+            }
+            else if (entity is EntityRegistry.FakeTypeEntity fakeEntity)
             {
                 builder.WriteCompressedInteger(CodedIndex.TypeDefOrRefOrSpec(fakeEntity.TypeSignatureHandle));
             }

src/tools/ilasm/src/ILAssembler/CompilationResult.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+
+namespace ILAssembler;
+
+/// <summary>Represents a compiled portable executable image.</summary>
+public sealed class CompilationResult
+{
+    private readonly PEBuilder _peBuilder;
+    private readonly Blob _mvidFixup;
+
+    internal CompilationResult(PEBuilder peBuilder, Blob mvidFixup)
+    {
+        _peBuilder = peBuilder;
+        _mvidFixup = mvidFixup;
+    }
+
+    /// <summary>Serializes the compiled image into the specified builder.</summary>
+    /// <param name="builder">The builder that receives the serialized image.</param>
+    /// <returns>The content identifier of the serialized image.</returns>
+    public BlobContentId Serialize(BlobBuilder builder)
+    {
+        BlobContentId contentId = _peBuilder.Serialize(builder);
+        if (!_mvidFixup.IsDefault)
+        {
+            new BlobWriter(_mvidFixup).WriteGuid(contentId.Guid);
+        }
+
+        return contentId;
+    }
+}

src/tools/ilasm/src/ILAssembler/Diagnostic.cs

@@ -1,8 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Collections.Immutable;
-
 namespace ILAssembler;
 
 public enum DiagnosticSeverity
@@ -52,6 +50,8 @@ public static class DiagnosticIds
     public const string DuplicateMethod = "ILA0030";
     public const string MissingExportedTypeImplementation = "ILA0031";
     public const string KeyFileError = "ILA0032";
+    public const string TooManyGenericParameters = "ILA0033";
+    public const string UnsupportedTlsData = "ILA0034";
 }
 
 internal static class DiagnosticMessageTemplates
@@ -87,4 +87,6 @@ internal static class DiagnosticMessageTemplates
     public const string ParameterIndexOutOfRange = "Parameter index {0} is out of range";
     public const string DuplicateMethod = "Duplicate method definition";
     public const string MissingExportedTypeImplementation = "Undefined implementation in ExportedType '{0}' -- ExportedType not emitted";
+    public const string TooManyGenericParameters = "Generic parameter count {0} exceeds the maximum of {1}";
+    public const string UnsupportedTlsData = "TLS RVA data declarations are not supported";
 }

src/tools/ilasm/src/ILAssembler/DocumentCompiler.cs

@@ -5,49 +5,77 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
-using System.Reflection;
-using System.Reflection.Metadata.Ecma335;
-using System.Reflection.PortableExecutable;
 using Antlr4.Runtime;
 
 namespace ILAssembler;
 public sealed class DocumentCompiler
 {
-    public (ImmutableArray<Diagnostic>, PEBuilder?) Compile(SourceText document, Func<string, SourceText> includedDocumentLoader, Func<string, byte[]> resourceLocator, Options options)
+    public (ImmutableArray<Diagnostic>, CompilationResult?) Compile(SourceText document, Func<string, SourceText> includedDocumentLoader, Func<string, byte[]> resourceLocator, Options options)
     {
-        var inputSource = new AntlrInputStream(document.Text)
-        {
-            name = document.Path
-        };
-        CILLexer lexer = new(inputSource);
-        Dictionary<string, SourceText> loadedDocuments = new()
-        {
-            {document.Path!, document }
-        };
-        PreprocessedTokenSource preprocessor = new(lexer, path =>
+        return Compile([document], includedDocumentLoader, resourceLocator, options);
+    }
+
+    public (ImmutableArray<Diagnostic>, CompilationResult?) Compile(ImmutableArray<SourceText> documents, Func<string, SourceText> includedDocumentLoader, Func<string, byte[]> resourceLocator, Options options)
+    {
+        Dictionary<string, SourceText> loadedDocuments = new();
+        ImmutableArray<Diagnostic>.Builder diagnostics = ImmutableArray.CreateBuilder<Diagnostic>();
+
+        GrammarVisitor? visitor = null;
+        IReadOnlyDictionary<string, string?>? definedVariables = null;
+
+        foreach (var document in documents)
         {
-            var includedDocument = includedDocumentLoader(path);
+            loadedDocuments[document.Path!] = document;
 
-            var includedSource = new AntlrInputStream(includedDocument.Text)
+            var inputSource = new AntlrInputStream(document.Text)
             {
-                name = includedDocument.Path
+                name = document.Path
             };
-            loadedDocuments.Add(includedDocument.Path, includedDocument);
-            return new CILLexer(includedSource);
-        });
+            CILLexer lexer = new(inputSource);
+            PreprocessedTokenSource preprocessor = new(lexer, path =>
+            {
+                var includedDocument = includedDocumentLoader(path);
+                var includedSource = new AntlrInputStream(includedDocument.Text)
+                {
+                    name = includedDocument.Path
+                };
+                loadedDocuments[includedDocument.Path!] = includedDocument;
+                return new CILLexer(includedSource);
+            }, text => new CILLexer(new AntlrInputStream(text)), definedVariables);
 
-        ImmutableArray<Diagnostic>.Builder diagnostics = ImmutableArray.CreateBuilder<Diagnostic>();
-        preprocessor.OnPreprocessorSyntaxError += (source, start, length, msg) =>
-        {
-            diagnostics.Add(new Diagnostic("Preprocessor", DiagnosticSeverity.Error, msg, new Location(new(start, length), loadedDocuments[source])));
-        };
+            preprocessor.OnPreprocessorSyntaxError += (source, start, length, msg) =>
+            {
+                if (loadedDocuments.TryGetValue(source, out var sourceText))
+                {
+                    diagnostics.Add(new Diagnostic("Preprocessor", DiagnosticSeverity.Error, msg, new Location(new(start, length), sourceText)));
+                }
+                else
+                {
+                    diagnostics.Add(new Diagnostic("Preprocessor", DiagnosticSeverity.Error, msg, new Location(new(start, length), new SourceText("", source))));
+                }
+            };
+
+            CILParser parser = new(new CommonTokenStream(preprocessor));
+            parser.RemoveErrorListeners();
+            var parserDiagnostics = ImmutableArray.CreateBuilder<Diagnostic>();
+            parser.AddErrorListener(new ParserErrorListener(parserDiagnostics, loadedDocuments));
+            var result = parser.decls();
+
+            // Add parser diagnostics to the main list
+            diagnostics.AddRange(parserDiagnostics);
 
-        // Note: Parser must use the preprocessor token stream (not the raw lexer)
-        // to properly handle #include, #define, and other preprocessor directives.
-        CILParser parser = new(new CommonTokenStream(preprocessor));
-        var result = parser.decls();
-        GrammarVisitor visitor = new GrammarVisitor(loadedDocuments, options, resourceLocator);
-        _ = result.Accept(visitor);
+            visitor ??= new GrammarVisitor(loadedDocuments, options, resourceLocator);
+
+            _ = result.Accept(visitor);
+
+            // Transfer defined constants to the next document
+            definedVariables = preprocessor.DefinedVariables;
+        }
+
+        if (visitor is null)
+        {
+            return (diagnostics.ToImmutable(), null);
+        }
 
         var image = visitor.BuildImage();
 
@@ -61,3 +89,29 @@ public sealed class DocumentCompiler
         return (diagnostics.ToImmutable(), returnImage ? image.Image : null);
     }
 }
+
+internal sealed class ParserErrorListener : Antlr4.Runtime.IAntlrErrorListener<IToken>
+{
+    private readonly ImmutableArray<Diagnostic>.Builder _diagnostics;
+    private readonly Dictionary<string, SourceText> _loadedDocuments;
+
+    public ParserErrorListener(ImmutableArray<Diagnostic>.Builder diagnostics, Dictionary<string, SourceText> loadedDocuments)
+    {
+        _diagnostics = diagnostics;
+        _loadedDocuments = loadedDocuments;
+    }
+
+    public void SyntaxError(TextWriter output, IRecognizer recognizer, IToken offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+    {
+        var sourceName = offendingSymbol?.TokenSource?.SourceName ?? "";
+        var span = new SourceSpan(offendingSymbol?.StartIndex ?? 0, offendingSymbol is null ? 0 : offendingSymbol.StopIndex - offendingSymbol.StartIndex + 1);
+        if (_loadedDocuments.TryGetValue(sourceName, out var sourceText))
+        {
+            _diagnostics.Add(new Diagnostic("Parser", DiagnosticSeverity.Error, $"line {line}:{charPositionInLine} {msg}", new Location(span, sourceText)));
+        }
+        else
+        {
+            _diagnostics.Add(new Diagnostic("Parser", DiagnosticSeverity.Error, $"line {line}:{charPositionInLine} {msg}", new Location(span, new SourceText("", sourceName))));
+        }
+    }
+}