AVRO-4357: [C#] Resolve schema types without enumerating all loaded types

lang/csharp/src/apache/benchmark/ObjectCreatorBenchmarks.cs

@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Emit;
+using Avro.Specific;
+using BenchmarkDotNet.Attributes;
+
+namespace Avro.Benchmark
+{
+    /// <summary>
+    /// Measures how long <see cref="ObjectCreator"/> takes to resolve the generated classes of a schema
+    /// the first time, depending on how many types the application has loaded. A fresh
+    /// <see cref="ObjectCreator"/> per operation keeps its type cache cold, which is what every
+    /// process pays once for every named schema type it deserializes.
+    /// </summary>
+    [SimpleJob(launchCount: 1, warmupCount: 2, iterationCount: 10)]
+    [MemoryDiagnoser]
+    public class ObjectCreatorBenchmarks
+    {
+        private const int NamedSchemaTypes = 50;
+        private const int TypesPerApplicationAssembly = 1_000;
+
+        private string[] _namedSchemaTypeNames;
+
+        /// <summary>
+        /// Number of types loaded in addition to the ones of the benchmark process, spread over
+        /// assemblies of 1,000 types each. An ASP.NET Core application loads roughly 25,000 types,
+        /// large applications several 100,000.
+        /// </summary>
+        [Params(0, 25_000, 100_000, 250_000)]
+        public int AdditionalLoadedTypes { get; set; }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            for (int i = 0; i < AdditionalLoadedTypes / TypesPerApplicationAssembly; i++)
+            {
+                DefineAssembly($"Bench.Application{i}", TypesPerApplicationAssembly);
+            }
+
+            // The generated classes live in their own assembly, like generated contracts referenced by a service
+            _namedSchemaTypeNames = DefineAssembly("Bench.Contracts.V1", NamedSchemaTypes)
+                .Select(type => type.FullName)
+                .ToArray();
+
+            Console.WriteLine($"// Loaded assemblies: {AppDomain.CurrentDomain.GetAssemblies().Length}, types: {CountLoadedTypes()}");
+        }
+
+        [Benchmark]
+        public void ResolveNamedSchemaTypes()
+        {
+            ObjectCreator objectCreator = new ObjectCreator();
+            foreach (string name in _namedSchemaTypeNames)
+            {
+                objectCreator.GetType(name, Schema.Type.Record);
+            }
+        }
+
+        private static IEnumerable<Type> DefineAssembly(string assemblyName, int typeCount)
+        {
+            AssemblyBuilder assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(
+                new AssemblyName(assemblyName),
+                AssemblyBuilderAccess.Run
+            );
+            ModuleBuilder moduleBuilder = assemblyBuilder.DefineDynamicModule(assemblyName);
+            List<Type> types = new List<Type>(typeCount);
+            for (int i = 0; i < typeCount; i++)
+            {
+                types.Add(
+                    moduleBuilder
+                        .DefineType(
+                            $"{assemblyName}.Type{i}",
+                            TypeAttributes.Public | TypeAttributes.Class
+                        )
+                        .CreateType()
+                );
+            }
+
+            return types;
+        }
+
+        private static int CountLoadedTypes()
+        {
+            int count = 0;
+            foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies())
+            {
+                try
+                {
+                    count += assembly.GetTypes().Length;
+                }
+                catch (ReflectionTypeLoadException e)
+                {
+                    count += e.Types.Count(type => type != null);
+                }
+            }
+
+            return count;
+        }
+    }
+}

lang/csharp/src/apache/main/Specific/ObjectCreator.cs

@@ -107,30 +107,10 @@ private Type FindType(string name)
                     type = Type.GetType(name);
                 }
 
-                // type is still not found, need to loop through all loaded assemblies
+                // type is still not found, need to look through all loaded assemblies
                 if (type == null)
                 {
-                    foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies())
-                    {
-                        // Loading all types from all assemblies could fail for a variety of
-                        // non -fatal reasons. If we fail to load types from an assembly, continue.
-                        try
-                        {
-                            // Change the search to look for Types by both NAME and FULLNAME
-                            foreach (Type t in assembly.GetTypes())
-                            {
-                                if (name == t.Name || name == t.FullName || CodeGenUtil.Instance.UnMangle(name) == t.FullName)
-                                {
-                                    type = t;
-                                    break;
-                                }
-                            }
-                        }
-                        catch
-                        {
-                            continue;
-                        }
-                    }
+                    type = FindTypeInLoadedAssemblies(name);
                 }
 
                 return type
@@ -288,6 +268,79 @@ public Type GetType(Schema schema)
             return FindType(schema.Name);
         }
 
+        /// <summary>
+        /// Finds the type with the given name in all loaded assemblies. If several assemblies contain
+        /// a matching type, the one found last in the loaded assemblies wins.
+        /// </summary>
+        /// <param name="name">the object type to locate</param>
+        /// <returns>the object type, or <c>null</c> if not found</returns>
+        private static Type FindTypeInLoadedAssemblies(string name)
+        {
+            Assembly[] assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            string unmangledName = CodeGenUtil.Instance.UnMangle(name);
+
+            // A name with a namespace can only match the full name of a type. Looking it up directly
+            // in each assembly finds the same type as enumerating all types of all assemblies, without
+            // loading all of them.
+            if (name.IndexOf('.') >= 0)
+            {
+                Type type = null;
+                foreach (Assembly assembly in assemblies)
+                {
+                    try
+                    {
+                        type = assembly.GetType(name, false) ?? assembly.GetType(unmangledName, false) ?? type;
+                    }
+                    catch
+                    {
+                        // Looking up a type could fail for a variety of non-fatal reasons, continue.
+                    }
+                }
+
+                if (type != null)
+                {
+                    return type;
+                }
+            }
+
+            return FindTypeByEnumeratingLoadedAssemblies(assemblies, name, unmangledName);
+        }
+
+        /// <summary>
+        /// Finds the type with the given name or full name by enumerating all types of all loaded assemblies.
+        /// </summary>
+        /// <param name="assemblies">the loaded assemblies</param>
+        /// <param name="name">the object type to locate</param>
+        /// <param name="unmangledName">the object type to locate, without escaped reserved words</param>
+        /// <returns>the object type, or <c>null</c> if not found</returns>
+        private static Type FindTypeByEnumeratingLoadedAssemblies(Assembly[] assemblies, string name, string unmangledName)
+        {
+            Type type = null;
+            foreach (Assembly assembly in assemblies)
+            {
+                // Loading all types from all assemblies could fail for a variety of
+                // non -fatal reasons. If we fail to load types from an assembly, continue.
+                try
+                {
+                    // Change the search to look for Types by both NAME and FULLNAME
+                    foreach (Type t in assembly.GetTypes())
+                    {
+                        if (name == t.Name || name == t.FullName || unmangledName == t.FullName)
+                        {
+                            type = t;
+                            break;
+                        }
+                    }
+                }
+                catch
+                {
+                    continue;
+                }
+            }
+
+            return type;
+        }
+
         /// <summary>
         /// Gets the type of the specified type name
         /// </summary>

lang/csharp/src/apache/test/Specific/ObjectCreatorTests.cs

@@ -20,6 +20,9 @@
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Emit;
 
 namespace Avro.Test.Specific
 {
@@ -103,6 +106,48 @@ public void TestComplexGetTypes(Type expecteType, string name)
             Assert.AreEqual(expecteType, objectCreator.GetType(name, Schema.Type.Record));
         }
 
+        [Test]
+        public void TestGetTypeByFullNameFromLoadedAssembly()
+        {
+            var name = $"Avro.Test.Dynamic{Guid.NewGuid():N}.Contracts.MyRecord";
+            var expected = DefineAssemblyWithType(name).GetType(name);
+            var objectCreator = new ObjectCreator();
+
+            Assert.AreEqual(expected, objectCreator.GetType(name, Schema.Type.Record));
+        }
+
+        [TestCase("Avro.Test.Specific.return.Record", TestName = "TestGetTypeReservedWordInNamespace_Unmangled")]
+        [TestCase("Avro.Test.Specific.@return.Record", TestName = "TestGetTypeReservedWordInNamespace_Mangled")]
+        public void TestGetTypeReservedWordInNamespace(string name)
+        {
+            var objectCreator = new ObjectCreator();
+
+            Assert.AreEqual(typeof(Avro.Test.Specific.@return.Record), objectCreator.GetType(name, Schema.Type.Record));
+        }
+
+        [Test]
+        public void TestGetTypeFullNameInSeveralAssembliesResolvesToLastLoadedAssembly()
+        {
+            var name = $"Avro.Test.Dynamic{Guid.NewGuid():N}.Contracts.MyRecord";
+            var first = DefineAssemblyWithType(name);
+            var second = DefineAssemblyWithType(name);
+            var lastInLoadOrder = AppDomain.CurrentDomain.GetAssemblies()
+                .Last(assembly => assembly.FullName == first.FullName || assembly.FullName == second.FullName);
+            var objectCreator = new ObjectCreator();
+
+            Assert.AreEqual(lastInLoadOrder.GetType(name), objectCreator.GetType(name, Schema.Type.Record));
+            Assert.AreNotEqual(first.GetType(name), second.GetType(name));
+        }
+
+        private static Assembly DefineAssemblyWithType(string typeFullName)
+        {
+            var assemblyName = $"Avro.Test.Dynamic{Guid.NewGuid():N}";
+            var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(assemblyName), AssemblyBuilderAccess.Run);
+            var moduleBuilder = assemblyBuilder.DefineDynamicModule(assemblyName);
+            moduleBuilder.DefineType(typeFullName, TypeAttributes.Public | TypeAttributes.Class).CreateType();
+            return assemblyBuilder;
+        }
+
         private class MyNullableFoo
         {
         }