From 427faf8a36075381824a898e211cce1748fd34cc Mon Sep 17 00:00:00 2001 From: Tuomas Hietanen Date: Tue, 17 Nov 2026 16:42:48 +0000 Subject: [PATCH] Evaluate captured values without compiling; cache row readers by shape - Patterns.evaluateExpression reads a query's one-off values (constants, members, method calls on captured values, conversions, quotes) by reflection and interprets the rest, in place of Expression.Lambda(e).Compile(). - QueryExpression.CompiledLambdas compiles a row reader once per shape and binds each run's constants into it. Not used where the runtime interprets lambdas (IsDynamicCodeSupported false). - Exceptions from a captured value are raised unwrapped. --- docs/RELEASE_NOTES.md | 5 + src/SQLProvider.Common/SqlRuntime.Patterns.fs | 244 ++++++++------- .../SqlRuntime.QueryExpression.fs | 158 +++++++++- .../ExpressionEvaluationTests.fs | 292 ++++++++++++++++++ .../SqlProvider.Tests.fsproj | 1 + 5 files changed, 586 insertions(+), 114 deletions(-) create mode 100644 tests/SqlProvider.Tests/ExpressionEvaluationTests.fs diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 2572051b..2b81aa54 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,3 +1,8 @@ +### 1.5.33 - 17.11.2026 +- A query's row reader is compiled once per shape and reused when the query runs again, with any captured values: a run no longer compiles and JITs a new delegate. Unchanged where the runtime interprets lambdas (IsDynamicCodeSupported false) +- Values a query reads once (a captured variable's member, a sub-query's source, a method call on captured values) are read by reflection, and other one-off expressions are interpreted where the runtime can (not on .NET Framework), instead of compiled +- An exception thrown while evaluating a query's captured value is raised as it is, not wrapped in TargetInvocationException + ### 1.5.32 - 14.11.2026 - Fix: PostgreSQL functions are called inside a SELECT. With Npgsql 7 and newer they failed with "procedure ... does not exist", as a stored-procedure command is a CALL there - PostgreSQL procedures (11 and newer) are under ctx.Procedures and called with CALL, with any Npgsql version; they weren't listed before. INOUT and OUT parameters are the result's properties diff --git a/src/SQLProvider.Common/SqlRuntime.Patterns.fs b/src/SQLProvider.Common/SqlRuntime.Patterns.fs index 4ea3e84b..259289ee 100644 --- a/src/SQLProvider.Common/SqlRuntime.Patterns.fs +++ b/src/SQLProvider.Common/SqlRuntime.Patterns.fs @@ -9,6 +9,114 @@ open FSharp.Data.Sql.Schema /// Marker interface type ISqlQueryable = interface end +/// Reflection raises a member's exception wrapped in TargetInvocationException; compiled code raises it as it is. +let private unwrapInvocation (read: unit -> obj) = + try read () + with :? TargetInvocationException as ex when not (isNull ex.InnerException) -> + Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex.InnerException).Throw() + null + +/// A Convert that keeps the value as it is: a reference cast, boxing, unboxing to the same type or wrapping in +/// Nullable. A numeric, enum or user-defined conversion computes a new value, so it isn't one. +let private convertsAsIs (u:UnaryExpression) (value:obj) = + isNull u.Method + && (if isNull value then not u.Type.IsValueType || not (isNull (Nullable.GetUnderlyingType u.Type)) + else u.Type.IsInstanceOfType value) + +/// Value types whose methods don't change the value. Reflection calls a method on a boxed copy, which compiled code +/// doesn't, so a method of any other value type is left to the interpreter. +let private unchangedByMethods = + Collections.Generic.HashSet([ typeof; typeof; typeof; typeof; typeof ]) + +let private callableByReflection (mc:MethodCallExpression) = + (mc.Method.IsStatic + || not mc.Object.Type.IsValueType || mc.Object.Type.IsPrimitive || mc.Object.Type.IsEnum + || unchangedByMethods.Contains mc.Object.Type) + && mc.Method.GetParameters() |> Array.forall (fun p -> not p.ParameterType.IsByRef) + +/// Finds a parameter that none of the expression's own lambdas, blocks or catch blocks declares. +type private OuterParameterFinder() = + inherit ExpressionVisitor() + let declared = Collections.Generic.HashSet() + let used = Collections.Generic.HashSet() + member _.Found = not (used.IsSubsetOf declared) + override _.Visit (e:Expression) = + match e with + | :? LambdaExpression as l -> declared.UnionWith l.Parameters + | :? BlockExpression as b -> declared.UnionWith b.Variables + | :? TryExpression as t -> for h in t.Handlers do if not (isNull h.Variable) then declared.Add h.Variable |> ignore + | :? ParameterExpression as p -> used.Add p |> ignore + | _ -> () + base.Visit e + +/// Whether readValue reads the expression: a constant, a field or property of a readable value or a static one, a +/// method call on readable values, a Convert or TypeAs of one, and a Quote that uses no outer lambda's parameter. +/// The answer comes from the expression's shape, so nothing has been evaluated when it is false. +let rec private isReadable (e:Expression) = + match e.NodeType, e with + | ExpressionType.Constant, _ -> true + | ExpressionType.MemberAccess, (:? MemberExpression as me) -> isNull me.Expression || isReadable me.Expression + | ExpressionType.Call, (:? MethodCallExpression as mc) -> + callableByReflection mc && (isNull mc.Object || isReadable mc.Object) && Seq.forall isReadable mc.Arguments + | (ExpressionType.Convert | ExpressionType.ConvertChecked | ExpressionType.TypeAs), (:? UnaryExpression as ue) -> + isReadable ue.Operand + | ExpressionType.Quote, (:? UnaryExpression as ue) -> + let finder = OuterParameterFinder() + finder.Visit ue.Operand |> ignore + not finder.Found + | _ -> false + +/// Runs a parameterless expression interpreted, as compiling it would JIT code that runs once. +let private interpret (e:Expression) : obj = + let body = + if Type.(=)(e.Type, typeof) then Expression.Block(e, Expression.Constant(null)) :> Expression + else Expression.Convert(e, typeof) :> Expression + let lambda = Expression.Lambda> body +#if NET461 || NET462 + lambda.Compile().Invoke() +#else + lambda.Compile(preferInterpretation = true).Invoke() +#endif + +/// The value of an expression isReadable accepts, each part of it evaluated once. Where reflection doesn't answer as +/// compiled code does (a member of null or of a Nullable, a conversion that computes a new value), the node is +/// interpreted over its operand's value. +let rec private readValue (e:Expression) : obj = + match e.NodeType, e with + | ExpressionType.MemberAccess, (:? MemberExpression as me) -> + let owner = if isNull me.Expression then null else readValue me.Expression + if not (isNull me.Expression) && (isNull owner || not (isNull (Nullable.GetUnderlyingType me.Expression.Type))) then + interpret (me.Update(Expression.Constant(owner, me.Expression.Type))) + else + match me.Member with + | :? FieldInfo as f -> f.GetValue owner + | m -> unwrapInvocation (fun () -> (m :?> PropertyInfo).GetValue(owner, null)) + | ExpressionType.Call, (:? MethodCallExpression as mc) -> + let owner = if isNull mc.Object then null else readValue mc.Object + let args = Array.zeroCreate mc.Arguments.Count + for i in 0 .. args.Length - 1 do + args.[i] <- readValue mc.Arguments.[i] + if isNull owner && not mc.Method.IsStatic then raise (NullReferenceException()) + unwrapInvocation (fun () -> mc.Method.Invoke(owner, args)) + | (ExpressionType.Convert | ExpressionType.ConvertChecked), (:? UnaryExpression as ue) -> + let value = readValue ue.Operand + if convertsAsIs ue value then value + else interpret (ue.Update(Expression.Constant(value, ue.Operand.Type))) + | ExpressionType.TypeAs, (:? UnaryExpression as ue) -> + let value = readValue ue.Operand + if isNull value || ue.Type.IsInstanceOfType value then value else null + | ExpressionType.Quote, (:? UnaryExpression as ue) -> box ue.Operand + | _ -> (e :?> ConstantExpression).Value + +/// The value of a parameterless expression when isReadable accepts it. +let private tryReadValue (e:Expression) : obj voption = + if isReadable e then ValueSome (readValue e) else ValueNone + +/// The value of a parameterless expression, evaluated once: read by reflection where isReadable accepts it, +/// interpreted otherwise. +let internal evaluateExpression (e:Expression) : obj = + if isReadable e then readValue e else interpret e + [] let inline (|MethodWithName|_|) (s:string) (m:MethodInfo) = if String.Equals(m.Name, s, StringComparison.Ordinal) then ValueSome () else ValueNone [] @@ -67,14 +175,11 @@ let (|SeqValuesQueryable|_|) (e:Expression) = match e.NodeType, e with | ExpressionType.Constant, (:? ConstantExpression as ce) -> match ce.Value with - | :? ISqlQueryable -> - let values = (Expression.Lambda(e).Compile() :?> Func).Invoke() - ValueSome values + | :? ISqlQueryable -> ValueSome (ce.Value :?> System.Linq.IQueryable) | _ -> ValueNone | _ -> - let values = (Expression.Lambda(e).Compile() :?> Func).Invoke() - match values with - | :? ISqlQueryable -> ValueSome values + match evaluateExpression e with + | :? ISqlQueryable as values -> ValueSome (values :?> System.Linq.IQueryable) | _ -> ValueNone [] @@ -92,22 +197,7 @@ let (|SeqValues|_|) (e:Expression) = // select x from xs where x in (select y from ys) // ...but instead we just execute the sub-query. Works when sub-query results are small. - let values = - match e.NodeType, e with - | ExpressionType.Constant, (:? ConstantExpression as ce) when not(isNull ce.Value) -> - ce.Value :?> System.Collections.IEnumerable - | ExpressionType.MemberAccess, (:? MemberExpression as me) when (me.Expression :? ConstantExpression) -> - let ceVal = (me.Expression :?> ConstantExpression).Value - let myVal = - match me.Member with - | :? FieldInfo as fieldInfo when not(isNull fieldInfo) -> - fieldInfo.GetValue ceVal - | :? PropertyInfo as propInfo when not(isNull propInfo) -> - propInfo.GetValue(ceVal, null) - | _ -> ceVal - myVal :?> System.Collections.IEnumerable - | _ -> - (Expression.Lambda(e).Compile() :?> Func).Invoke() + let values = evaluateExpression e :?> System.Collections.IEnumerable // Working with untyped IEnumerable so need to do a lot manually instead of using Seq // Work out the size the sequence @@ -174,11 +264,7 @@ let (|ConstantOrNullableConstant|_|) (e:Expression) = match ue.Operand.NodeType, ue.Operand with | ExpressionType.Constant, (:? ConstantExpression as ce) -> if isNull ce.Value then Some None else Some(Some ce.Value) | ExpressionType.New, (:? NewExpression as ne) -> - try - if ne.Type.IsClass then - Some(Some((Expression.Lambda(ne).Compile() :?> Func<_>).Invoke())) - else - Some(Some(Expression.Lambda(ne).Compile().DynamicInvoke())) + try Some(Some(evaluateExpression ne)) with err -> failwith ("unsupported nullable new expression " + ne.ToString() + " err: " + err.Message) | _ -> failwith ("unsupported nullable expression " + e.ToString()) | _ -> None @@ -392,82 +478,26 @@ let intType (typ:Type) = elif (not (isNull typ)) && Common.Utilities.isVOpt typ then typeof> else typeof -/// Fast type-safe evaluation without boxing for common types +/// Return types whose evaluation errors propagate; for other types evaluateMethodCall gives None on an error. +let private propagatingReturnTypes = + System.Collections.Generic.HashSet( + [ typeof; typeof; typeof; typeof; typeof; typeof; typeof + typeof; typeof; typeof; typeof; typeof; typeof; typeof + typeof; typeof + typeof>; typeof>; typeof>; typeof> + typeof>; typeof>; typeof>; typeof> + typeof>; typeof>; typeof>; typeof> + typeof>; typeof>; typeof>; typeof> ]) + +/// Evaluates a parameterless method call once let internal evaluateMethodCall (meth: MethodCallExpression) : obj option = let returnType = meth.Method.ReturnType - - if returnType.IsClass then - try - Some((Expression.Lambda(meth).Compile() :?> Func<_>).Invoke() |> box) - with - _ -> None - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then - // Should we handle case where isNull e.Object and e.Method.Name in (get_Now, get_UtcNow, get_Today) ? - Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then - // Should we handle case where isNull e.Object and e.Method.Name in (get_Now, get_UtcNow) ? - Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - elif Type.(=)(returnType, typeof) then Some((Expression.Lambda(meth).Compile() :?> Func).Invoke() |> box) - - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) - elif Type.(=)(returnType, typeof>) then Some((Expression.Lambda(meth).Compile() :?> Func>).Invoke() |> box) + if not returnType.IsClass && propagatingReturnTypes.Contains returnType then Some(evaluateExpression meth) else - try - Some(Expression.Lambda(meth).Compile().DynamicInvoke()) - with err -> None - -let internal getRightFromOp (right:Expression) = - match right.NodeType, right with - | ExpressionType.Constant, (:? ConstantExpression as ce) -> ce.Value - | ExpressionType.MemberAccess, (:? MemberExpression as me) when (me.Expression :? ConstantExpression) -> - let ceVal = (me.Expression :?> ConstantExpression).Value - let myVal = - match me.Member with - | :? FieldInfo as fieldInfo when not(isNull fieldInfo) -> - fieldInfo.GetValue ceVal - | :? PropertyInfo as propInfo when not(isNull propInfo) -> - propInfo.GetValue(ceVal, null) - | _ -> ceVal - myVal - | ExpressionType.Call, (:? MethodCallExpression as e) -> - match evaluateMethodCall e with - | Some x -> x - | None -> Expression.Lambda(right).Compile().DynamicInvoke() - | _ -> - if right.Type.IsClass then - try - (Expression.Lambda(right).Compile() :?> Func<_>).Invoke() |> box - with - | _ -> Expression.Lambda(right).Compile().DynamicInvoke() - else Expression.Lambda(right).Compile().DynamicInvoke() + try Some(evaluateExpression meth) + with _ -> None +let internal getRightFromOp (right:Expression) = evaluateExpression right let (|SqlExistsClause|_|) = function | MethodCall(None, (MethodWithName "Any" as meth), [ SeqValuesQueryable src; OptionalQuote qual ]) -> @@ -783,15 +813,7 @@ and (|SimpleCondition|_|) exp = match meth.NodeType, meth with | ExpressionType.Constant, (:? ConstantExpression as ce) -> Some ce.Value | ExpressionType.MemberAccess, (:? MemberExpression as me) when (me.Expression :? ConstantExpression) -> - let ceVal = (me.Expression :?> ConstantExpression).Value - let myVal = - match me.Member with - | :? FieldInfo as fieldInfo when not(isNull fieldInfo) -> - fieldInfo.GetValue ceVal - | :? PropertyInfo as propInfo when not(isNull propInfo) -> - propInfo.GetValue(ceVal, null) - | _ -> ceVal - Some myVal + Some(evaluateExpression me) | ExpressionType.Call, (:? MethodCallExpression as e) -> evaluateMethodCall e @@ -803,10 +825,8 @@ and (|SimpleCondition|_|) exp = | ExpressionType.MemberAccess, (:? MemberExpression as me) when Type.(=)(me.Type, typeof) && me.Member.Name = "UtcNow" -> Some(DateTimeOffset.UtcNow |> box) | _ -> // In case user feeds us incomplete lambda, we will not execute: the user might have a context needed for the compilation. - try - if meth.Type.IsClass then (Expression.Lambda(meth).Compile() :?> Func<_>).Invoke() |> box |> Some - else Some(Expression.Lambda(meth).Compile().DynamicInvoke()) - with err -> None + try Some(evaluateExpression meth) + with _ -> None match invokation with | None -> None | Some invokedResult -> diff --git a/src/SQLProvider.Common/SqlRuntime.QueryExpression.fs b/src/SQLProvider.Common/SqlRuntime.QueryExpression.fs index 2eaf1f21..0ce8c1af 100644 --- a/src/SQLProvider.Common/SqlRuntime.QueryExpression.fs +++ b/src/SQLProvider.Common/SqlRuntime.QueryExpression.fs @@ -9,6 +9,160 @@ open FSharp.Data.Sql.Common open FSharp.Data.Sql.Patterns open FSharp.Data.Sql.Schema +/// Compiled lambdas cached by shape. A lambda's constants become reads of an array argument, so lambdas that differ +/// only in their constants' values share one compiled factory, which binds the values into a new delegate. +module internal CompiledLambdas = + + /// A lambda's shape as a sequence of tokens: node types, types, members, methods, counts and parameter + /// positions, with each constant's type but not its value. + [] + type ShapeKey = + { Tokens: obj[]; Hash: int } + override this.GetHashCode() = this.Hash + override this.Equals other = + match other with + | :? ShapeKey as o -> + this.Hash = o.Hash && this.Tokens.Length = o.Tokens.Length + && Array.forall2 (fun (a: obj) b -> obj.Equals(a, b)) this.Tokens o.Tokens + | _ -> false + + /// Assignments: a constant they write to can't become an array read. + let private assignments = + HashSet( + [ ExpressionType.Assign; ExpressionType.AddAssign; ExpressionType.AddAssignChecked; ExpressionType.AndAssign + ExpressionType.DivideAssign; ExpressionType.ExclusiveOrAssign; ExpressionType.LeftShiftAssign + ExpressionType.ModuloAssign; ExpressionType.MultiplyAssign; ExpressionType.MultiplyAssignChecked + ExpressionType.OrAssign; ExpressionType.PowerAssign; ExpressionType.RightShiftAssign + ExpressionType.SubtractAssign; ExpressionType.SubtractAssignChecked; ExpressionType.PreIncrementAssign + ExpressionType.PreDecrementAssign; ExpressionType.PostIncrementAssign; ExpressionType.PostDecrementAssign ]) + + /// Reads a lambda's shape and its constants' values, both in the order ExpressionVisitor visits the nodes. + /// Supported is false when the lambda holds a node kind the shape doesn't describe, a Quote (its expression is a + /// value, whose constants must stay constants) or an assignment to anything but a variable. + type private ShapeReader() = + inherit ExpressionVisitor() + let tokens = ResizeArray 64 + let values = ResizeArray() + let parameters = Dictionary(HashIdentity.Reference) + let positionOf (p: ParameterExpression) = + match parameters.TryGetValue p with + | true, i -> i + | false, _ -> + parameters.Add(p, parameters.Count) + parameters.Count - 1 + let add (token: obj) = tokens.Add token + member val Supported = true with get, set + member _.Tokens = tokens.ToArray() + member _.Values = values.ToArray() + override this.Visit (e: Expression) = + if not this.Supported then e + elif isNull e then + add null + e + else + add (box e.NodeType) + add e.Type + let supported = + match e with + | :? ConstantExpression -> true + | :? ParameterExpression as p -> + add (box (positionOf p)) + add (box p.IsByRef) + true + | :? LambdaExpression as l -> + add (box l.Parameters.Count) + true + | :? MemberExpression as m -> + add m.Member + true + | :? MethodCallExpression as c -> + add c.Method + add (box c.Arguments.Count) + true + | :? UnaryExpression as u -> + add u.Method + add (box u.IsLiftedToNull) + e.NodeType <> ExpressionType.Quote + && not (assignments.Contains e.NodeType && not (u.Operand :? ParameterExpression)) + | :? BinaryExpression as b -> + add b.Method + add (box b.IsLiftedToNull) + not (assignments.Contains e.NodeType && not (b.Left :? ParameterExpression)) + | :? ConditionalExpression | :? DefaultExpression -> true + | :? NewExpression as n -> + add n.Constructor + add (box n.Arguments.Count) + true + | :? NewArrayExpression as n -> + add (box n.Expressions.Count) + true + | :? InvocationExpression as i -> + add (box i.Arguments.Count) + true + | :? TypeBinaryExpression as t -> + add t.TypeOperand + true + | :? BlockExpression as b -> + add (box b.Variables.Count) + add (box b.Expressions.Count) + true + | :? MemberInitExpression as mi -> + add (box mi.Bindings.Count) + mi.Bindings |> Seq.forall (fun b -> add b.Member; b :? MemberAssignment) + | :? ListInitExpression as li -> + add (box li.Initializers.Count) + li.Initializers |> Seq.iter (fun i -> add i.AddMethod; add (box i.Arguments.Count)) + true + | _ -> false + if supported then base.Visit e + else + this.Supported <- false + e + override _.VisitConstant c = + values.Add c.Value + upcast c + + /// Replaces each constant with a read of the array parameter, numbered in the order ShapeReader reads them. + type private ConstantLifter(constants: ParameterExpression) = + inherit ExpressionVisitor() + let mutable count = 0 + override _.VisitConstant c = + let read = Expression.ArrayIndex(constants, Expression.Constant count) + count <- count + 1 + if Type.(=)(c.Type, typeof) then upcast read else upcast Expression.Convert(read, c.Type) + + let private factories = System.Collections.Concurrent.ConcurrentDictionary>() + + /// The number of shapes compiled. + let internal cachedShapes () = factories.Count + + /// Whether the runtime compiles lambdas to IL. Where it interprets them (RuntimeFeature.IsDynamicCodeSupported is + /// false), a lambda costs little to compile and a factory's array reads slow every row, so the cache isn't used. + let private compilesToIL = + match Type.GetType "System.Runtime.CompilerServices.RuntimeFeature" with + | null -> true + | feature -> + match feature.GetProperty "IsDynamicCodeSupported" with + | null -> true + | supported -> supported.GetValue null :?> bool + + /// The lambda's delegate, from the factory compiled for its shape; a lambda the shape doesn't describe is compiled alone. + let internal compile (lambda: LambdaExpression) : Delegate = + if not compilesToIL then lambda.Compile() + else + let reader = ShapeReader() + reader.Visit lambda |> ignore + if not reader.Supported then lambda.Compile() + else + let tokens = reader.Tokens + let key = { Tokens = tokens; Hash = tokens |> Array.fold (fun h t -> h * 31 + (if isNull t then 0 else t.GetHashCode())) 17 } + let factory = + factories.GetOrAdd(key, fun _ -> + let constants = Expression.Parameter(typeof, "constants") + let lifted = ConstantLifter(constants).Visit lambda + Expression.Lambda>(Expression.Convert(lifted, typeof), constants).Compile()) + factory.Invoke reader.Values + module internal QueryExpressionTransformer = /// One lock for each table of each provider. The first query of a table reads its columns from the database inside @@ -742,7 +896,7 @@ module internal QueryExpressionTransformer = | _ -> Expression.Parameter(typeof,"result") let pmap = Dictionary() pmap.Add(baseAlias, ResizeArray<_>()) - (Expression.Lambda(initDbParam,initDbParam).Compile(),pmap) + (CompiledLambdas.compile (Expression.Lambda(initDbParam,initDbParam)),pmap) | projs -> let replaceParams = Dictionary() @@ -978,7 +1132,7 @@ module internal QueryExpressionTransformer = QueryEvents.PublishExpression generatedMegaLambda if membershipSubquery && not (isNull generatedMegaLambda) && not (isColumnRead generatedMegaLambda.Body) then failwithf "IN / NOT IN needs a sub-query that selects one column as the database computes it, but this projection is computed in .NET after the rows are read: %O. Select the column or aggregate itself, or read the values first with Seq.toArray." generatedMegaLambda - (generatedMegaLambda.Compile(),finalParams) + (CompiledLambdas.compile generatedMegaLambda,finalParams) let sqlQuery = if groupgin.Count > 0 then diff --git a/tests/SqlProvider.Tests/ExpressionEvaluationTests.fs b/tests/SqlProvider.Tests/ExpressionEvaluationTests.fs new file mode 100644 index 00000000..21d0b9d8 --- /dev/null +++ b/tests/SqlProvider.Tests/ExpressionEvaluationTests.fs @@ -0,0 +1,292 @@ +module ExpressionEvaluationTests + +// Values of parameterless expressions read without compiling (Patterns.evaluateExpression), and compiled lambdas +// cached by shape (CompiledLambdas.compile). Both are internal, so they are called by reflection. + +open System +open System.Linq +open System.Linq.Expressions +open NUnit.Framework + +let private flags = Reflection.BindingFlags.Static ||| Reflection.BindingFlags.Public ||| Reflection.BindingFlags.NonPublic +let private commonAssembly = typeof.Assembly + +let private invokeStatic (typeName: string) (name: string) (arguments: obj array) = + let methodInfo = commonAssembly.GetType(typeName, true).GetMethod(name, flags) + try methodInfo.Invoke(null, arguments) + with :? Reflection.TargetInvocationException as ex -> + Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex.InnerException).Throw() + null + +/// The expression's value, read by reflection or interpreted. +let private evaluate (e: Expression) = invokeStatic "FSharp.Data.Sql.Patterns" "evaluateExpression" [| box e |] + +/// The expression's value when it is read by reflection alone; ValueNone when it would be interpreted. +let private readByReflection (e: Expression) = + invokeStatic "FSharp.Data.Sql.Patterns" "tryReadValue" [| box e |] :?> obj voption + +let private compile (lambda: LambdaExpression) = + invokeStatic "FSharp.Data.Sql.QueryExpression.CompiledLambdas" "compile" [| box lambda |] :?> Delegate + +let private cachedShapes () = + invokeStatic "FSharp.Data.Sql.QueryExpression.CompiledLambdas" "cachedShapes" [||] :?> int + +[] +type Point = { X: int; Y: int } + +type Holder(name: string, count: int, point: Point) = + member _.Name = name + member _.Count = count + member _.Point = point + member _.Failing: string = invalidOp "getter failed" + +/// Closure classes as C# generates them: a nested lambda's class holds the outer one in a field. +type OuterClosure() = + [] val mutable Ids: int list + +type InnerClosure() = + [] val mutable Outer: OuterClosure + [] val mutable Name: string + +type Statics = + static member Greeting = "hello" + static member Fail() : string = invalidOp "method failed" + +type ToExpression = + static member Of(e: Expression>) = e + +let private field (owner: Expression) (name: string) = Expression.Field(owner, name) :> Expression +let private property (owner: Expression) (name: string) = Expression.Property(owner, name) :> Expression +let private constant (value: obj) = Expression.Constant value :> Expression + +[] +let ``constants are read as they are``() = + Assert.AreEqual(ValueSome(box 42), readByReflection (constant 42)) + Assert.AreEqual(ValueSome(box "text"), readByReflection (constant "text")) + Assert.AreEqual(ValueSome(null: obj), readByReflection (Expression.Constant(null, typeof))) + +[] +let ``properties and fields of a constant are read by reflection``() = + let holder = Holder("name", 3, { X = 1; Y = 2 }) + Assert.AreEqual(ValueSome(box "name"), readByReflection (property (constant holder) "Name")) + Assert.AreEqual(ValueSome(box 3), readByReflection (property (constant holder) "Count")) + let closure = InnerClosure(Name = "closure") + Assert.AreEqual(ValueSome(box "closure"), readByReflection (field (constant closure) "Name")) + +[] +let ``a member whose value is null reads null``() = + let holder = Holder(null, 0, { X = 0; Y = 0 }) + Assert.AreEqual(ValueSome(null: obj), readByReflection (property (constant holder) "Name")) + Assert.AreEqual(ValueSome(null: obj), readByReflection (field (constant (InnerClosure())) "Outer")) + +[] +let ``a member of null throws as compiled code does``() = + let read = property (Expression.Constant(null, typeof)) "Name" + Assert.Throws(fun () -> evaluate read |> ignore) |> ignore + +[] +let ``members of a Nullable give what compiled code gives``() = + let none = Expression.Constant(null, typeof>) + Assert.AreEqual(ValueSome(box false), readByReflection (property none "HasValue")) + Assert.Throws(fun () -> evaluate (property none "Value") |> ignore) |> ignore + let some = Expression.Constant(Nullable 5, typeof>) + Assert.AreEqual(ValueSome(box true), readByReflection (property some "HasValue")) + Assert.AreEqual(ValueSome(box 5), readByReflection (property some "Value")) + +[] +let ``members of value types are read``() = + let point = { X = 5; Y = 7 } + Assert.AreEqual(ValueSome(box 5), readByReflection (property (constant point) "X")) + let holder = Holder("h", 1, point) + Assert.AreEqual(ValueSome(box 7), readByReflection (property (property (constant holder) "Point") "Y")) + +[] +let ``static members are read``() = + let staticProperty = Expression.Property(null, typeof.GetProperty "Greeting") + Assert.AreEqual(ValueSome(box "hello"), readByReflection staticProperty) + let staticField = Expression.Field(null, typeof.GetField "Empty") + Assert.AreEqual(ValueSome(box ""), readByReflection staticField) + let literalField = Expression.Field(null, typeof.GetField "MaxValue") + Assert.AreEqual(ValueSome(box Int32.MaxValue), readByReflection literalField) + +[] +let ``nested closures are read through their member chain``() = + let inner = InnerClosure(Outer = OuterClosure(Ids = [ 1; 2; 3 ])) + let ids = field (field (constant inner) "Outer") "Ids" + Assert.AreEqual(ValueSome(box [ 1; 2; 3 ]), readByReflection ids) + +[] +let ``values captured by F# lambdas are evaluated``() = + let outer (x: int) = + let y = x + 1 + ToExpression.Of(fun () -> (x, y)) + let lambda = outer 41 + Assert.AreEqual(box (41, 42), evaluate lambda.Body) + let ids = [ "a"; "b" ] + Assert.AreEqual(box ids, evaluate (ToExpression.Of(fun () -> ids)).Body) + +[] +let ``conversions are read, and ones that compute a value give what compiled code gives``() = + Assert.AreEqual(ValueSome(box 5), readByReflection (Expression.Convert(constant 5, typeof))) + Assert.AreEqual(ValueSome(box 5), readByReflection (Expression.Convert(constant 5, typeof>))) + Assert.AreEqual(ValueSome(box 5), readByReflection (Expression.Convert(Expression.Constant(box 5, typeof), typeof))) + Assert.AreEqual(ValueSome(null: obj), readByReflection (Expression.Convert(Expression.Constant(null, typeof), typeof))) + Assert.AreEqual(ValueSome(box 5L), readByReflection (Expression.Convert(constant 5, typeof))) + Assert.AreEqual(ValueSome(box 2), readByReflection (Expression.Convert(constant 2.7, typeof))) + Assert.AreEqual(ValueSome(box DayOfWeek.Friday), readByReflection (Expression.Convert(constant 5, typeof))) + Assert.Throws(fun () -> evaluate (Expression.ConvertChecked(constant 300, typeof)) |> ignore) |> ignore + Assert.Throws(fun () -> evaluate (Expression.Convert(Expression.Constant(box 5, typeof), typeof)) |> ignore) |> ignore + Assert.Throws(fun () -> evaluate (Expression.Convert(Expression.Constant(null, typeof), typeof)) |> ignore) |> ignore + Assert.AreEqual(ValueSome(box "s"), readByReflection (Expression.TypeAs(Expression.Constant(box "s", typeof), typeof))) + Assert.AreEqual(ValueSome(null: obj), readByReflection (Expression.TypeAs(Expression.Constant(box 5, typeof), typeof))) + +[] +let ``a quote evaluates to its expression``() = + let lambda = Expression.Lambda>(constant 1) + match readByReflection (Expression.Quote lambda) with + | ValueSome value -> Assert.AreSame(lambda, value) + | ValueNone -> Assert.Fail "a quote is read without evaluating" + +[] +let ``a getter's exception is raised as it is``() = + let holder = Holder("h", 1, { X = 0; Y = 0 }) + Assert.Throws(fun () -> evaluate (property (constant holder) "Failing") |> ignore) |> ignore + +[] +let ``other expressions are interpreted``() = + let sum = Expression.Add(constant 1, constant 2) + Assert.AreEqual(ValueNone, readByReflection sum) + Assert.AreEqual(box 3, evaluate sum) + let created = Expression.New(typeof.GetConstructor [| typeof; typeof; typeof |], constant 2000, constant 1, constant 2) + Assert.AreEqual(ValueNone, readByReflection created) + Assert.AreEqual(box (DateTime(2000, 1, 2)), evaluate created) + +[] +let ``static and instance method calls on readable values are read by reflection``() = + let concat = Expression.Call(typeof.GetMethod("Concat", [| typeof; typeof |]), constant "a", constant "b") + Assert.AreEqual(ValueSome(box "ab"), readByReflection concat) + let holder = Holder("name", 1, { X = 0; Y = 0 }) + let upper = Expression.Call(property (constant holder) "Name", typeof.GetMethod("ToUpperInvariant")) + Assert.AreEqual(ValueSome(box "NAME"), readByReflection upper) + let addDays = Expression.Call(constant (DateTime(2026, 1, 1)), typeof.GetMethod("AddDays"), constant 1.0) + Assert.AreEqual(ValueSome(box (DateTime(2026, 1, 2))), readByReflection addDays) + let voidCall = Expression.Call(typeof.GetMethod("KeepAlive"), Expression.Constant(null, typeof)) + Assert.AreEqual(ValueSome(null: obj), readByReflection voidCall) + +/// Queryable.Where over ints with a quoted predicate comparing the item to the given expression. +let private whereGreaterThan (limit: Expression) = + let x = Expression.Parameter(typeof, "x") + let predicate = Expression.Lambda>(Expression.GreaterThan(x, limit), x) + let where = typeof.GetMethods() |> Array.find (fun m -> m.Name = "Where" && m.GetParameters().[1].ParameterType.GetGenericArguments().[0].GetGenericArguments().Length = 2) + Expression.Call(where.MakeGenericMethod typeof, constant ([ 1; 2; 3 ].AsQueryable()), Expression.Quote predicate) + +[] +let ``a call with a quoted argument passes the expression``() = + match readByReflection (whereGreaterThan (constant 1)) with + | ValueSome (:? IQueryable as filtered) -> Assert.AreEqual([ 2; 3 ], filtered |> Seq.toList) + | other -> Assert.Fail(sprintf "a quoted argument is read without evaluating, got %A" other) + +[] +let ``a quote that uses an outer lambda's parameter is not evaluated``() = + let call = whereGreaterThan (Expression.Parameter(typeof, "outer")) + Assert.AreEqual(ValueNone, readByReflection call) + Assert.Throws(fun () -> evaluate call |> ignore) |> ignore + +type Counter() = + let mutable calls = 0 + member _.Calls = calls + member _.Next() = + calls <- calls + 1 + calls + member _.Missing() : string = + calls <- calls + 1 + null + +[] +let ``a captured method is called once``() = + let call (counter: Counter) (name: string) = Expression.Call(constant counter, typeof.GetMethod name) :> Expression + let max = typeof.GetMethod("Max", [| typeof; typeof |]) + let cases: (string * (Counter -> Expression)) list = + [ "under a conversion that computes a value", fun c -> Expression.Convert(call c "Next", typeof) + "beside an argument that is interpreted", fun c -> Expression.Call(max, call c "Next", Expression.Negate(constant 4)) + "before a member of its null result", fun c -> Expression.Property(call c "Missing", "Length") ] + for name, build in cases do + let counter = Counter() + try evaluate (build counter) |> ignore + with :? NullReferenceException -> () + Assert.AreEqual(1, counter.Calls, name) + +[] +let ``calls left to the interpreter``() = + let point = { X = 1; Y = 2 } + let structCall = Expression.Call(constant point, typeof.GetMethod("ToString", Type.EmptyTypes)) + Assert.AreEqual(ValueNone, readByReflection structCall) + Assert.AreEqual(box (point.ToString()), evaluate structCall) + let unreadableArgument = Expression.Call(typeof.GetMethod("Abs", [| typeof |]), Expression.Negate(constant 4)) + Assert.AreEqual(ValueNone, readByReflection unreadableArgument) + Assert.AreEqual(box 4, evaluate unreadableArgument) + +[] +let ``a method of null throws as compiled code does``() = + let call = Expression.Call(Expression.Constant(null, typeof), typeof.GetMethod("ToUpperInvariant")) + Assert.Throws(fun () -> evaluate call |> ignore) |> ignore + +[] +let ``a method's exception is raised as it is``() = + let call = Expression.Call(typeof.GetMethod("Fail")) + Assert.Throws(fun () -> evaluate call |> ignore) |> ignore + +let private addConstant (value: int) = + let x = Expression.Parameter(typeof, "x") + Expression.Lambda>(Expression.Add(x, Expression.Constant value), x) + +[] +let ``lambdas that differ only in constants share one compiled shape``() = + let first = compile (addConstant 1000001) :?> Func + let before = cachedShapes () + let second = compile (addConstant 1000002) :?> Func + Assert.AreEqual(before, cachedShapes ()) + Assert.AreEqual(1000011, first.Invoke 10) + Assert.AreEqual(1000012, second.Invoke 10) + +[] +let ``lambdas of other shapes are not mixed up``() = + let x = Expression.Parameter(typeof, "x") + let multiply = Expression.Lambda>(Expression.Multiply(x, Expression.Constant 3), x) + Assert.AreEqual(30, (compile multiply :?> Func).Invoke 10) + Assert.AreEqual(13, (compile (addConstant 3) :?> Func).Invoke 10) + let a, b = Expression.Parameter(typeof, "a"), Expression.Parameter(typeof, "b") + let aMinusB = Expression.Lambda>(Expression.Subtract(a, b), a, b) + let bMinusA = Expression.Lambda>(Expression.Subtract(b, a), a, b) + Assert.AreEqual(7, (compile aMinusB :?> Func).Invoke(10, 3)) + Assert.AreEqual(-7, (compile bMinusA :?> Func).Invoke(10, 3)) + +[] +let ``a constant used twice and two constants of one value are each bound``() = + let x = Expression.Parameter(typeof, "x") + let shared = Expression.Constant 2000001 + let twice = Expression.Lambda>(Expression.Add(x, Expression.Add(shared, shared)), x) + let distinct = Expression.Lambda>(Expression.Add(x, Expression.Add(Expression.Constant 2000001, Expression.Constant 5)), x) + Assert.AreEqual(4000002, (compile twice :?> Func).Invoke 0) + Assert.AreEqual(2000006, (compile distinct :?> Func).Invoke 0) + +[] +let ``compiled lambdas keep reference, null and nullable constants``() = + let s = Expression.Parameter(typeof, "s") + let concat = typeof.GetMethod("Concat", [| typeof; typeof |]) + let prefixed (prefix: string) = Expression.Lambda>(Expression.Call(concat, Expression.Constant(prefix, typeof), s), s) + Assert.AreEqual("ab", (compile (prefixed "a") :?> Func).Invoke "b") + Assert.AreEqual("b", (compile (prefixed null) :?> Func).Invoke "b") + let n = Expression.Parameter(typeof>, "n") + let orDefault (value: Nullable) = Expression.Lambda, Nullable>>(Expression.Coalesce(n, Expression.Constant(value, typeof>)), n) + Assert.AreEqual(Nullable 4, (compile (orDefault (Nullable 4)) :?> Func, Nullable>).Invoke(Nullable())) + Assert.AreEqual(Nullable(), (compile (orDefault (Nullable())) :?> Func, Nullable>).Invoke(Nullable())) + +[] +let ``a lambda holding a quote is compiled without the cache``() = + let inner = Expression.Lambda>(Expression.Constant 3000001) + let quoting = Expression.Lambda>(Expression.Convert(Expression.Quote inner, typeof)) + let before = cachedShapes () + let quoted = (compile quoting :?> Func).Invoke() :?> Expression> + Assert.AreEqual(before, cachedShapes ()) + Assert.AreEqual(3000001, quoted.Compile().Invoke()) diff --git a/tests/SqlProvider.Tests/SqlProvider.Tests.fsproj b/tests/SqlProvider.Tests/SqlProvider.Tests.fsproj index 14951dfc..c1c02e71 100644 --- a/tests/SqlProvider.Tests/SqlProvider.Tests.fsproj +++ b/tests/SqlProvider.Tests/SqlProvider.Tests.fsproj @@ -36,6 +36,7 @@ +