From 4ff975c6f3d7005ffccd15ac4abbab0e4acc789f Mon Sep 17 00:00:00 2001 From: Tuomas Hietanen Date: Sat, 21 Oct 2023 03:20:20 +0100 Subject: [PATCH] Fix for #770. Note: 1000 would have created big JSON schemas and slow VS. --- src/SQLProvider.DesignTime/SqlDesignTime.fs | 70 ++++++++++---------- src/SQLProvider.Runtime/SqlRuntime.Common.fs | 7 +- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/src/SQLProvider.DesignTime/SqlDesignTime.fs b/src/SQLProvider.DesignTime/SqlDesignTime.fs index 73e5223f..100197f0 100644 --- a/src/SQLProvider.DesignTime/SqlDesignTime.fs +++ b/src/SQLProvider.DesignTime/SqlDesignTime.fs @@ -191,23 +191,43 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = | None -> prov.GetSchemaCache().Columns.TryGetValue(table.FullName) |> function | true,cols -> cols | false, _ -> Map.empty match prov.GetPrimaryKey table with | Some pkName -> + + let rec (|FixedType|_|) (o:obj) = + match o, o.GetType().IsValueType with + // watch out for normal strings + | :? string, _ -> Some o + // special case for guids as they are not a supported quotable constant in the TP mechanics, + // but we can deal with them as strings. + | :? Guid, _ -> Some (box (o.ToString())) + // Postgres also supports arrays + | :? Array as arr, _ when dbVendor = DatabaseProviderTypes.POSTGRESQL -> Some (box arr) + // value types in general work + | _, true -> Some o + // can't support any other types + | _, _ -> None + let entities = - match con with - | Some con -> - use com = prov.CreateCommand(con,prov.GetIndividualsQueryText(table,individualsAmount)) - if con.State <> ConnectionState.Open then con.Open() - use reader = com.ExecuteReader() - let ret = (designTimeDc :> ISqlDataContext).ReadEntities(table.FullName, columns, reader) - if (dbVendor <> DatabaseProviderTypes.MSACCESS) then con.Close() - if ret.Length > 0 then - prov.GetSchemaCache().Individuals.AddRange ret - ret - | None -> prov.GetSchemaCache().Individuals |> Seq.toArray + prov.GetSchemaCache().Individuals.GetOrAdd((table.FullName+"_"+pkName), fun k -> + match con with + | Some con -> + use com = prov.CreateCommand(con,prov.GetIndividualsQueryText(table,individualsAmount)) + if con.State <> ConnectionState.Open then con.Open() + use reader = com.ExecuteReader() + let ret = (designTimeDc :> ISqlDataContext).ReadEntities(table.FullName+"_"+pkName, columns, reader) + if (dbVendor <> DatabaseProviderTypes.MSACCESS) then con.Close() + let mapped = ret |> Array.choose(fun e -> + match e.GetColumn pkName with + | FixedType pkValue -> Some (pkValue, e.ColumnValues |> dict) + | _ -> None) + mapped + | None -> [||] + ) if Array.isEmpty entities then [] else // for each column in the entity except the primary key, create a new type that will read ``As Column 1`` etc // inside that type the individuals will be listed again but with the text for the relevant column as the name // of the property and the primary key e.g. ``1, Dennis The Squirrel`` let buildFieldName = SchemaProjections.buildFieldName + let propertyMap = match con with | Some con -> prov.GetColumns(con,table) @@ -221,21 +241,6 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = Some(col.Key,(ty,ProvidedProperty(sprintf "As %s" (buildFieldName col.Key),ty, getterCode = fun args -> <@@ ((%%args.[0] : obj) :?> ISqlDataContext)@@> )))) |> Map.ofSeq - let rec (|FixedType|_|) (o:obj) = - match o, o.GetType().IsValueType with - // watch out for normal strings - | :? string, _ -> Some o - // special case for guids as they are not a supported quotable constant in the TP mechanics, - // but we can deal with them as strings. - | :? Guid, _ -> Some (box (o.ToString())) - // Postgres also supports arrays - | :? Array as arr, _ when dbVendor = DatabaseProviderTypes.POSTGRESQL -> Some (box arr) - // value types in general work - | _, true -> Some o - // can't support any other types - | _, _ -> None - - let prettyPrint (value : obj) = let dirtyName = match value with @@ -247,17 +252,14 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = // on the main object create a property for each entity simply using the primary key let props = entities - |> Array.choose(fun e -> - match e.GetColumn pkName with - | FixedType pkValue -> - + |> Array.choose(fun (pkValue, columnValues) -> let tableName = table.FullName let getterCode (args : Expr list) = let a0 = args.[0] <@@ ((%%a0 : obj) :?> ISqlDataContext).GetIndividual(tableName, pkValue) @@> // this next bit is just side effect to populate the "As Column" types for the supported columns - for colName, colValue in e.ColumnValues do + for colName, colValue in columnValues |> Seq.map(fun kvp -> kvp.Key, kvp.Value) do if colName <> pkName then let colDefinition, _ = propertyMap.[colName] colDefinition.AddMemberDelayed(fun() -> @@ -271,7 +273,7 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = , tableTypeDef , getterCode = getterCode ) - | _ -> None) + ) |> Array.append( propertyMap |> Map.toArray |> Array.map (snd >> snd)) propertyMap @@ -1044,7 +1046,7 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = let connStringName = ProvidedStaticParameter("ConnectionStringName", typeof, "") let optionTypes = ProvidedStaticParameter("UseOptionTypes",typeof,NullableColumnType.NO_OPTION) let dbVendor = ProvidedStaticParameter("DatabaseVendor",typeof,DatabaseProviderTypes.MSSQLSERVER) - let individualsAmount = ProvidedStaticParameter("IndividualsAmount",typeof,1000) + let individualsAmount = ProvidedStaticParameter("IndividualsAmount",typeof,50) let owner = ProvidedStaticParameter("Owner", typeof, "") let resolutionPath = ProvidedStaticParameter("ResolutionPath",typeof, "") let caseSensitivity = ProvidedStaticParameter("CaseSensitivityChange",typeof,CaseSensitivityChange.ORIGINAL) @@ -1057,7 +1059,7 @@ type public SqlTypeProvider(config: TypeProviderConfig) as this = The connection string for the SQL database The connection string name to select from a configuration file The target database vendor - The amount of sample entities to project into the type system for each SQL entity type. Default 1000. + The amount of sample entities to project into the type system for each SQL entity type. Default 50. Note GDPR/PII regulations if using individuals with ContextSchemaPath. If set, F# option types will be used in place of nullable database columns. If not, you will always receive the default value of the column's type even if it is null in the database. The location to look for dynamically loaded assemblies containing database vendor specific connections and custom types. Oracle: The owner of the schema for this provider to resolve. PostgreSQL: A list of schemas to resolve, separated by spaces, newlines, commas, or semicolons. diff --git a/src/SQLProvider.Runtime/SqlRuntime.Common.fs b/src/SQLProvider.Runtime/SqlRuntime.Common.fs index cce26892..5c30c0c1 100644 --- a/src/SQLProvider.Runtime/SqlRuntime.Common.fs +++ b/src/SQLProvider.Runtime/SqlRuntime.Common.fs @@ -645,7 +645,7 @@ and internal SchemaCache = Sprocs : ResizeArray SprocsParams : ConcurrentDictionary //sproc name and params Packages : ResizeArray - Individuals : ResizeArray + Individuals : ConcurrentDictionary) array> //table name and entities IsOffline : bool } with static member Empty = { @@ -656,7 +656,7 @@ and internal SchemaCache = Sprocs = ResizeArray() SprocsParams = ConcurrentDictionary() Packages = ResizeArray() - Individuals = ResizeArray() + Individuals = ConcurrentDictionary() IsOffline = false } static member Load(filePath) = use ms = new MemoryStream(Encoding.UTF8.GetBytes(File.ReadAllText(filePath))) @@ -868,7 +868,8 @@ module public OfflineTools = SprocsParams = System.Collections.Concurrent.ConcurrentDictionary( Seq.concat [|s1.SprocsParams ; s2.SprocsParams |] |> Seq.distinctBy(fun d -> d.Key)); Packages = ResizeArray(Seq.concat [| s1.Packages ; s2.Packages |] |> Seq.distinctBy(fun s -> s.ToString())); - Individuals = ResizeArray(Seq.concat [| s1.Individuals ; s2.Individuals |] |> Seq.distinct); + Individuals = System.Collections.Concurrent.ConcurrentDictionary( + Seq.concat [|s1.Individuals ; s2.Individuals |] |> Seq.distinctBy(fun d -> d.Key)); IsOffline = s1.IsOffline || s2.IsOffline} merged.Save targetfile "Merge saved " + targetfile + " at " + DateTime.Now.ToString("hh:mm:ss")