diff --git a/README.md b/README.md index b1f5954..17f28a0 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ The goal is to create a provider which allows lightweight access to your Azure s Support exists reading and writing from Blobs, Tables and Queues, as well as fallback to the standard .NET Azure SDK. +[![AppVeyor build status](https://ci.appveyor.com/api/projects/status/github/fsprojects/AzureStorageTypeProvider?svg=true)](https://ci.appveyor.com/project/fsprojectsgit/azurestoragetypeprovider) + ### Maintainer(s) - [@isaacabraham](https://github.com/isaacabraham) diff --git a/paket.dependencies b/paket.dependencies index 371c813..d10153b 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -8,6 +8,7 @@ nuget Unquote nuget WindowsAzure.Storage 6.0.0 nuget xunit 1.9.2 nuget xunit.runners 1.9.2 +nuget xunit.runner.visualstudio version_in_path: true nuget FSharp.Formatting nuget Deedle nuget FSharp.Charting @@ -16,4 +17,3 @@ nuget Newtonsoft.Json 6.0.8 github fsprojects/FSharp.TypeProviders.StarterPack src/ProvidedTypes.fsi github fsprojects/FSharp.TypeProviders.StarterPack src/ProvidedTypes.fs -github fsprojects/FSharp.TypeProviders.StarterPack src/DebugProvidedTypes.fs diff --git a/paket.lock b/paket.lock index 926141c..47fcb3c 100644 --- a/paket.lock +++ b/paket.lock @@ -1,40 +1,38 @@ REDIRECTS: ON NUGET - remote: https://nuget.org/api/v2 - specs: + remote: https://www.nuget.org/api/v2 Deedle (1.2.4) FAKE (4.4.5) FSharp.Charting (0.90.12) FSharp.Compiler.Service (0.0.90) FSharp.Core (3.1.2.5) - FSharp.Formatting (2.11.0) + FSharp.Formatting (2.11) FSharp.Compiler.Service (>= 0.0.90 <= 1.3) - FSharpVSPowerTools.Core (1.9.0) - FSharpVSPowerTools.Core (1.9.0) + FSharpVSPowerTools.Core (1.9) + FSharpVSPowerTools.Core (1.9) FSharp.Compiler.Service (>= 0.0.90) FsUnit.xUnit (1.3.1) xunit (1.9.2) - Microsoft.Azure.KeyVault.Core (1.0.0) - framework: wpv8.0, >= net40 - Microsoft.Data.Edm (5.6.4) - framework: winv4.5, wpv8.1, wpv8.0, >= net40 - Microsoft.Data.OData (5.6.4) - framework: winv4.5, wpv8.1, wpv8.0, >= net40 + Microsoft.Azure.KeyVault.Core (1.0) - framework: >= net40, wpv8.0 + Microsoft.Data.Edm (5.6.4) - framework: >= net40, winv4.5, wpv8.0, wpav8.1 + Microsoft.Data.OData (5.6.4) - framework: >= net40, winv4.5, wpv8.0, wpav8.1 Microsoft.Data.Edm (5.6.4) System.Spatial (5.6.4) Microsoft.Data.Services.Client (5.6.4) - framework: >= net40 Microsoft.Data.OData (5.6.4) Newtonsoft.Json (6.0.8) NuGet.CommandLine (2.8.6) - System.Spatial (5.6.4) - framework: winv4.5, wpv8.1, wpv8.0, >= net40 - Unquote (3.0.0) - WindowsAzure.Storage (6.0.0) - Microsoft.Azure.KeyVault.Core (>= 1.0.0) - framework: wpv8.0, >= net40 - Microsoft.Data.OData (>= 5.6.4) - framework: winv4.5, wpv8.1, wpv8.0, >= net40 + System.Spatial (5.6.4) - framework: >= net40, winv4.5, wpv8.0, wpav8.1 + Unquote (3.0) + WindowsAzure.Storage (6.0) + Microsoft.Azure.KeyVault.Core (>= 1.0) - framework: >= net40, wpv8.0 + Microsoft.Data.OData (>= 5.6.4) - framework: >= net40, winv4.5, wpv8.0, wpav8.1 Microsoft.Data.Services.Client (>= 5.6.4) - framework: >= net40 - Newtonsoft.Json (>= 6.0.8) - framework: winv4.5, wpv8.1, wpv8.0, >= net40 + Newtonsoft.Json (>= 6.0.8) - framework: >= net40, winv4.5, wpv8.0, wpav8.1 xunit (1.9.2) + xunit.runner.visualstudio (2.1.0) - version_in_path: true xunit.runners (1.9.2) GITHUB remote: fsprojects/FSharp.TypeProviders.StarterPack - specs: - src/DebugProvidedTypes.fs (f1234ba078dad7cbb674b1d1ab8b94f92f827911) - src/ProvidedTypes.fs (f1234ba078dad7cbb674b1d1ab8b94f92f827911) - src/ProvidedTypes.fsi (f1234ba078dad7cbb674b1d1ab8b94f92f827911) \ No newline at end of file + src/ProvidedTypes.fs (dfbca9b83fb70067e85abddcb8b89332aae4c28d) + src/ProvidedTypes.fsi (dfbca9b83fb70067e85abddcb8b89332aae4c28d) \ No newline at end of file diff --git a/src/FSharp.Azure.StorageTypeProvider/FSharp.Azure.StorageTypeProvider.fsproj b/src/FSharp.Azure.StorageTypeProvider/FSharp.Azure.StorageTypeProvider.fsproj index 0126cd2..dc2f852 100644 --- a/src/FSharp.Azure.StorageTypeProvider/FSharp.Azure.StorageTypeProvider.fsproj +++ b/src/FSharp.Azure.StorageTypeProvider/FSharp.Azure.StorageTypeProvider.fsproj @@ -43,6 +43,14 @@ ..\..\bin\FSharp.Azure.StorageTypeProvider.XML + + True + paket-files/ProvidedTypes.fsi + + + True + paket-files/ProvidedTypes.fs + @@ -54,15 +62,6 @@ - - paket-files/ProvidedTypes.fsi - - - paket-files/ProvidedTypes.fs - - - paket-files/DebugProvidedTypes.fs - @@ -106,7 +105,7 @@ - + ..\..\packages\FSharp.Core\lib\net40\FSharp.Core.dll @@ -115,46 +114,46 @@ - + - ..\..\packages\FSharp.Core\lib\portable-net45+netcore45\FSharp.Core.dll + ..\..\packages\FSharp.Core\lib\portable-net45+monoandroid10+monotouch10+xamarinios10\FSharp.Core.dll True True - + - ..\..\packages\FSharp.Core\lib\portable-net45+monoandroid10+monotouch10+xamarinios10\FSharp.Core.dll + ..\..\packages\FSharp.Core\lib\portable-net45+netcore45\FSharp.Core.dll True True - + - ..\..\packages\FSharp.Core\lib\portable-net45+sl5+netcore45\FSharp.Core.dll + ..\..\packages\FSharp.Core\lib\portable-net45+netcore45+wp8\FSharp.Core.dll True True - + - ..\..\packages\FSharp.Core\lib\portable-net45+netcore45+wp8\FSharp.Core.dll + ..\..\packages\FSharp.Core\lib\portable-net45+netcore45+wpa81+wp8\FSharp.Core.dll True True - + - ..\..\packages\FSharp.Core\lib\portable-net45+netcore45+wpa81+wp8\FSharp.Core.dll + ..\..\packages\FSharp.Core\lib\portable-net45+sl5+netcore45\FSharp.Core.dll True True @@ -162,7 +161,7 @@ - + ..\..\packages\Microsoft.Azure.KeyVault.Core\lib\net40\Microsoft.Azure.KeyVault.Core.dll @@ -171,7 +170,7 @@ - + ..\..\packages\Microsoft.Azure.KeyVault.Core\lib\portable-net45+wp8+wpa81+win\Microsoft.Azure.KeyVault.Core.dll @@ -182,7 +181,7 @@ - + ..\..\packages\Microsoft.Data.Edm\lib\net40\Microsoft.Data.Edm.dll @@ -191,7 +190,7 @@ - + ..\..\packages\Microsoft.Data.Edm\lib\portable-net45+wp8+win8+wpa\Microsoft.Data.Edm.dll @@ -202,7 +201,7 @@ - + ..\..\packages\Microsoft.Data.OData\lib\net40\Microsoft.Data.OData.dll @@ -211,7 +210,7 @@ - + ..\..\packages\Microsoft.Data.OData\lib\portable-net45+wp8+win8+wpa\Microsoft.Data.OData.dll @@ -222,7 +221,7 @@ - + ..\..\packages\Microsoft.Data.Services.Client\lib\net40\Microsoft.Data.Services.Client.dll @@ -233,10 +232,10 @@ - + - ..\..\packages\Newtonsoft.Json\lib\netcore45\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\net20\Newtonsoft.Json.dll True True @@ -251,46 +250,46 @@ - + - ..\..\packages\Newtonsoft.Json\lib\net20\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\net40\Newtonsoft.Json.dll True True - + - ..\..\packages\Newtonsoft.Json\lib\net40\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll True True - + - ..\..\packages\Newtonsoft.Json\lib\net45\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\netcore45\Newtonsoft.Json.dll True True - + - ..\..\packages\Newtonsoft.Json\lib\portable-net45+wp80+win8+wpa81+aspnetcore50\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\portable-net40+sl5+wp80+win8+wpa81\Newtonsoft.Json.dll True True - + - ..\..\packages\Newtonsoft.Json\lib\portable-net40+sl5+wp80+win8+wpa81\Newtonsoft.Json.dll + ..\..\packages\Newtonsoft.Json\lib\portable-net45+wp80+win8+wpa81+aspnetcore50\Newtonsoft.Json.dll True True @@ -298,7 +297,7 @@ - + ..\..\packages\System.Spatial\lib\net40\System.Spatial.dll @@ -307,7 +306,7 @@ - + ..\..\packages\System.Spatial\lib\portable-net45+wp8+win8+wpa\System.Spatial.dll @@ -318,19 +317,19 @@ - + - ..\..\packages\WindowsAzure.Storage\lib\win8\Microsoft.WindowsAzure.Storage.dll + ..\..\packages\WindowsAzure.Storage\lib\net40\Microsoft.WindowsAzure.Storage.dll True True - + - ..\..\packages\WindowsAzure.Storage\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\..\packages\WindowsAzure.Storage\lib\win8\Microsoft.WindowsAzure.Storage.dll True True diff --git a/src/FSharp.Azure.StorageTypeProvider/Shared.fs b/src/FSharp.Azure.StorageTypeProvider/Shared.fs index c6eebd3..45d7fb9 100644 --- a/src/FSharp.Azure.StorageTypeProvider/Shared.fs +++ b/src/FSharp.Azure.StorageTypeProvider/Shared.fs @@ -9,4 +9,10 @@ let internal ofNullable (value : Nullable<_>) = let internal toNullable = function | Some x -> Nullable x - | None -> Nullable() \ No newline at end of file + | None -> Nullable() + +let internal ofObj (value) = if value = null then None else Some value + +let internal toObj = function + | Some x -> x + | None -> null \ No newline at end of file diff --git a/src/FSharp.Azure.StorageTypeProvider/Table/ProvidedTableTypes.fs b/src/FSharp.Azure.StorageTypeProvider/Table/ProvidedTableTypes.fs index 125ca08..2ead074 100644 --- a/src/FSharp.Azure.StorageTypeProvider/Table/ProvidedTableTypes.fs +++ b/src/FSharp.Azure.StorageTypeProvider/Table/ProvidedTableTypes.fs @@ -11,28 +11,36 @@ type AzureTable internal (defaultConnection, tableName) = let getConnectionDetails (insertMode, connectionString) = defaultArg insertMode TableInsertMode.Insert, defaultArg connectionString defaultConnection let getTableForConnection = getTable tableName - - /// Gets a handle to the Azure SDK client for this table. - member __.AsCloudTable(?connectionString) = getTableForConnection (defaultArg connectionString defaultConnection) - /// Inserts a batch of entities into the table, using all public properties on the object as fields. - member __.Insert(entities : seq, ?insertMode, ?connectionString) = + let buildInsertParams insertMode connectionString (entities : seq) = let insertMode, connectionString = getConnectionDetails (insertMode, connectionString) let table = getTableForConnection connectionString let insertOp = createInsertOperation insertMode - + let propBuilders = typeof<'b>.GetProperties(Reflection.BindingFlags.Public ||| Reflection.BindingFlags.Instance) |> Seq.map (fun prop entity -> prop.Name, prop.GetValue(entity, null)) |> Seq.toArray - entities - |> Seq.map (fun (partitionKey, rowKey, entity) -> - LightweightTableEntity(partitionKey, rowKey, DateTimeOffset.MinValue, - propBuilders - |> Seq.map (fun builder -> builder (entity)) - |> Map.ofSeq)) - |> Seq.map buildDynamicTableEntity - |> executeBatchOperation insertOp table + + let tblEntities = + entities + |> Seq.map (fun (partitionKey, rowKey, entity) -> + let values = propBuilders |> Seq.map (fun builder -> builder entity) |> Map.ofSeq + LightweightTableEntity(partitionKey, rowKey, DateTimeOffset.MinValue, values) |> buildDynamicTableEntity) + tblEntities, insertOp, table + + /// Gets a handle to the Azure SDK client for this table. + member __.AsCloudTable(?connectionString) = getTableForConnection (defaultArg connectionString defaultConnection) + + /// Inserts a batch of entities into the table, using all public properties on the object as fields. + member __.Insert(entities : seq, ?insertMode, ?connectionString) = + let tblEntities, insertOp, table = buildInsertParams insertMode connectionString entities + tblEntities |> executeBatchOperation insertOp table + + /// Inserts a batch of entities into the table, using all public properties on the object as fields. + member __.InsertAsync(entities : seq, ?insertMode, ?connectionString) = async{ + let tblEntities, insertOp, table = buildInsertParams insertMode connectionString entities + return! tblEntities |> executeBatchOperationAsync insertOp table } /// Inserts a single entity into the table, using public properties on the object as fields. member this.Insert(partitionKey, rowKey, entity, ?insertMode, ?connectionString) = @@ -41,26 +49,60 @@ type AzureTable internal (defaultConnection, tableName) = |> Seq.head |> snd |> Seq.head + + /// Inserts a single entity into the table asynchronously, using public properties on the object as fields. + member this.InsertAsync(partitionKey, rowKey, entity, ?insertMode, ?connectionString) = async{ + let insertMode, connectionString = getConnectionDetails (insertMode, connectionString) + let! insertRes = this.InsertAsync([ partitionKey, rowKey, entity ], insertMode, connectionString) + return + insertRes + |> Seq.head + |> snd + |> Seq.head } - /// Deletes a batch of entities from the table using the supplied pairs of Partition and Row keys. + /// Deletes a batch of entities from the table using the supplied pairs of Partition and Row keys. member __.Delete(entities, ?connectionString) = let table = getTableForConnection (defaultArg connectionString defaultConnection) entities |> Seq.map (fun entityId -> - let Partition(partitionKey), Row(rowKey) = entityId - DynamicTableEntity(partitionKey, rowKey, ETag = "*")) + let Partition(partitionKey), Row(rowKey) = entityId + DynamicTableEntity(partitionKey, rowKey, ETag = "*")) |> executeBatchOperation TableOperation.Delete table + /// Asynchronously deletes a batch of entities from the table using the supplied pairs of Partition and Row keys. + member __.DeleteAsync(entities, ?connectionString) = async { + let table = getTableForConnection (defaultArg connectionString defaultConnection) + return! entities + |> Seq.map (fun entityId -> + let Partition(partitionKey), Row(rowKey) = entityId + DynamicTableEntity(partitionKey, rowKey, ETag = "*")) + |> executeBatchOperationAsync TableOperation.Delete table } + /// Deletes an entire partition from the table member __.DeletePartition(partitionKey, ?connectionString) = let table = getTableForConnection (defaultArg connectionString defaultConnection) let filter = Table.TableQuery.GenerateFilterCondition ("PartitionKey", Table.QueryComparisons.Equal, partitionKey) let projection = [|"RowKey"|] + (new Table.TableQuery()).Where(filter).Select(projection) |> table.ExecuteQuery |> Seq.map(fun e -> (Partition(e.PartitionKey), Row(e.RowKey))) |> __.Delete - |> ignore + |> ignore + + /// Asynchronously deletes an entire partition from the table + member __.DeletePartitionAsync(partitionKey, ?connectionString) = async { + let table = getTableForConnection (defaultArg connectionString defaultConnection) + let connStringToUse = + match connectionString with + | Some c -> c + | None -> defaultConnection + let filter = Table.TableQuery.GenerateFilterCondition ("PartitionKey", Table.QueryComparisons.Equal, partitionKey) + let! response = executeGenericQueryAsync connStringToUse table.Name Int32.MaxValue filter (fun e -> (Partition(e.PartitionKey), Row(e.RowKey))) + return! + response + |> __.DeleteAsync + |> Async.Ignore } /// Gets the name of the table. member __.Name = tableName diff --git a/src/FSharp.Azure.StorageTypeProvider/Table/TableEntityMemberFactory.fs b/src/FSharp.Azure.StorageTypeProvider/Table/TableEntityMemberFactory.fs index 31fb254..a985c91 100644 --- a/src/FSharp.Azure.StorageTypeProvider/Table/TableEntityMemberFactory.fs +++ b/src/FSharp.Azure.StorageTypeProvider/Table/TableEntityMemberFactory.fs @@ -96,8 +96,17 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable ProvidedParameter("connectionString", typeof, optionalValue = connection) ], parentTableEntityType.MakeArrayType(), InvokeCode = (fun args -> <@@ getPartitionRows %%args.[1] %%args.[2] tableName @@>)) getPartition.AddXmlDocDelayed <| fun _ -> "Eagerly retrieves all entities in a table partition by its key." + + let getPartitionAsync = + ProvidedMethod + ("GetPartitionAsync", + [ ProvidedParameter("key", typeof) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], + typeof>.GetGenericTypeDefinition().MakeGenericType(parentTableEntityType.MakeArrayType()), + InvokeCode = (fun args -> <@@ getPartitionRowsAsync %%args.[1] %%args.[2] tableName @@>)) + getPartitionAsync.AddXmlDocDelayed <| fun _ -> "Asynchronously retrieves all entities in a table partition by its key." + let queryBuilderType, childTypes = TableQueryBuilder.createTableQueryType parentTableEntityType connection tableName propertiesCreated - let executeQuery = ProvidedMethod ("Query", @@ -116,6 +125,18 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable (typeof>).GetGenericTypeDefinition().MakeGenericType(parentTableEntityType), InvokeCode = (fun args -> <@@ getEntity (%%args.[1] : Row) (%%args.[2] : Partition) (%%args.[3] : string) tableName @@>)) getEntity.AddXmlDocDelayed <| fun _ -> "Gets a single entity based on the row and partition key." + let getEntityAsync = + let returnType = + let optionType = typeof>.GetGenericTypeDefinition().MakeGenericType(parentTableEntityType) + typeof>.GetGenericTypeDefinition().MakeGenericType(optionType) + ProvidedMethod + ("GetAsync", + [ ProvidedParameter("rowKey", typeof) + ProvidedParameter("partitionKey", typeof) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], + returnType, + InvokeCode = (fun args -> <@@ getEntityAsync (%%args.[1] : Row) (%%args.[2] : Partition) (%%args.[3] : string) tableName @@>)) + getEntityAsync.AddXmlDocDelayed <| fun _ -> "Gets a single entity based on the row and partition key asynchronously." let deleteEntity = ProvidedMethod ("Delete", @@ -123,6 +144,13 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof, InvokeCode = (fun args -> <@@ deleteEntity %%args.[2] tableName %%args.[1] @@>)) deleteEntity.AddXmlDocDelayed <| fun _ -> "Deletes a single entity from the table." + let deleteEntityAsync = + ProvidedMethod + ("DeleteAsync", + [ ProvidedParameter("entity", parentTableEntityType) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof>, + InvokeCode = (fun args -> <@@ deleteEntityAsync %%args.[2] tableName %%args.[1] @@>)) + deleteEntityAsync.AddXmlDocDelayed <| fun _ -> "Deletes a single entity from the table asynchronously." let deleteEntities = ProvidedMethod ("Delete", @@ -130,6 +158,13 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof<(string * TableResponse []) []>, InvokeCode = (fun args -> <@@ deleteEntities %%args.[2] tableName %%args.[1] @@>)) deleteEntities.AddXmlDocDelayed <| fun _ -> "Deletes a batch of entities from the table." + let deleteEntitiesAsync = + ProvidedMethod + ("DeleteAsync", + [ ProvidedParameter("entities", parentTableEntityType.MakeArrayType()) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof>, + InvokeCode = (fun args -> <@@ deleteEntitiesAsync %%args.[2] tableName %%args.[1] @@>)) + deleteEntitiesAsync.AddXmlDocDelayed <| fun _ -> "Deletes a batch of entities from the table asynchronously." let insertEntity = ProvidedMethod ("Insert", @@ -138,6 +173,14 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof, InvokeCode = (fun args -> <@@ insertEntity (%%args.[3] : string) tableName %%args.[2] (%%args.[1] : LightweightTableEntity) @@>)) insertEntity.AddXmlDocDelayed <| fun _ -> "Inserts a single entity with the inferred table schema into the table." + let insertEntityAsync = + ProvidedMethod + ("InsertAsync", + [ ProvidedParameter("entity", parentTableEntityType) + ProvidedParameter("insertMode", typeof, optionalValue = TableInsertMode.Insert) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof>, + InvokeCode = (fun args -> <@@ insertEntityAsync (%%args.[3] : string) tableName %%args.[2] (%%args.[1] : LightweightTableEntity) @@>)) + insertEntityAsync.AddXmlDocDelayed <| fun _ -> "Asynchronously inserts a single entity with the inferred table schema into the table." let insertEntities = ProvidedMethod ("Insert", @@ -146,5 +189,22 @@ let buildTableEntityMembers (parentTableType:ProvidedTypeDefinition, parentTable ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof<(string * TableResponse [])[]>, InvokeCode = (fun args -> <@@ insertEntityBatch (%%args.[3] : string) tableName %%args.[2] (%%args.[1] : LightweightTableEntity []) @@>)) insertEntities.AddXmlDocDelayed <| fun _ -> "Inserts a batch of entities with the inferred table schema into the table." + let insertEntitiesAsync = + ProvidedMethod + ("InsertAsync", + [ ProvidedParameter("entities", parentTableEntityType.MakeArrayType()) + ProvidedParameter("insertMode", typeof, optionalValue = TableInsertMode.Insert) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], returnType = typeof>, + InvokeCode = (fun args -> <@@ insertEntityBatchAsync (%%args.[3] : string) tableName %%args.[2] (%%args.[1] : LightweightTableEntity []) @@>)) + insertEntitiesAsync.AddXmlDocDelayed <| fun _ -> "Asynchronously inserts a batch of entities with the inferred table schema into the table." + domainType.AddMembers(queryBuilderType :: childTypes) - [ getPartition; getEntity; buildQuery; executeQuery; deleteEntity; deleteEntities; insertEntity; insertEntities ]) \ No newline at end of file + + [ getPartition; getPartitionAsync + getEntity; getEntityAsync + buildQuery + executeQuery + deleteEntity; deleteEntityAsync + deleteEntities; deleteEntitiesAsync + insertEntity; insertEntityAsync + insertEntities; insertEntitiesAsync ]) \ No newline at end of file diff --git a/src/FSharp.Azure.StorageTypeProvider/Table/TableQueryBuilder.fs b/src/FSharp.Azure.StorageTypeProvider/Table/TableQueryBuilder.fs index 48df189..3932832 100644 --- a/src/FSharp.Azure.StorageTypeProvider/Table/TableQueryBuilder.fs +++ b/src/FSharp.Azure.StorageTypeProvider/Table/TableQueryBuilder.fs @@ -56,7 +56,16 @@ let createTableQueryType (tableEntityType: ProvidedTypeDefinition) connection ta "RowKey", buildPropertyOperatorsType tableName "RowKey" EdmType.String tableQueryType "Timestamp", buildPropertyOperatorsType tableName "Timestamp" EdmType.DateTime tableQueryType ] @ [ for (name, value) in properties -> name, buildPropertyOperatorsType tableName name value.PropertyType tableQueryType ] + tableQueryType.AddMembersDelayed(fun () -> + let executeQueryMethodAsync = + ProvidedMethod + ("ExecuteAsync", [ ProvidedParameter("maxResults", typeof, optionalValue = 0) + ProvidedParameter("connectionString", typeof, optionalValue = connection) ], + typeof>.GetGenericTypeDefinition().MakeGenericType(tableEntityType.MakeArrayType()), + InvokeCode = (fun args -> <@@ executeQueryAsync (%%args.[2] : string) tableName %%args.[1] (composeAllFilters((%%args.[0]: obj) :?> string list)) @@>)) + executeQueryMethodAsync.AddXmlDocDelayed <| fun _ -> "Executes the current query asynchronously." + let executeQueryMethod = ProvidedMethod ("Execute", [ ProvidedParameter("maxResults", typeof, optionalValue = 0) @@ -64,10 +73,15 @@ let createTableQueryType (tableEntityType: ProvidedTypeDefinition) connection ta tableEntityType.MakeArrayType(), InvokeCode = (fun args -> <@@ executeQuery (%%args.[2] : string) tableName %%args.[1] (composeAllFilters((%%args.[0]: obj) :?> string list)) @@>)) executeQueryMethod.AddXmlDocDelayed <| fun _ -> "Executes the current query." + let customQueryProperties = [ for (name, operatorType) in operatorTypes -> let queryProperty = ProvidedProperty("Where" + name + "Is" |> splitOnCaps, operatorType, GetterCode = (fun args -> <@@ (%%args.[0]: obj) :?> string list @@>)) queryProperty.AddXmlDocDelayed <| fun _ -> sprintf "Creates a query part for the %s property." name queryProperty :> MemberInfo ] - (executeQueryMethod :> MemberInfo) :: customQueryProperties) + + (executeQueryMethodAsync :> MemberInfo) :: + (executeQueryMethod :> MemberInfo) :: + customQueryProperties) + tableQueryType, operatorTypes |> List.unzip |> snd \ No newline at end of file diff --git a/src/FSharp.Azure.StorageTypeProvider/Table/TableRepository.fs b/src/FSharp.Azure.StorageTypeProvider/Table/TableRepository.fs index e202552..64fb7e9 100644 --- a/src/FSharp.Azure.StorageTypeProvider/Table/TableRepository.fs +++ b/src/FSharp.Azure.StorageTypeProvider/Table/TableRepository.fs @@ -5,7 +5,6 @@ module FSharp.Azure.StorageTypeProvider.Table.TableRepository open FSharp.Azure.StorageTypeProvider.Table open Microsoft.WindowsAzure.Storage open Microsoft.WindowsAzure.Storage.Table -open Microsoft.WindowsAzure.Storage.Table.Queryable open System /// Suggests batch sizes based on a given entity type and published EDM property type sizes (source: https://msdn.microsoft.com/en-us/library/dd179338.aspx) @@ -62,19 +61,48 @@ let internal getRowsForSchema (rowCount: int) connection tableName = |> Seq.truncate rowCount |> Seq.toArray +let toLightweightTableEntity (dte:DynamicTableEntity) = + LightweightTableEntity( + Partition dte.PartitionKey, + Row dte.RowKey, + dte.Timestamp, + dte.Properties + |> Seq.map(fun p -> p.Key, p.Value.PropertyAsObject) + |> Map.ofSeq) + +let executeGenericQueryAsync connection tableName maxResults filterString mapToReturnEntity = async { + let query = + let query = DynamicQuery().Where(filterString) + if maxResults > 0 then query.Take(Nullable maxResults) else query + let table = getTable tableName connection + + let output = ResizeArray() + + let rec controlRec token = async { + let! rows, token = async { + let! batch = table.ExecuteQuerySegmentedAsync(query, token) |> Async.AwaitTask + return batch |> Seq.map mapToReturnEntity, batch.ContinuationToken |> Option.ofObj } + output.AddRange rows + + do! + match token with + | Some token -> controlRec token + | None -> async.Return() + + return () } + + do! controlRec null + return output |> Seq.toArray } + +let executeQueryAsync connection tableName maxResults filterString = + executeGenericQueryAsync connection tableName maxResults filterString toLightweightTableEntity + let executeQuery connection tableName maxResults filterString = let query = DynamicQuery().Where(filterString) let query = if maxResults > 0 then query.Take(Nullable maxResults) else query (getTable tableName connection).ExecuteQuery(query) - |> Seq.map(fun dte -> - LightweightTableEntity( - Partition dte.PartitionKey, - Row dte.RowKey, - dte.Timestamp, - dte.Properties - |> Seq.map(fun p -> p.Key, p.Value.PropertyAsObject) - |> Map.ofSeq)) + |> Seq.map(toLightweightTableEntity) |> Seq.toArray let internal buildDynamicTableEntity(entity:LightweightTableEntity) = @@ -107,7 +135,7 @@ let private batch size source = | head::tail -> doBatch output (head::currentBatch) (counter + 1) tail doBatch [] [] 0 (source |> Seq.toList) -let internal executeBatchOperation createTableOp (table:CloudTable) entities = +let private splitIntoBatches createTableOp entities = let batchSize = entities |> Seq.head |> BatchCalculator.getBatchSize entities |> Seq.groupBy(fun (entity:DynamicTableEntity) -> entity.PartitionKey) @@ -118,28 +146,58 @@ let internal executeBatchOperation createTableOp (table:CloudTable) entities = let batchForPartition = TableBatchOperation() entityBatch |> Seq.iter (createTableOp >> batchForPartition.Add) partitionKey, entityBatch, batchForPartition)) - |> Seq.map(fun (partitionKey, entityBatch, batchOperation) -> - let buildEntityId (entity:DynamicTableEntity) = Partition(entity.PartitionKey), Row(entity.RowKey) - let responses = - try - table.ExecuteBatch(batchOperation) + +let private processErrorResp entityBatch buildEntityId (ex:StorageException) = + let requestInformation = ex.RequestInformation + match requestInformation.ExtendedErrorInformation.ErrorMessage.Split('\n').[0].Split(':') with + | [|index; _|] -> + match Int32.TryParse(index) with + | true, index -> + entityBatch + |> Seq.mapi(fun entityIndex entity -> + if entityIndex = index then EntityError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode) + else BatchOperationFailedError(buildEntityId entity)) + | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) + | [| _ |] -> entityBatch |> Seq.map(fun entity -> EntityError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) + | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) + +let internal executeBatchAsynchronously batchOp entityBatch buildEntityId (table:CloudTable) = async{ + let! response = table.ExecuteBatchAsync(batchOp) |> Async.AwaitTask |> Async.Catch + match response with + | Choice1Of2 successResp -> + return + successResp |> Seq.zip entityBatch |> Seq.map(fun (entity, res) -> SuccessfulResponse(buildEntityId entity, res.HttpStatusCode)) - with :? StorageException as ex -> - let requestInformation = ex.RequestInformation - match requestInformation.ExtendedErrorInformation.ErrorMessage.Split('\n').[0].Split(':') with - | [|index;message|] -> - match Int32.TryParse(index) with - | true, index -> - entityBatch - |> Seq.mapi(fun entityIndex entity -> - if entityIndex = index then EntityError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode) - else BatchOperationFailedError(buildEntityId entity)) - | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) - | [|message|] -> entityBatch |> Seq.map(fun entity -> EntityError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) - | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestInformation.HttpStatusCode, requestInformation.ExtendedErrorInformation.ErrorCode)) - partitionKey, responses |> Seq.toArray) + | Choice2Of2 err -> + return + match err with + | :? StorageException as ex -> processErrorResp entityBatch buildEntityId ex + | _ -> raise (err) } + +let internal executeBatchSyncronously batchOp entityBatch buildEntityId (table:CloudTable) = + try + table.ExecuteBatch(batchOp) + |> Seq.zip entityBatch + |> Seq.map(fun (entity, res) -> SuccessfulResponse(buildEntityId entity, res.HttpStatusCode)) + with :? StorageException as ex -> processErrorResp entityBatch buildEntityId ex + +let internal executeBatchOperationAsync createTableOp (table:CloudTable) entities = async { + return! + splitIntoBatches createTableOp entities + |> Seq.map(fun (partitionKey, entityBatch, batchOperation) -> async{ + let buildEntityId (entity:DynamicTableEntity) = Partition(entity.PartitionKey), Row(entity.RowKey) + let! responses = executeBatchAsynchronously batchOperation entityBatch buildEntityId table + return (partitionKey, responses |> Seq.toArray) + }) + |> Async.Parallel } +let internal executeBatchOperation createTableOp (table:CloudTable) entities = + splitIntoBatches createTableOp entities + |> Seq.map(fun (partitionKey, entityBatch, batchOperation) -> + let buildEntityId (entity:DynamicTableEntity) = Partition(entity.PartitionKey), Row(entity.RowKey) + let responses = executeBatchSyncronously batchOperation entityBatch buildEntityId table + partitionKey, responses |> Seq.toArray) |> Seq.toArray let deleteEntities connection tableName entities = @@ -148,8 +206,27 @@ let deleteEntities connection tableName entities = |> Array.map buildDynamicTableEntity |> executeBatchOperation TableOperation.Delete table +let deleteEntitiesAsync connection tableName entities = async { + let table = getTable tableName connection + return! + entities + |> Array.map buildDynamicTableEntity + |> executeBatchOperationAsync TableOperation.Delete table } + let deleteEntity connection tableName entity = deleteEntities connection tableName [| entity |] |> Seq.head |> snd |> Seq.head + +let deleteEntityAsync connection tableName entity = async { + let! resp = deleteEntitiesAsync connection tableName [| entity |] + return resp |> Seq.head |> snd |> Seq.head } + +let insertEntityBatchAsync connection tableName insertMode entities = async { + let table = getTable tableName connection + let insertOp = createInsertOperation insertMode + return! + entities + |> Seq.map buildDynamicTableEntity + |> executeBatchOperationAsync insertOp table } let insertEntityBatch connection tableName insertMode entities = let table = getTable tableName connection @@ -159,7 +236,11 @@ let insertEntityBatch connection tableName insertMode entities = |> executeBatchOperation insertOp table let insertEntity connection tableName insertMode entity = - insertEntityBatch connection tableName insertMode [entity] |> Seq.head |> snd |> Seq.head + insertEntityBatch connection tableName insertMode [entity] |> Seq.head |> snd |> Seq.head + +let insertEntityAsync connection tableName insertMode entity = async { + let! resp = insertEntityBatchAsync connection tableName insertMode [entity] + return resp |> Seq.head |> snd |> Seq.head } let composeAllFilters filters = match filters with @@ -180,17 +261,33 @@ let buildFilter(propertyName, comparison, value) = | :? Guid as value -> TableQuery.GenerateFilterConditionForGuid(propertyName, comparison, value) | _ -> TableQuery.GenerateFilterCondition(propertyName, comparison, value.ToString()) -let getEntity rowKey partitionKey connection tableName = +let buildGetEntityQry rowKey partitionKey = let (Row rowKey, Partition partitionKey) = rowKey, partitionKey - let results = - [ ("RowKey", rowKey) - ("PartitionKey", partitionKey) ] - |> List.map(fun (prop, value) -> buildFilter(prop, QueryComparisons.Equal, value)) - |> composeAllFilters - |> executeQuery connection tableName 0 + [ ("RowKey", rowKey); ("PartitionKey", partitionKey) ] + |> List.map(fun (prop, value) -> buildFilter(prop, QueryComparisons.Equal, value)) + |> composeAllFilters + +let parseGetEntityResults results = match results with | [| exactMatch |] -> Some exactMatch | _ -> None +let getEntity rowKey partitionKey connection tableName = + buildGetEntityQry rowKey partitionKey + |> executeQuery connection tableName 0 + |> parseGetEntityResults + +let getEntityAsync rowKey partitionKey connection tableName = async { + let! results = + buildGetEntityQry rowKey partitionKey + |> executeQueryAsync connection tableName 0 + return results |> parseGetEntityResults } + let getPartitionRows (partitionKey:string) connection tableName = - buildFilter("PartitionKey", QueryComparisons.Equal, partitionKey) |> executeQuery connection tableName 0 \ No newline at end of file + buildFilter("PartitionKey", QueryComparisons.Equal, partitionKey) + |> executeQuery connection tableName 0 + +let getPartitionRowsAsync (partitionKey:string) connection tableName = async { + return! + buildFilter("PartitionKey", QueryComparisons.Equal, partitionKey) + |> executeQueryAsync connection tableName 0 } \ No newline at end of file diff --git a/src/FSharp.Azure.StorageTypeProvider/paket.references b/src/FSharp.Azure.StorageTypeProvider/paket.references index aabf0af..0fd0f36 100644 --- a/src/FSharp.Azure.StorageTypeProvider/paket.references +++ b/src/FSharp.Azure.StorageTypeProvider/paket.references @@ -1,5 +1,4 @@ WindowsAzure.Storage FSharp.Core File:ProvidedTypes.fsi -File:ProvidedTypes.fs -File:DebugProvidedTypes.fs \ No newline at end of file +File:ProvidedTypes.fs \ No newline at end of file diff --git a/tests/IntegrationTests/IntegrationTests.fsproj b/tests/IntegrationTests/IntegrationTests.fsproj index 561214b..a277f2f 100644 --- a/tests/IntegrationTests/IntegrationTests.fsproj +++ b/tests/IntegrationTests/IntegrationTests.fsproj @@ -74,6 +74,31 @@ + + + + <__paket__xunit_runner_visualstudio_props>net20\xunit.runner.visualstudio + + + + + <__paket__xunit_runner_visualstudio_props>portable-net45+win8+wp8+wpa81\xunit.runner.visualstudio + + + + + <__paket__xunit_runner_visualstudio_props>win81\xunit.runner.visualstudio + <__paket__xunit_runner_visualstudio_targets>win81\xunit.runner.visualstudio + + + + + <__paket__xunit_runner_visualstudio_props>wpa81\xunit.runner.visualstudio + <__paket__xunit_runner_visualstudio_targets>wpa81\xunit.runner.visualstudio + + + +