Integrating torchsharp functionality into Bonsai

Bonsai.ML.sln

@@ -30,6 +30,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.LinearDynamicalSy
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.HiddenMarkovModels.Design", "src\Bonsai.ML.HiddenMarkovModels.Design\Bonsai.ML.HiddenMarkovModels.Design.csproj", "{FC395DDC-62A4-4E14-A198-272AB05B33C7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Torch", "src\Bonsai.ML.Torch\Bonsai.ML.Torch.csproj", "{06FCC9AF-CE38-44BB-92B3-0D451BE88537}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -72,6 +74,10 @@ Global
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
         {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.Build.0 = Release|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.Build.0 = Debug|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.ActiveCfg = Release|Any CPU
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -86,6 +92,7 @@ Global
 		{39A4414F-52B1-42D7-82FA-E65DAD885264} = {12312384-8828-4786-AE19-EFCEDF968290}
         {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13} = {12312384-8828-4786-AE19-EFCEDF968290}
         {17DF50BE-F481-4904-A4C8-5DF9725B2CA1} = {12312384-8828-4786-AE19-EFCEDF968290}
+        {06FCC9AF-CE38-44BB-92B3-0D451BE88537} = {12312384-8828-4786-AE19-EFCEDF968290}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {B6468F13-97CD-45E0-9E1E-C122D7F1E09F}

README.md

@@ -42,6 +42,9 @@ Facilitates inference using Hidden Markov Models (HMMs). It interfaces with the
 ### Bonsai.ML.HiddenMarkovModels.Design
 Visualizers and editor features for the HiddenMarkovModels package.
 
+### Bonsai.ML.Torch
+Interfaces with the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# wrapper around the torch library. Provides tooling for manipulating tensors, performing linear algebra, training and inference with deep neural networks, and more. 
+
 > [!NOTE]
 > Bonsai.ML packages can be installed through Bonsai's integrated package manager and are generally ready for immediate use. However, some packages may require additional installation steps. Refer to the specific package section for detailed installation guides and documentation.
 

docs/articles/Torch/torch-getting-started.md

@@ -0,0 +1,13 @@
+# Getting Started
+
+The aim of the `Bonsai.ML.Torch` package is to integrate the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# wrapper around the powerful libtorch library, into Bonsai. In the current version, the package primarily provides tooling and functionality for users to interact with and manipulate `Tensor`s, the core data type of libtorch which underlies many of the advanced torch operations. Additionally, the package provides some capabilities for defining neural network architectures, running forward inference, and learning via back propogation.
+
+## Tensor Operations
+The package provides several ways to work with tensors. Users can initialize tensors, (`Ones`, `Zeros`, etc.), create tensors from .NET data types, (`ToTensor`), and define custom tensors using Python-like syntax (`CreateTensor`). Tensors can be converted back to .NET types using the `ToArray` node (for flattening tensors into a single array) or the `ToNDArray` node (for preserving multidimensional array shapes). Furthermore, the `Tensor` data types contains many extension methods which can be used via scripting, such as using `ExpressionTransform` (for example, it.sum() to sum a tensor, or it.T to transpose), and works with overloaded operators, for example, `Zip` -> `Multiply`. Thus, `ExpressionTransform` can also be used to access individual elements of a tensor, using the syntax `it.data<T>.ReadCpuT(0)` where `T` is a primitive .NET data type.
+
+
+## Running on the GPU
+Users must be explicit about running tensors on the GPU. First, the `InitializeDeviceType` node must run with a CUDA-compatible GPU. Afterwards, tensors are moved to the GPU using the `ToDevice` node. Converting tensors back to .NET data types requires moving the tensor back to the CPU before converting.
+
+## Neural Networks
+The package provides initial support for working with torch `Module`s, the conventional object for deep neural networks. The `LoadModuleFromArchitecture` node allows users to select from a list of common architectures, and can optionally load in pretrained weights from disk. Additionally, the package supports loading `TorchScript` modules with the `LoadScriptModule` node, which enables users to use torch modules saved in the `.pt` file format. Users can then use the `Forward` node to run inference and the `Backward` node to run back propogation.   

docs/articles/Torch/torch-overview.md

@@ -0,0 +1,27 @@
+# Bonsai.ML.Torch Overview
+
+The Torch package provides a Bonsai interface to interact with the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# implementation of the torch library.
+
+## General Guide
+
+The Bonsai.ML.Torch package can be installed through the Bonsai Package Manager and depends on the TorchSharp library. Additionally, running the package requires installing the specific torch DLLs needed for your desired application. The steps for installing are outlined below.
+
+### Running on the CPU 
+For running the package using the CPU, the `TorchSharp-cpu` library can be installed though the `nuget` package source.
+
+### Running on the GPU
+To run torch on the GPU, you first need to ensure that you have a CUDA compatible device installed on your system. 
+
+Next, you must follow the [CUDA installation guide for Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html) or the [guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html). Make sure to install the correct `CUDA v12.1` version [found here](https://developer.nvidia.com/cuda-12-1-0-download-archive). Ensure that you have the correct CUDA version (v12.1) installed, as `TorchSharp` currently only supports this version.
+
+Next, you need to install the `cuDNN v9` library following the [guide for Windows](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/windows.html) or the [guide for Linux](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/linux.html). Again, you need to ensure you have the correct version installed (v9). You should consult [nvidia's support matrix](https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html) to ensure the versions of CUDA and cuDNN you installed are compatible with your specific OS, graphics driver, and hardware.
+
+Once complete, you need to install the cuda-compatible torch libraries and place them into the correct location. You can download the libraries from [the pytorch website](https://pytorch.org/get-started/locally/) with the following options selected:
+
+- PyTorch Build: Stable (2.5.1)
+- OS: [Your OS]
+- Package: LibTorch
+- Language: C++/Java
+- Compute Platform: CUDA 12.1
+
+Finally, extract the zip folder and copy the contents of the `lib` folder into the `Extensions` folder of your bonsai installation directory.

docs/articles/toc.yml

@@ -13,4 +13,9 @@
 - name: Overview
   href: HiddenMarkovModels/hmm-overview.md
 - name: Getting Started
-  href: HiddenMarkovModels/hmm-getting-started.md
+  href: HiddenMarkovModels/hmm-getting-started.md
+- name: Torch
+- name: Overview
+  href: Torch/torch-overview.md
+- name: Getting Started
+  href: Torch/torch-getting-started.md

src/Bonsai.ML.Torch/Arange.cs

@@ -0,0 +1,43 @@
+using System;
+using System.ComponentModel;
+using System.Reactive.Linq;
+using static TorchSharp.torch;
+using TorchSharp;
+
+namespace Bonsai.ML.Torch
+{
+    /// <summary>
+    /// Creates a 1-D tensor of values within a given range given the start, end, and step.
+    /// </summary>
+    [Combinator]
+    [Description("Creates a 1-D tensor of values within a given range given the start, end, and step.")]
+    [WorkflowElementCategory(ElementCategory.Source)]
+    public class Arange
+    {
+        /// <summary>
+        /// The start of the range.
+        /// </summary>
+        [Description("The start of the range.")]
+        public int Start { get; set; } = 0;
+
+        /// <summary>
+        /// The end of the range.
+        /// </summary>
+        [Description("The end of the range.")]
+        public int End { get; set; } = 10;
+
+        /// <summary>
+        /// The step size between values.
+        /// </summary>
+        [Description("The step size between values.")]
+        public int Step { get; set; } = 1;
+
+        /// <summary>
+        /// Generates an observable sequence of 1-D tensors created with the <see cref="arange(Scalar, Scalar, Scalar, ScalarType?, Device?, bool)"/> function.
+        /// </summary>
+        public IObservable<Tensor> Process()
+        {
+            return Observable.Defer(() => Observable.Return(arange(Start, End, Step)));
+        }
+    }
+}

src/Bonsai.ML.Torch/Bonsai.ML.Torch.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Title>Bonsai.ML.Torch</Title>
+    <Description>A Bonsai package for TorchSharp tensor manipulations.</Description>
+    <PackageTags>Bonsai Rx ML Tensors TorchSharp</PackageTags>
+    <TargetFrameworks>net472;netstandard2.0</TargetFrameworks>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="OpenCV.Net" Version="3.4.2" />
+    <PackageReference Include="TorchSharp" Version="0.105.0" />
+    <PackageReference Include="TorchVision" Version="0.105.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Bonsai.ML\Bonsai.ML.csproj" />
+    <ProjectReference Include="..\Bonsai.ML.Data\Bonsai.ML.Data.csproj" />
+    <ProjectReference Include="..\Bonsai.ML.Python\Bonsai.ML.Python.csproj" />
+  </ItemGroup>
+</Project>

src/Bonsai.ML.Torch/Clone.cs

@@ -0,0 +1,25 @@
+using System;
+using System.ComponentModel;
+using System.Reactive.Linq;
+using static TorchSharp.torch;
+
+namespace Bonsai.ML.Torch;
+
+/// <summary>
+/// Clones the input tensor.
+/// </summary>
+[Combinator]
+[Description("Clones the input tensor.")]
+[WorkflowElementCategory(ElementCategory.Transform)]
+public class Clone
+{
+    /// <summary>
+    /// Clones the input tensor.
+    /// </summary>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    public IObservable<Tensor> Process(IObservable<Tensor> source)
+    {
+        return source.Select(tensor => tensor.clone());
+    }
+}

src/Bonsai.ML.Torch/Concat.cs

@@ -0,0 +1,100 @@
+using System;
+using System.ComponentModel;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Collections.Generic;
+using static TorchSharp.torch;
+
+namespace Bonsai.ML.Torch
+{
+    /// <summary>
+    /// Concatenates tensors along a given dimension.
+    /// </summary>
+    [Combinator]
+    [Description("Concatenates tensors along a given dimension.")]
+    [WorkflowElementCategory(ElementCategory.Combinator)]
+    public class Concat
+    {
+        /// <summary>
+        /// The dimension along which to concatenate the tensors.
+        /// </summary>
+        [Description("The dimension along which to concatenate the tensors.")]
+        public long Dimension { get; set; } = 0;
+
+        /// <summary>
+        /// Takes any number of observable sequences of tensors and concatenates the input tensors along the specified dimension by zipping each tensor together.
+        /// </summary>
+        public IObservable<Tensor> Process(params IObservable<Tensor>[] sources)
+        {
+            return sources.Aggregate((current, next) =>
+                current.Zip(next, (tensor1, tensor2) =>
+                    cat([tensor1, tensor2], Dimension)));
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tuple<Tensor, Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat([value.Item1, value.Item2], Dimension);
+            });
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tuple<Tensor, Tensor, Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat([value.Item1, value.Item2, value.Item3], Dimension);
+            });
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tuple<Tensor, Tensor, Tensor, Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat([value.Item1, value.Item2, value.Item3, value.Item4], Dimension);
+            });
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tuple<Tensor, Tensor, Tensor, Tensor, Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat([value.Item1, value.Item2, value.Item3, value.Item4, value.Item5], Dimension);
+            });
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tuple<Tensor, Tensor, Tensor, Tensor, Tensor, Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat([value.Item1, value.Item2, value.Item3, value.Item4, value.Item5, value.Item6], Dimension);
+            });
+        }
+
+        /// <summary>
+        /// Concatenates the input tensors along the specified dimension.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<IEnumerable<Tensor>> source)
+        {
+            return source.Select(value =>
+            {
+                return cat(value.ToList(), Dimension);
+            });
+        }
+    }
+}

src/Bonsai.ML.Torch/ConvertDataType.cs

@@ -0,0 +1,33 @@
+using System;
+using System.ComponentModel;
+using System.Reactive.Linq;
+using static TorchSharp.torch;
+
+namespace Bonsai.ML.Torch
+{
+    /// <summary>
+    /// Converts the input tensor to the specified scalar type.
+    /// </summary>
+    [Combinator]
+    [Description("Converts the input tensor to the specified scalar type.")]
+    [WorkflowElementCategory(ElementCategory.Transform)]
+    public class ConvertDataType
+    {
+        /// <summary>
+        /// The scalar type to which to convert the input tensor.
+        /// </summary>
+        [Description("The scalar type to which to convert the input tensor.")]
+        public ScalarType Type { get; set; } = ScalarType.Float32;
+
+        /// <summary>
+        /// Returns an observable sequence that converts the input tensor to the specified scalar type.
+        /// </summary>
+        public IObservable<Tensor> Process(IObservable<Tensor> source)
+        {
+            return source.Select(tensor =>
+            {
+                return tensor.to_type(Type);
+            });
+        }
+    }
+}