Publish branch to remote

synapseclient/models/__init__.py

@@ -0,0 +1,6 @@
+# These are all of the models that are used by the Synapse client.
+from .file import FileDataClass
+from .folder import FolderDataClass
+from .project import ProjectDataClass
+
+__all__ = ["FileDataClass", "FolderDataClass", "ProjectDataClass"]

synapseclient/models/activity.py

@@ -0,0 +1 @@
+# TODO

synapseclient/models/annotations.py

@@ -0,0 +1 @@
+# TODO

synapseclient/models/evaluation.py

@@ -0,0 +1 @@
+# TODO

synapseclient/models/file.py

@@ -0,0 +1,84 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Union
+from opentelemetry import trace, context
+
+# import uuid
+
+from synapseclient.entity import File
+
+from typing import Optional, TYPE_CHECKING
+
+
+# TODO - Is this an issue, is it needed?
+if TYPE_CHECKING:
+    from synapseclient import Synapse
+    from .folder import FolderDataClass
+    from .project import ProjectDataClass
+
+
+tracer = trace.get_tracer("synapseclient")
+
+
+@dataclass()
+class FileDataClass:
+    id: str
+    name: str
+    path: str
+    synapse: "Synapse"  # TODO: How can we remove the need to pass this in???
+    # TODO - Should a file also have a folder, or a method that figures out the folder class?
+
+    description: Optional[str] = None
+    etag: Optional[str] = None
+    createdOn: Optional[str] = None
+    modifiedOn: Optional[str] = None
+    createdBy: Optional[str] = None
+    modifiedBy: Optional[str] = None
+    parentId: Optional[str] = None
+    concreteType: Optional[str] = None
+    versionNumber: Optional[int] = None
+    versionLabel: Optional[str] = None
+    versionComment: Optional[str] = None
+    isLatestVersion: Optional[bool] = False
+    dataFileHandleId: Optional[str] = None
+    fileNameOverride: Optional[str] = None
+
+    isLoaded: bool = False
+
+    # TODO: How the parent is stored/referenced needs to be thought through
+    async def store(
+        self,
+        parent: Union["FolderDataClass", "ProjectDataClass"],
+    ):
+        """Storing file to synapse."""
+        with tracer.start_as_current_span(f"File_Store: {self.path}"):
+            # TODO - We need to add in some validation before the store to verify we have enough
+            # information to store the data
+            # print(f"Storing file {self.name}: {self.path}")
+            # await asyncio.sleep(1)
+
+            # Call synapse
+            loop = asyncio.get_event_loop()
+            synapse_file = File(path=self.path, name=self.name, parent=parent.id)
+            # TODO: Propogating OTEL context is not working in this case
+            entity = await loop.run_in_executor(
+                None,
+                lambda: self.synapse.store(
+                    obj=synapse_file, opentelemetry_context=context.get_current()
+                ),
+            )
+            print(entity)
+            self.id = entity.id
+
+            # TODO - This is temporary, we need to generate a real id
+            # self.id = uuid.uuid4()
+
+            print(f"Stored file {self.name}, id: {self.id}: {self.path}")
+
+            return self
+
+    async def get(self):
+        """Get metadata about the folder from synapse."""
+        print(f"Getting file {self.name}")
+        await asyncio.sleep(1)
+        self.isLoaded = True

synapseclient/models/folder.py

@@ -0,0 +1,111 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import List, Union
+from opentelemetry import trace, context
+
+# import uuid
+
+from synapseclient.entity import Folder
+
+from typing import Optional, TYPE_CHECKING
+
+
+# TODO - Is this an issue, is it needed??
+if TYPE_CHECKING:
+    from synapseclient import Synapse
+    from .file import FileDataClass
+    from .project import ProjectDataClass
+
+MAX_CO_ROUTINES = 2
+
+tracer = trace.get_tracer("synapseclient")
+
+
+@dataclass()
+class FolderDataClass:
+    id: str
+    name: str
+    parentId: str
+    synapse: "Synapse"  # TODO: How can we remove the need to pass this in?
+
+    description: Optional[str] = None
+    etag: Optional[str] = None
+    createdOn: Optional[str] = None
+    modifiedOn: Optional[str] = None
+    createdBy: Optional[str] = None
+    modifiedBy: Optional[str] = None
+    concreteType: Optional[str] = None  # TODO - This is likely not needed
+    # Files that exist within this folder
+    files: Optional[List["FileDataClass"]] = field(default_factory=list)
+    # Folders that exist within this folder
+    folders: Optional[List["FolderDataClass"]] = field(default_factory=list)
+    isLoaded: bool = False
+
+    # def __post_init__(self):
+    #     # TODO - What is the best way to enforce this, basically we need a minimum amount
+    #     # of information to be required such that we can save or load the data properly
+    #     if not ((self.name is not None and self.parentId is not None) or self.id is not None):
+    #         raise ValueError("Either name and parentId or id must be present")
+
+    async def store(
+        self,
+        parent: Union["FolderDataClass", "ProjectDataClass"],
+    ):
+        """Storing folder and files to synapse."""
+        with tracer.start_as_current_span(f"Folder_Store: {self.name}"):
+            # TODO - We need to add in some validation before the store to verify we have enough
+            # information to store the data
+            # print(f"Storing folder {self.name}")
+            # await asyncio.sleep(1)
+
+            # Call synapse
+            loop = asyncio.get_event_loop()
+            synapse_folder = Folder(self.name, parent=parent.id)
+            # TODO: Propogating OTEL context is not working in this case
+            entity = await loop.run_in_executor(
+                None,
+                lambda: self.synapse.store(
+                    obj=synapse_folder, opentelemetry_context=context.get_current()
+                ),
+            )
+            print(entity)
+            self.id = entity.id
+
+            # TODO - This is temporary, we need to generate a real id
+            # self.id = uuid.uuid4()
+
+            print(f"Stored folder {self.name}, id: {self.id}")
+
+            tasks = []
+            if self.files:
+                tasks.extend(file.store(parent=self) for file in self.files)
+
+            if self.folders:
+                tasks.extend(folder.store(parent=self) for folder in self.folders)
+
+            try:
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+
+                # TODO: Proper exception handling
+                for result in results:
+                    if isinstance(result, FolderDataClass):
+                        print(f"Stored {result.name}")
+                    elif isinstance(result, FileDataClass):
+                        print(f"Stored {result.name} at: {result.path}")
+                    else:
+                        raise ValueError(f"Unknown type: {type(result)}")
+            except Exception as ex:
+                self.synapse.logger.error(ex)
+                print("I hit an exception")
+
+            print(f"Saved all files and folders in {self.name}")
+
+            return self
+
+    async def get(self):
+        """Getting metadata about the folder from synapse."""
+        # TODO - We will want to add some recursive logic to this for traversing child files/folders
+        print(f"Loading folder {self.name}")
+        await asyncio.sleep(1)
+        self.isLoaded = True
+        return self

synapseclient/models/project.py

@@ -0,0 +1,102 @@
+import asyncio
+from dataclasses import dataclass, field
+from typing import List
+
+# import uuid
+
+from synapseclient.entity import Project
+from opentelemetry import trace, context
+
+from typing import Optional, TYPE_CHECKING
+
+from .folder import FolderDataClass
+from .file import FileDataClass
+
+# TODO - Is this an issue, is it needed??
+if TYPE_CHECKING:
+    from synapseclient import Synapse
+
+tracer = trace.get_tracer("synapseclient")
+
+MAX_CO_ROUTINES = 2
+
+
+@dataclass()
+class ProjectDataClass:
+    id: str
+    name: str
+    parentId: str  # TODO - Does a project have a parent?
+    synapse: "Synapse"  # TODO: How can we remove the need to pass this in?
+
+    description: Optional[str] = None
+    etag: Optional[str] = None
+    createdOn: Optional[str] = None
+    modifiedOn: Optional[str] = None
+    createdBy: Optional[str] = None
+    modifiedBy: Optional[str] = None
+    concreteType: Optional[str] = None
+    alias: Optional[str] = None
+    # Files at the root directory of the project
+    files: Optional[List["FileDataClass"]] = field(default_factory=list)
+    # Folder at the root directory of the project
+    folders: Optional[List["FolderDataClass"]] = field(default_factory=list)
+    isLoaded: bool = False
+
+    # TODO: What if I don't handle queue size, but handle it in the HTTP REST API layer?
+    # TODO: https://www.python-httpx.org/advanced/#pool-limit-configuration
+    # TODO: Test out changing the underlying layer to httpx
+
+    async def store(self):
+        """Storing project, files, and folders to synapse."""
+        with tracer.start_as_current_span(f"Project_Store: {self.name}"):
+            # print(f"Storing project {self.name}")
+            # await asyncio.sleep(1)
+
+            # Call synapse
+            loop = asyncio.get_event_loop()
+            synapse_project = Project(self.name)
+            # TODO: Propogating OTEL context is not working in this case
+            entity = await loop.run_in_executor(
+                None,
+                lambda: self.synapse.store(
+                    obj=synapse_project, opentelemetry_context=context.get_current()
+                ),
+            )
+            print(entity)
+            self.id = entity.id
+
+            # # TODO - This is temporary, we need to generate a real id
+            # self.id = uuid.uuid4()
+
+            tasks = []
+            if self.files:
+                tasks.extend(file.store(parent=self) for file in self.files)
+
+            if self.folders:
+                tasks.extend(folder.store(parent=self) for folder in self.folders)
+
+            try:
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+                # TODO: Proper exception handling
+
+                for result in results:
+                    if isinstance(result, FolderDataClass):
+                        print(f"Stored {result.name}")
+                    elif isinstance(result, FileDataClass):
+                        print(f"Stored {result.name} at: {result.path}")
+                    else:
+                        raise ValueError(f"Unknown type: {type(result)}")
+            except Exception as ex:
+                self.synapse.logger.error(ex)
+                print("I hit an exception")
+
+            print(f"Saved all files and folders  in {self.name}")
+
+            return self
+
+    async def get(self):
+        """Getting metadata about the project from synapse."""
+        print(f"Loading project {self.name}")
+        await asyncio.sleep(1)
+        self.isLoaded = True
+        return self

synapseclient/models/table.py

@@ -0,0 +1 @@
+# TODO

synapseclient/models/team.py

@@ -0,0 +1 @@
+# TODO

synapseclient/models/wiki.py

@@ -0,0 +1 @@
+# TODO