diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py new file mode 100644 index 000000000..70fc9ab19 --- /dev/null +++ b/synapseclient/models/__init__.py @@ -0,0 +1,6 @@ +# These are all of the models that are used by the Synapse client. +from .file import FileDataClass +from .folder import FolderDataClass +from .project import ProjectDataClass + +__all__ = ["FileDataClass", "FolderDataClass", "ProjectDataClass"] diff --git a/synapseclient/models/activity.py b/synapseclient/models/activity.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/activity.py @@ -0,0 +1 @@ +# TODO diff --git a/synapseclient/models/annotations.py b/synapseclient/models/annotations.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/annotations.py @@ -0,0 +1 @@ +# TODO diff --git a/synapseclient/models/evaluation.py b/synapseclient/models/evaluation.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/evaluation.py @@ -0,0 +1 @@ +# TODO diff --git a/synapseclient/models/file.py b/synapseclient/models/file.py new file mode 100644 index 000000000..7edfa6d0a --- /dev/null +++ b/synapseclient/models/file.py @@ -0,0 +1,84 @@ +import asyncio +from dataclasses import dataclass +from typing import Union +from opentelemetry import trace, context + +# import uuid + +from synapseclient.entity import File + +from typing import Optional, TYPE_CHECKING + + +# TODO - Is this an issue, is it needed? +if TYPE_CHECKING: + from synapseclient import Synapse + from .folder import FolderDataClass + from .project import ProjectDataClass + + +tracer = trace.get_tracer("synapseclient") + + +@dataclass() +class FileDataClass: + id: str + name: str + path: str + synapse: "Synapse" # TODO: How can we remove the need to pass this in??? + # TODO - Should a file also have a folder, or a method that figures out the folder class? + + description: Optional[str] = None + etag: Optional[str] = None + createdOn: Optional[str] = None + modifiedOn: Optional[str] = None + createdBy: Optional[str] = None + modifiedBy: Optional[str] = None + parentId: Optional[str] = None + concreteType: Optional[str] = None + versionNumber: Optional[int] = None + versionLabel: Optional[str] = None + versionComment: Optional[str] = None + isLatestVersion: Optional[bool] = False + dataFileHandleId: Optional[str] = None + fileNameOverride: Optional[str] = None + + isLoaded: bool = False + + # TODO: How the parent is stored/referenced needs to be thought through + async def store( + self, + parent: Union["FolderDataClass", "ProjectDataClass"], + ): + """Storing file to synapse.""" + with tracer.start_as_current_span(f"File_Store: {self.path}"): + # TODO - We need to add in some validation before the store to verify we have enough + # information to store the data + # print(f"Storing file {self.name}: {self.path}") + # await asyncio.sleep(1) + + # Call synapse + loop = asyncio.get_event_loop() + synapse_file = File(path=self.path, name=self.name, parent=parent.id) + # TODO: Propogating OTEL context is not working in this case + entity = await loop.run_in_executor( + None, + lambda: self.synapse.store( + obj=synapse_file, opentelemetry_context=context.get_current() + ), + ) + print(entity) + self.id = entity.id + + # TODO - This is temporary, we need to generate a real id + # self.id = uuid.uuid4() + + print(f"Stored file {self.name}, id: {self.id}: {self.path}") + + return self + + async def get(self): + """Get metadata about the folder from synapse.""" + print(f"Getting file {self.name}") + await asyncio.sleep(1) + self.isLoaded = True diff --git a/synapseclient/models/folder.py b/synapseclient/models/folder.py new file mode 100644 index 000000000..e15e3c843 --- /dev/null +++ b/synapseclient/models/folder.py @@ -0,0 +1,111 @@ +import asyncio +from dataclasses import dataclass, field +from typing import List, Union +from opentelemetry import trace, context + +# import uuid + +from synapseclient.entity import Folder + +from typing import Optional, TYPE_CHECKING + + +# TODO - Is this an issue, is it needed?? +if TYPE_CHECKING: + from synapseclient import Synapse + from .file import FileDataClass + from .project import ProjectDataClass + +MAX_CO_ROUTINES = 2 + +tracer = trace.get_tracer("synapseclient") + + +@dataclass() +class FolderDataClass: + id: str + name: str + parentId: str + synapse: "Synapse" # TODO: How can we remove the need to pass this in? + + description: Optional[str] = None + etag: Optional[str] = None + createdOn: Optional[str] = None + modifiedOn: Optional[str] = None + createdBy: Optional[str] = None + modifiedBy: Optional[str] = None + concreteType: Optional[str] = None # TODO - This is likely not needed + # Files that exist within this folder + files: Optional[List["FileDataClass"]] = field(default_factory=list) + # Folders that exist within this folder + folders: Optional[List["FolderDataClass"]] = field(default_factory=list) + isLoaded: bool = False + + # def __post_init__(self): + # # TODO - What is the best way to enforce this, basically we need a minimum amount + # # of information to be required such that we can save or load the data properly + # if not ((self.name is not None and self.parentId is not None) or self.id is not None): + # raise ValueError("Either name and parentId or id must be present") + + async def store( + self, + parent: Union["FolderDataClass", "ProjectDataClass"], + ): + """Storing folder and files to synapse.""" + with tracer.start_as_current_span(f"Folder_Store: {self.name}"): + # TODO - We need to add in some validation before the store to verify we have enough + # information to store the data + # print(f"Storing folder {self.name}") + # await asyncio.sleep(1) + + # Call synapse + loop = asyncio.get_event_loop() + synapse_folder = Folder(self.name, parent=parent.id) + # TODO: Propogating OTEL context is not working in this case + entity = await loop.run_in_executor( + None, + lambda: self.synapse.store( + obj=synapse_folder, opentelemetry_context=context.get_current() + ), + ) + print(entity) + self.id = entity.id + + # TODO - This is temporary, we need to generate a real id + # self.id = uuid.uuid4() + + print(f"Stored folder {self.name}, id: {self.id}") + + tasks = [] + if self.files: + tasks.extend(file.store(parent=self) for file in self.files) + + if self.folders: + tasks.extend(folder.store(parent=self) for folder in self.folders) + + try: + results = await asyncio.gather(*tasks, return_exceptions=True) + + # TODO: Proper exception handling + for result in results: + if isinstance(result, FolderDataClass): + print(f"Stored {result.name}") + elif isinstance(result, FileDataClass): + print(f"Stored {result.name} at: {result.path}") + else: + raise ValueError(f"Unknown type: {type(result)}") + except Exception as ex: + self.synapse.logger.error(ex) + print("I hit an exception") + + print(f"Saved all files and folders in {self.name}") + + return self + + async def get(self): + """Getting metadata about the folder from synapse.""" + # TODO - We will want to add some recursive logic to this for traversing child files/folders + print(f"Loading folder {self.name}") + await asyncio.sleep(1) + self.isLoaded = True + return self diff --git a/synapseclient/models/project.py b/synapseclient/models/project.py new file mode 100644 index 000000000..a3ca21666 --- /dev/null +++ b/synapseclient/models/project.py @@ -0,0 +1,102 @@ +import asyncio +from dataclasses import dataclass, field +from typing import List + +# import uuid + +from synapseclient.entity import Project +from opentelemetry import trace, context + +from typing import Optional, TYPE_CHECKING + +from .folder import FolderDataClass +from .file import FileDataClass + +# TODO - Is this an issue, is it needed?? +if TYPE_CHECKING: + from synapseclient import Synapse + +tracer = trace.get_tracer("synapseclient") + +MAX_CO_ROUTINES = 2 + + +@dataclass() +class ProjectDataClass: + id: str + name: str + parentId: str # TODO - Does a project have a parent? + synapse: "Synapse" # TODO: How can we remove the need to pass this in? + + description: Optional[str] = None + etag: Optional[str] = None + createdOn: Optional[str] = None + modifiedOn: Optional[str] = None + createdBy: Optional[str] = None + modifiedBy: Optional[str] = None + concreteType: Optional[str] = None + alias: Optional[str] = None + # Files at the root directory of the project + files: Optional[List["FileDataClass"]] = field(default_factory=list) + # Folder at the root directory of the project + folders: Optional[List["FolderDataClass"]] = field(default_factory=list) + isLoaded: bool = False + + # TODO: What if I don't handle queue size, but handle it in the HTTP REST API layer? + # TODO: https://www.python-httpx.org/advanced/#pool-limit-configuration + # TODO: Test out changing the underlying layer to httpx + + async def store(self): + """Storing project, files, and folders to synapse.""" + with tracer.start_as_current_span(f"Project_Store: {self.name}"): + # print(f"Storing project {self.name}") + # await asyncio.sleep(1) + + # Call synapse + loop = asyncio.get_event_loop() + synapse_project = Project(self.name) + # TODO: Propogating OTEL context is not working in this case + entity = await loop.run_in_executor( + None, + lambda: self.synapse.store( + obj=synapse_project, opentelemetry_context=context.get_current() + ), + ) + print(entity) + self.id = entity.id + + # # TODO - This is temporary, we need to generate a real id + # self.id = uuid.uuid4() + + tasks = [] + if self.files: + tasks.extend(file.store(parent=self) for file in self.files) + + if self.folders: + tasks.extend(folder.store(parent=self) for folder in self.folders) + + try: + results = await asyncio.gather(*tasks, return_exceptions=True) + # TODO: Proper exception handling + + for result in results: + if isinstance(result, FolderDataClass): + print(f"Stored {result.name}") + elif isinstance(result, FileDataClass): + print(f"Stored {result.name} at: {result.path}") + else: + raise ValueError(f"Unknown type: {type(result)}") + except Exception as ex: + self.synapse.logger.error(ex) + print("I hit an exception") + + print(f"Saved all files and folders in {self.name}") + + return self + + async def get(self): + """Getting metadata about the project from synapse.""" + print(f"Loading project {self.name}") + await asyncio.sleep(1) + self.isLoaded = True + return self diff --git a/synapseclient/models/table.py b/synapseclient/models/table.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/table.py @@ -0,0 +1 @@ +# TODO diff --git a/synapseclient/models/team.py b/synapseclient/models/team.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/team.py @@ -0,0 +1 @@ +# TODO diff --git a/synapseclient/models/wiki.py b/synapseclient/models/wiki.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/synapseclient/models/wiki.py @@ -0,0 +1 @@ +# TODO