fixed alignment and blast search (#80)

* fixed alignment and blast search * API update --------- Co-authored-by: sdRDM Bot <[email protected]>
PyEED · May 25, 2024 · 42838d1 · 42838d1
1 parent 3ee56e9
commit 42838d1
Show file tree

Hide file tree

Showing 17 changed files with 50 additions and 40 deletions.
diff --git a/pyeed/core/abstractannotation.py b/pyeed/core/abstractannotation.py
@@ -49,7 +49,7 @@ class AbstractAnnotation(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/alignmentresult.py b/pyeed/core/alignmentresult.py
@@ -53,7 +53,7 @@ class AlignmentResult(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/blastdata.py b/pyeed/core/blastdata.py
@@ -90,7 +90,7 @@ class BlastData(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/clustalomegaresult.py b/pyeed/core/clustalomegaresult.py
@@ -32,7 +32,7 @@ class ClustalOmegaResult(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/dnarecord.py b/pyeed/core/dnarecord.py
@@ -32,7 +32,7 @@ class DNARecord(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/organism.py b/pyeed/core/organism.py
@@ -108,7 +108,7 @@ class Organism(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/pairwisealignmentresult.py b/pyeed/core/pairwisealignmentresult.py
@@ -60,7 +60,7 @@ class PairwiseAlignmentResult(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/proteinrecord.py b/pyeed/core/proteinrecord.py
@@ -10,7 +10,8 @@
 from lxml.etree import _Element
 from pydantic import PrivateAttr, model_validator
 from pydantic_xml import attr, element
-from rich.status import Console, Status
+from rich.console import Console
+from rich.status import Status
 from sdRDM.base.listplus import ListPlus
 from sdRDM.tools.utils import elem2dict
 
@@ -74,7 +75,7 @@ class ProteinRecord(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)
@@ -144,7 +145,6 @@ def get_id(cls, protein_id: str) -> "ProteinRecord":
 
         import nest_asyncio
 
-        from pyeed.fetch.proteinfetcher import ProteinFetcher
 
         nest_asyncio.apply()
 
@@ -166,7 +166,6 @@ def get_ids(cls, accession_ids: List[str]) -> List["ProteinRecord"]:
 
         import nest_asyncio
 
-        from pyeed.fetch.proteinfetcher import ProteinFetcher
 
         nest_asyncio.apply()
 
@@ -203,7 +202,6 @@ def from_sequence(
         """
 
         from pyeed.fetch.blast import BlastProgram
-        from pyeed.fetch.proteinfetcher import ProteinFetcher
 
         nest_asyncio.apply()
 
@@ -244,7 +242,7 @@ def ncbi_blast(
         self,
         n_hits: int,
         e_value: float = 10.0,
-        database: str = "nr",
+        db: str = "swissprot",
         matrix: str = "BLOSUM62",
         identity: float = 0.0,
         **kwargs,
@@ -255,7 +253,7 @@ def ncbi_blast(
         Args:
             n_hits (int): The number of hits to retrieve.
             e_value (float, optional): The maximum E-value threshold for reporting hits. Defaults to 10.0.
-            database (str, optional): The database to search against. Defaults to "nr".
+            db (str, optional): The database to search against. Defaults to "swissprot".
             matrix (str, optional): The substitution matrix to use. Defaults to "BLOSUM62".
             identity (float, optional): The minimum sequence identity threshold for reporting hits. Defaults to 0.0.
             **kwargs: Additional keyword arguments.
@@ -277,7 +275,9 @@ def ncbi_blast(
 
         nest_asyncio.apply()
 
-        assert database in NCBIDataBase
+        assert (
+            db in NCBIDataBase
+        ), f"Database needs to be one of {NCBIDataBase.__members__.keys()}"
 
         program = BlastProgram.BLASTP.value
         executor = ThreadPoolExecutor(max_workers=1)
@@ -292,12 +292,12 @@ def ncbi_blast(
         with Status(
             "Running BLAST", console=Console(force_terminal=False, force_jupyter=True)
         ):
-            result = asyncio.run(blaster.async_run(database, program, executor))
+            result = asyncio.run(blaster.async_run(db, program, executor))
             clear_output()
 
         accessions = blaster.extract_accession(result)
 
-        return asyncio.run(ProteinFetcher(ids=accessions).fetch(force_terminal=False))
+        return self.get_ids(accessions)
 
     # def blastp(
     #     self,

diff --git a/pyeed/core/region.py b/pyeed/core/region.py
@@ -43,7 +43,7 @@ class Region(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/regionset.py b/pyeed/core/regionset.py
@@ -35,7 +35,7 @@ class RegionSet(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _object_terms: Set[str] = PrivateAttr(

diff --git a/pyeed/core/sequence.py b/pyeed/core/sequence.py
@@ -38,7 +38,7 @@ class Sequence(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/sequencerecord.py b/pyeed/core/sequencerecord.py
@@ -103,7 +103,7 @@ class SequenceRecord(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/site.py b/pyeed/core/site.py
@@ -35,7 +35,7 @@ class Site(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/core/standardnumbering.py b/pyeed/core/standardnumbering.py
@@ -47,7 +47,7 @@ class StandardNumbering(
 
     _repo: Optional[str] = PrivateAttr(default="https://github.com/PyEED/pyeed")
     _commit: Optional[str] = PrivateAttr(
-        default="ff1fb2064e9efbdf71ccde1d8f08b9af434150bb"
+        default="63f43b11e0d359e1d0a1f541cea25dd484ad0072"
     )
 
     _raw_xml_data: Dict = PrivateAttr(default_factory=dict)

diff --git a/pyeed/fetch/blast.py b/pyeed/fetch/blast.py
@@ -3,7 +3,7 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor
 from enum import Enum, EnumMeta
-from typing import List
+from typing import List, Optional
 
 from Bio.Blast import NCBIWWW, NCBIXML
 from Bio.Blast.Record import Blast as BlastRecord
@@ -96,11 +96,9 @@ def run(self, program: str, ncbi_db: str) -> io.StringIO:
     async def async_run(
         self,
         ncbi_db: str,
-        program: str = BlastProgram.BLASTP.value,
-        foreign_executor: ThreadPoolExecutor = None,
+        program: str,
+        foreign_executor: Optional[ThreadPoolExecutor] = None,
     ) -> io.StringIO:
-        assert program in BlastProgram
-        assert ncbi_db in NCBIDataBase
 
         if not foreign_executor:
             executor = ThreadPoolExecutor()

diff --git a/pyeed/fetch/ncbiproteinmapper.py b/pyeed/fetch/ncbiproteinmapper.py
@@ -5,8 +5,10 @@
 import re
 from typing import TYPE_CHECKING, List
 
-from Bio import SeqFeature, SeqIO
+from Bio import SeqIO
+from Bio.SeqFeature import SeqFeature
 from Bio.SeqRecord import SeqRecord
+from pydantic import ValidationError
 
 from pyeed.core.annotation import Annotation
 from pyeed.core.dnarecord import DNARecord
@@ -44,9 +46,16 @@ def map(self, responses: List[str]) -> List[ProteinRecord]:
 
         protein_infos = []
         for record in seq_records:
+
             protein_info = ProteinRecord(id=record.id, sequence=str(record.seq))
 
-            protein_info.organism = Organism(**self.map_organism(record))
+            try:
+                protein_info.organism = Organism(**self.map_organism(record))
+            except ValidationError as e:
+                LOGGER.error(
+                    f"Error mapping organism for {record.id}: {e.errors()} {e.json()}"
+                )
+                continue
 
             protein_info = self.map_protein(record, protein_info)
 
@@ -67,7 +76,7 @@ def map_organism(self, seq_record: SeqRecord) -> dict:
         """
 
         feature = self.get_feature(seq_record, "source")
-        if len(feature) != 1:
+        if len(feature) < 1:
             LOGGER.debug(
                 f"Multiple features ({len(feature)}) of type `source` found for {seq_record.id}: {feature}"
             )
@@ -78,26 +87,29 @@ def map_organism(self, seq_record: SeqRecord) -> dict:
                 LOGGER.info(
                     f"For {seq_record.id} {feature.qualifiers['db_xref']} taxonomy ID(s) were found, using the first one. Skipping organism assignment"
                 )
-                return None
+                return {}
 
-            taxonomy_id = feature.qualifiers["db_xref"][0]
+            try:
+                taxonomy_id = next(feature for feature in feature.qualifiers["db_xref"] if "taxon" in feature)
+                if ":" in taxonomy_id:
+                    taxonomy_id = taxonomy_id.split(":")[1]
+            except StopIteration:
+                taxonomy_id = None
 
-            if ":" in taxonomy_id:
-                taxonomy_id = int(taxonomy_id.split(":")[1])
 
         except KeyError:
             LOGGER.debug(f"No taxonomy ID found for {seq_record.id}: {feature}")
-            return None
+            return {}
 
         try:
             organism_name = feature.qualifiers["organism"]
         except KeyError:
             LOGGER.debug(
                 f"No organism name found for {seq_record.id}: {feature[0].qualifiers}"
             )
-            organism_name = None
+            organism_name = ""
 
-        return {"name": organism_name[0], "taxonomy_id": taxonomy_id}
+        return {"id": taxonomy_id, "name": organism_name[0], "taxonomy_id": taxonomy_id}
 
     def map_protein(self, seq_record: SeqRecord, protein_info: ProteinRecord):
         """Maps protein data from a `Bio.SeqRecord` to a `ProteinInfo` object."""
@@ -259,7 +271,7 @@ def get_cds_regions(coded_by: dict) -> List[DNARecord]:
 
         return regions
 
-    def get_feature(self, seq_record: SeqRecord, feature_type: str) -> SeqFeature:
+    def get_feature(self, seq_record: SeqRecord, feature_type: str) -> List[SeqFeature]:
         """Returns a list of features of a given type from a `Bio.SeqRecord` object."""
         return [
             feature

diff --git a/pyeed/fetch/requester.py b/pyeed/fetch/requester.py
@@ -66,7 +66,7 @@ async def send_request(self, args: RequestArgs) -> str:
         url = args.url
 
         LOGGER.debug(f"Sending request to {url}")
-        response = await client.get(url, timeout=30)
+        response = await client.get(url, timeout=120)
 
         LOGGER.debug(f"Received response from {url}. Code: {response.status_code}")