Skip to content

Commit

Permalink
2.5.0
Browse files Browse the repository at this point in the history
  • Loading branch information
tdayris committed Dec 13, 2024
1 parent 87843af commit e481d8c
Show file tree
Hide file tree
Showing 9 changed files with 233 additions and 107 deletions.
238 changes: 149 additions & 89 deletions .test/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,7 @@
import os
import yaml


def get_latest_release(address: str) -> str:
git = Github()
repo = git.get_repo(address)
releases = repo.get_releases()
return releases[0].tag_name


snakemake_wrappers_version = os.environ.get("SNAKEMAKE_WRAPPERS_VERSION")
if not snakemake_wrappers_version:
print("Searching snakemake wrappers version on the web...")
snakemake_wrappers_version = get_latest_release("snakemake/snakemake-wrappers")

fair_genome_indexer_version = os.environ.get("FAIR_GENOME_INDEXER_VERSION")
if not fair_genome_indexer_version:
print("Searching fair-genome-indexer version on the web...")
fair_genome_indexer_version = get_latest_release("tdayris/fair_genome_indexer")

fair_fastqc_multiqc_version = os.environ.get("FAIR_FASTQC_MULTIQC_VERSION")
if not fair_fastqc_multiqc_version:
print("Searching fair-fastqc-multiqc version on the web...")
fair_fastqc_multiqc_version = get_latest_release("tdayris/fair_fastqc_multiqc")

fair_bowtie2_mapping_version = os.environ.get("FAIR_BOWTIE2_MAPPING_VERSION")
if not fair_bowtie2_mapping_version:
print("Searching fair-bowtie2-mapping version on the web...")
fair_bowtie2_mapping_version = get_latest_release("tdayris/fair_bowtie2_mapping")

fair_star_mapping_version = os.environ.get("FAIR_STAR_MAPPING_VERSION")
if not fair_star_mapping_version:
print("Searching fair-star-mapping version on the web...")
fair_star_mapping_version = get_latest_release("tdayris/fair_star_mapping")
tag_re = r"v\?[0-9]\+\.[0-9]\+\.[0-9]\+"

locations = {
"snakefile": "../workflow/Snakefile",
Expand All @@ -49,10 +18,13 @@ def get_latest_release(address: str) -> str:
"changelog": "../CHANGELOG.md",
"cff": "../CITATION.cff",
}
for location in locations.values():
for name, location in locations.items():
if not op.exists(location):
raise FileNotFoundError(f"Could not find {location=}")

else:
locations[name] = op.realpath(location)


def get_future_version(
changelog: str = locations["changelog"],
Expand All @@ -68,6 +40,56 @@ def get_future_version(
future_version = get_future_version()


def get_latest_release(
address: str,
future: str = future_version,
known: str | None = None,
) -> str:
if address.split("/")[-1] == op.basename(op.dirname(os.getcwd())):
print(f"Current pipeline detected, using {future=}...")
return get_future_version(changelog=locations["changelog"])

if known is not None:
print(f"Known version provided {known=}")
return known

print(f"Seaching for {address=} on github...")
git = Github()
repo = git.get_repo(address)
releases = repo.get_releases()
return releases[0].tag_name

snakemake_wrappers_version = get_latest_release(
address="snakemake/snakemake-wrappers",
future=future_version,
known=os.environ.get("SNAKEMAKE_WRAPPERS_VERSION"),
)

fair_genome_indexer_version = get_latest_release(
"tdayris/fair_genome_indexer",
future=future_version,
known=os.environ.get("FAIR_GENOME_INDEXER_VERSION"),
)

fair_fastqc_multiqc_version = get_latest_release(
"tdayris/fair_fastqc_multiqc",
future=future_version,
known=os.environ.get("FAIR_FASTQC_MULTIQC_VERSION"),
)

fair_bowtie2_mapping_version = get_latest_release(
"tdayris/fair_bowtie2_mapping",
future=future_version,
known=os.environ.get("FAIR_BOWTIE2_MAPPING_VERSION"),
)

fair_star_mapping_version = get_latest_release(
"tdayris/fair_star_mapping",
future=future_version,
known=os.environ.get("FAIR_STAR_MAPPING_VERSION"),
)


@task
def clean(
c,
Expand Down Expand Up @@ -96,6 +118,7 @@ def clean(
"Snakefile",
"wrappers_update.txt",
"conda_update.txt",
"update_docs.txt",
"docs_update.txt",
]
if conda:
Expand All @@ -104,10 +127,10 @@ def clean(

for pattern in patterns:
if op.exists(pattern):
print(f"Removing {pattern=}")
c.run(f"rm -rf '{pattern}'")
else:
print(f"Skipping {pattern=}")
c.run(
f"rm --force --recursive --verbose '{pattern}'",
echo=True,
)


@task
Expand All @@ -116,7 +139,8 @@ def update_docs_cff(
to: str = future_version,
cff_path: str = locations["cff"],
):
today = datetime.today().strftime("%Y-%m-%d")
today: str = datetime.today().strftime("%Y-%m-%d")
name: str = op.basename(op.dirname(os.getcwd()))
cff = {
"cff-version": "1.2.0",
"message": "If you use this software, please cite it as below.",
Expand All @@ -127,55 +151,75 @@ def update_docs_cff(
"orcid": "https://orcid.org/0009-0009-2758-8450",
}
],
"title": "fair-star-mapping",
"title": name.replace("_", "-"),
"version": future_version,
"date-released": today,
"url": "https://github.com/tdayris/fair_star_mapping",
"url": f"https://github.com/tdayris/{name}",
}
print(cff)
with open(cff_path, "w") as yaml_cff_stream:
yaml.dump(cff, yaml_cff_stream, default_flow_style=False)


@task(update_docs_cff)
def update_docs_wrappers(c, to: str = snakemake_wrappers_version, future_version: str = future_version):
def update_docs_wrappers(
c,
to: str = snakemake_wrappers_version,
future_version: str = future_version,
tag_re: str = tag_re,
):
today = datetime.today().strftime("%Y-%m-%d")
regex = rf"s|v[0-9]\+\.[0-9]\+\.[0-9]\+/wrappers|{to}/wrappers|g"
regex = rf"s|{tag_re}/wrappers|{to}/wrappers|g"
print(f"Updating snakemake wrappers in README.md to {to=}")
c.run(f"sed -i '{regex}' ../README.md >> docs_update.txt 2>&1")
c.run(
f"sed -i '{regex}' ../README.md >> docs_update.txt 2>&1",
echo=True,
)

for root, dirs, files in os.walk("../workflow/report"):
for file in files:
if file.endswith(".rst"):
print(f"Updating snakemake wrappers in '{root}/{file}'...")
c.run(f"sed -i '{regex}' '{root}/{file}' >> docs_update.txt 2>&1")
c.run(
f"sed -i '{regex}' '{root}/{file}' >> docs_update.txt 2>&1",
echo=True,
)

regex = (
's|snakemake_wrappers_prefix: str = "v4.5.0"|'
f'snakemake_wrappers_prefix: str = "{to}"|g'
)
print("Updating '../workflow/rules/common.smk'...")
c.run(f"sed -i '{regex}' '../workflow/rules/common.smk' >> update_docs.txt 2>&1")
c.run(
f"sed -i '{regex}' '../workflow/rules/common.smk' >> update_docs.txt 2>&1",
echo=True,
)

regex = (
r's|fair-genome-indexer (Version v\?[0-9]\+\.[0-9]\+\.[0-9]\+)'
f'|fair-genome-indexer (Version {fair_genome_indexer_version})|g;'
r's|fair-fastqc-multiqc (Version v\?[0-9]\+\.[0-9]\+\.[0-9]\+)'
f'|fair-fastqc-multiqc (Version {fair_fastqc_multiqc_version})|g;'
r's|fair-bowtie2-mapping (Version v\?[0-9]\+\.[0-9]\+\.[0-9]\+)'
f'|fair-bowtie2-mapping (Version {fair_bowtie2_mapping_version})|g;'
r's|fair-star-mapping (Version v\?[0-9]\+\.[0-9]\+\.[0-9]\+)'
f'|fair-star-mapping (Version {future_version})|g'
r"s|fair-genome-indexer (Version {tag_re})"
f"|fair-genome-indexer (Version {fair_genome_indexer_version})|g;"
r"s|fair-fastqc-multiqc (Version {tag_re})"
f"|fair-fastqc-multiqc (Version {fair_fastqc_multiqc_version})|g;"
r"s|fair-bowtie2-mapping (Version {tag_re})"
f"|fair-bowtie2-mapping (Version {fair_bowtie2_mapping_version})|g;"
r"s|fair-star-mapping (Version {tag_re})"
f"|fair-star-mapping (Version {fair_star_mapping_version})|g"
)
print("Updating '../workflow/report/material_methods.rst'...")
c.run(f"sed -i '{regex}' '../workflow/report/material_methods.rst' >> update_docs.txt 2>&1")
c.run(
f"sed -i '{regex}' '../workflow/report/material_methods.rst' >> update_docs.txt 2>&1",
echo=True,
)

regex = (
r's|\:Version\: [0-9]\+\.[0-9]\+\.[0-9]\+ of [0-9]\+\-[0-9]\+\-[0-9]\+$'
fr'|\:Version\: {future_version} of {today}|g'
rf"s|\:Version\: {tag_re} of [0-9]\+\-[0-9]\+\-[0-9]\+$"
rf"|\:Version\: {future_version} of {today}|g"
)
c.run(
f"sed -i '{regex}' '../workflow/report/material_methods.rst' "
">> update_docs.txt 2>&1",
echo=True,
)
c.run(f"sed -i '{regex}' '../workflow/report/material_methods.rst' >> update_docs.txt 2>&1")



@task(update_docs_wrappers)
Expand All @@ -184,37 +228,45 @@ def update_wrappers_rules(
to: str = snakemake_wrappers_version,
snakefile: str = locations["snakefile"],
rules: str = locations["rules"],
tag_re: str = tag_re,
):
print(f"Updating {snakefile=}...")
c.run(
"snakedeploy update-snakemake-wrappers "
f"--git-ref '{snakemake_wrappers_version}' '{snakefile}' "
">> wrappers_update.txt 2>&1"
">> wrappers_update.txt 2>&1",
echo=True,
)

for root, dirs, files in os.walk(rules):
for file in files:
if file == "fair_genome_indexer.smk":
print("Updating fair_genome_indexer...")
c.run(fr"""sed -i 's|tag="[0-9]\+\.[0-9]\+\.[0-9]\+"|tag="{fair_genome_indexer_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""")
c.run(
rf"""sed -i 's|tag="{tag_re}"|tag="{fair_genome_indexer_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""",
echo=True,
)
elif file == "fair_fastqc_multiqc.smk":
print("Updating fair_fastq_multiqc...")
c.run(fr"""sed -i 's|tag="[0-9]\+\.[0-9]\+\.[0-9]\+"|tag="{fair_fastqc_multiqc_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""")
c.run(
rf"""sed -i 's|tag="{tag_re}"|tag="{fair_fastqc_multiqc_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""",
echo=True,
)
elif file == "fair_bowtie2_mapping.smk":
print("Updating fair_bowtie2_mapping...")
c.run(fr"""sed -i 's|tag="[0-9]\+\.[0-9]\+\.[0-9]\+"|tag="{fair_bowtie2_mapping_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""")
c.run(
rf"""sed -i 's|tag="{tag_re}"|tag="{fair_bowtie2_mapping_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""",
echo=True,
)
elif file == "fair_star_mapping.smk":
print("Updating fair_star_mapping...")
c.run(fr"""sed -i 's|tag="[0-9]\+\.[0-9]\+\.[0-9]\+"|tag="{fair_star_mapping_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""")
c.run(
rf"""sed -i 's|tag="{tag_re}"|tag="{fair_star_mapping_version}"|g' '{root}/{file}' >> wrappers_update.txt 2>&1""",
echo=True,
)
elif file.endswith(".smk"):
print(f"Updating '{root}/{file}'...")
c.run(
"snakedeploy update-snakemake-wrappers --git-ref "
f"'{snakemake_wrappers_version}' '{root}/{file}' "
">> wrappers_update.txt 2>&1"
">> wrappers_update.txt 2>&1",
echo=True,
)
else:
print(f"Skipping '{root}/{file}'...")


@task(update_wrappers_rules)
Expand All @@ -225,10 +277,10 @@ def update_conda(
for root, dirs, files in os.walk(envs):
for file in files:
if file.endswith(".yaml"):
print(f"Updating '{root}/{file}'. This may take some time...")
c.run(
"snakedeploy update-conda-envs --conda-frontend mamba "
f"--pin-envs '{root}/{file}' > conda_update.txt 2>&1"
f"--pin-envs '{root}/{file}' > conda_update.txt 2>&1",
echo=True,
)


Expand All @@ -238,8 +290,10 @@ def black(c, scripts: str = locations["scripts"]):
for root, dirs, files in os.walk(scripts):
for file in files:
if file.endswith(".py"):
print(f"Formatting '{root}/{file}'...")
c.run(f"black '{root}/{file}' >> {log} 2>&1")
c.run(
f"black '{root}/{file}' >> {log} 2>&1",
echo=True,
)


@task(black)
Expand All @@ -249,52 +303,58 @@ def snakefmt(
rules: str = locations["rules"],
):
log: str = "format.txt"
print(f"Formatting {snakefile=}...")
c.run(f"snakefmt '{snakefile}' >> '{log}' 2>&1")
c.run(
f"snakefmt '{snakefile}' >> '{log}' 2>&1",
echo=True,
)

for root, dirs, files in os.walk(rules):
for file in files:
if file.endswith(".smk"):
print(f"Formatting '{root}/{file}'...")
c.run(f"snakefmt '{root}/{file}' >> '{log}' 2>&1")
c.run(
f"snakefmt '{root}/{file}' >> '{log}' 2>&1",
echo=True,
)


@task(snakefmt)
def linter(
c,
snakefile: str = locations["snakefile"],
):
print(f"Linting {snakefile=}...")
c.run(f"snakemake --lint -s '{snakefile}' > linter_info.txt 2>&1")
c.run(
f"snakemake --lint -s '{snakefile}' > linter_info.txt 2>&1",
echo=True,
)


@task(linter)
def pipeline(
c,
snakefile: str = locations["snakefile"],
):
print("Running pipeline...")
cmd = (
c.run(
f"snakemake -s '{snakefile}' "
"--cores 7 --restart-times 0 "
"--rerun-incomplete --printshellcmds "
"--shadow-prefix 'tmp' --rerun-triggers 'mtime' "
"--software-deployment-method conda "
"--benchmark-extended > pipeline.txt 2>&1"
"--benchmark-extended > pipeline.txt 2>&1",
echo=True,
)
print(cmd)
c.run(cmd)


@task(pipeline)
def report(
c,
snakefile: str = locations["snakefile"],
):
print("Building test report...")
c.run(f"snakemake -s '{snakefile}' --report report.zip > report.txt 2>&1")
c.run(
f"snakemake -s '{snakefile}' --report report.zip > report.txt 2>&1",
echo=True,
)


@task(clean, update_conda, report)
def all(c):
print("All done.")

Loading

0 comments on commit e481d8c

Please sign in to comment.