diff --git a/pretext/cli.py b/pretext/cli.py index d0bd9492..e13d9dbc 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -599,7 +599,7 @@ def build( try: for t in targets: log.info(f"Generating assets for {t.name}") - t.generate_assets(only_changed=False, xmlid=xmlid) + t.generate_assets(only_changed=False, xmlid=xmlid, clean=clean) no_generate = True except Exception as e: log.error(f"Failed to generate assets: {e} \n") @@ -672,6 +672,18 @@ def build( default=False, help="Generate all possible asset formats rather than just the defaults for the specified target.", ) +@click.option( + "--clean", + is_flag=True, + default=False, + help="Remove all generated assets, including the cache, before generating new ones.", +) +@click.option( + "-f", + "--force", + is_flag=True, + help="Force generation of assets; do not rely on assets in the cache.", +) @click.pass_context @nice_errors def generate( @@ -681,6 +693,8 @@ def generate( all_formats: bool, only_changed: bool, xmlid: Optional[str], + clean: bool, + force: bool, ) -> None: """ Generate specified (or all) assets for the default target (first target in "project.ptx"). Asset "generation" is typically @@ -717,6 +731,8 @@ def generate( all_formats=all_formats, only_changed=only_changed, # Unless requested, generate all assets, so don't check the cache. xmlid=xmlid, + clean=clean, + skip_cache=force, ) log.info("Finished generating assets.\n") except ValidationError as e: diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 62523efe..5954694a 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -551,6 +551,25 @@ def clean_output(self) -> None: ) shutil.rmtree(self.output_dir_abspath()) + def clean_assets(self) -> None: + for asset_dir in [self.generated_dir_abspath(), self.generated_cache_abspath()]: + # refuse to clean if generated is not a subdirectory of the project + if self._project.abspath() not in asset_dir.parents: + log.warning( + "Refusing to clean generated directory that isn't a proper subdirectory of the project." + ) + # handle request to clean directory that does not exist + elif not asset_dir.exists(): + log.warning( + f"Directory {asset_dir} already does not exist, nothing to clean." + ) + # destroy the generated directory + else: + log.warning( + f"Destroying directory {asset_dir} to clean previously built assets." + ) + shutil.rmtree(asset_dir) + def build_theme(self) -> None: """ Builds or copies the theme for an HTML-formatted target. @@ -792,6 +811,8 @@ def generate_assets( all_formats: bool = False, only_changed: bool = True, xmlid: t.Optional[str] = None, + clean: bool = False, + skip_cache: bool = False, ) -> None: """ Generates assets for the current target. Options: @@ -801,6 +822,11 @@ def generate_assets( - xmlid: optional string to specify the root of the subtree of the xml document to generate assets within. """ log.info("Generating any needed assets.") + + # clear out the generated assets and cache if requested + if clean: + self.clean_assets() + # Ensure that the generated_cache directory exists: if not self.generated_cache_abspath().exists(): self.generated_cache_abspath().mkdir(parents=True, exist_ok=True) @@ -924,8 +950,9 @@ def generate_assets( ext_converter=partial( generate.individual_latex_image, cache_dir=self.generated_cache_abspath(), + skip_cache=skip_cache, ), - # Note: partial(...) is from functools and allows us to pass the extra argument cache_dir and still pass the resulting function object to core's conversion function. + # Note: partial(...) is from functools and allows us to pass the extra arguments cache_dir and skip_cache and still pass the resulting function object to core's conversion function. ) successful_assets.append("latex-image") except Exception as e: @@ -945,6 +972,7 @@ def generate_assets( ext_converter=partial( generate.individual_asymptote, cache_dir=self.generated_cache_abspath(), + skip_cache=skip_cache, ), ) successful_assets.append("asymptote") @@ -964,6 +992,7 @@ def generate_assets( ext_converter=partial( generate.individual_sage, cache_dir=self.generated_cache_abspath(), + skip_cache=skip_cache, ), ) successful_assets.append("sageplot") @@ -981,7 +1010,7 @@ def generate_assets( dest_dir=self.generated_dir_abspath() / "prefigure", outformat=outformat, ) - successful_assets.append(("prefigure", id)) + successful_assets.append("prefigure") except Exception as e: log.error(f"Unable to generate some prefigure images:\n {e}") log.debug(e, exc_info=True) @@ -1095,8 +1124,6 @@ def generate_assets( log.debug(f"Updated these assets successfully: {successful_assets}") if len(successful_assets) > 0: for asset_type in successful_assets: - if asset_type not in saved_asset_table: - saved_asset_table[asset_type] = {} saved_asset_table[asset_type] = source_asset_table[asset_type] # Save the asset table to disk: self.save_asset_table(saved_asset_table) @@ -1137,7 +1164,7 @@ def validate_path(cls, path: t.Union[Path, str]) -> Path: # A path, relative to the project directory, prepended to any target's `xsl`. xsl: Path = pxml.attr(default=Path("xsl")) # A path, relative to the project directory, for storing cached generated assets - generated_cache: Path = pxml.attr(default=Path(".generated-cache")) + generated_cache: Path = pxml.attr(default=Path(".cache")) targets: t.List[Target] = pxml.wrapped( "targets", pxml.element(tag="target", default=[]) ) diff --git a/pretext/project/generate.py b/pretext/project/generate.py index 1d0157c5..c3e55501 100644 --- a/pretext/project/generate.py +++ b/pretext/project/generate.py @@ -1,3 +1,4 @@ +import typing as t import logging import hashlib from pathlib import Path @@ -12,8 +13,16 @@ def individual_asymptote( - asydiagram, outformat, method, asy_cli, asyversion, alberta, dest_dir, cache_dir -): + asydiagram: str, + outformat: str, + method: str, + asy_cli: t.List[str], + asyversion: str, + alberta: str, + dest_dir: Path, + cache_dir: Path, + skip_cache: bool = False, +) -> None: """ Checks whether a cached version of the diagram in the correct outformat exists. If it does, copies it to the dest_dir and returns. If it does not, calls the core.individual_asymptote_conversion function to generate the diagram in the correct outformat and then copies it to the dest_dir. In the latter case, also makes a copy to the cached version in the cache_dir. - outformat will be a file extension. @@ -22,7 +31,7 @@ def individual_asymptote( asset_file = Path(asydiagram).resolve() cache_file = cache_asset_filename(asset_file, outformat, cache_dir) output_file = dest_dir / asset_file.with_suffix(f".{outformat}").name - if cache_file.exists(): + if cache_file.exists() and not skip_cache: log.debug(f"Copying cached asymptote diagram {cache_file} to {output_file}") shutil.copy2(cache_file, output_file) else: @@ -37,7 +46,14 @@ def individual_asymptote( log.debug("Finished individual_asymptote function") -def individual_sage(sageplot, outformat, dest_dir, sage_executable_cmd, cache_dir): +def individual_sage( + sageplot: str, + outformat: str, + dest_dir: Path, + sage_executable_cmd: t.List[str], + cache_dir: Path, + skip_cache: bool = False, +) -> None: """ Checks whether a cached version of the diagram in the correct outformat exists. If it does, copies it to the dest_dir and returns. If it does not, calls the core.individual_asymptote_conversion function to generate the diagram in the correct outformat and then copies it to the dest_dir. In the latter case, also makes a copy to the cached version in the cache_dir. - outformat will be a file extension. @@ -47,7 +63,7 @@ def individual_sage(sageplot, outformat, dest_dir, sage_executable_cmd, cache_di asset_file = Path(sageplot).resolve() cache_file = cache_asset_filename(asset_file, outformat, cache_dir) output_file = dest_dir / asset_file.with_suffix(f".{outformat}").name - if cache_file.exists(): + if cache_file.exists() and not skip_cache: log.debug(f"Copying cached sageplot diagram {cache_file} to {output_file}") shutil.copy2(cache_file, output_file) else: @@ -62,7 +78,14 @@ def individual_sage(sageplot, outformat, dest_dir, sage_executable_cmd, cache_di log.debug("Finished individual_sage function") -def individual_latex_image(latex_image, outformat, dest_dir, method, cache_dir): +def individual_latex_image( + latex_image: str, + outformat: str, + dest_dir: Path, + method: str, + cache_dir: Path, + skip_cache: bool = False, +) -> None: """ Checks whether a cached version of the diagram in the correct outformat exists. If it does, copies it to the dest_dir and returns. If it does not, calls the core.individual_latex_image_conversion function to generate the diagram in the correct outformat and then copies it to the dest_dir. In the latter case, also makes a copy to the cached version in the cache_dir. - outformat will be 'all' or a file extension. @@ -82,7 +105,7 @@ def individual_latex_image(latex_image, outformat, dest_dir, method, cache_dir): if not cache_files[ext].exists(): all_cached = False break - if all_cached: + if all_cached and not skip_cache: for ext in outformats: log.debug( f"Copying cached latex-image {cache_files[ext]} to {output_files[ext]}" diff --git a/pretext/types.py b/pretext/types.py index 0aa971a3..755c1a54 100644 --- a/pretext/types.py +++ b/pretext/types.py @@ -1,4 +1,4 @@ import typing as t # AssetTable is a dictionary of asset types mapped to dictionaries of xml:ids to hashes of the source of that xml:id. -AssetTable = t.Dict[str, t.Dict[str, bytes]] +AssetTable = t.Dict[str, str] diff --git a/templates/.gitignore b/templates/.gitignore index 2e640cb4..ea415215 100644 --- a/templates/.gitignore +++ b/templates/.gitignore @@ -11,7 +11,7 @@ published # don't track assets generated from source generated-assets -.generated-cache +.cache # don't track the executables.ptx file executables.ptx diff --git a/templates/project.ptx b/templates/project.ptx index a8ac299f..0035dba5 100644 --- a/templates/project.ptx +++ b/templates/project.ptx @@ -22,6 +22,7 @@ stage="output/stage" xsl="xsl" asy-method="server" + generated-cache=".cache" >