release.sos

#!/usr/bin/env sos-runner
#fileformat=SOS1.0
import os, sys, glob

sys.path.append('docs')
parameter: pip = 'pip'

[default_1 (Configure website): shared = ['conf', 'binders', 'notebook_files', 'rmd_files', 'index_files', 'pipeline_files', 'generated_files', 'password', 'status']]
depends: Py_Module('bs4', autoinstall=True), Py_Module('date_parser', autoinstall=True)
DEFAULT_CONF = {
           '__version__': '0.9.7',
           '__sos_version__': '0.19.12',
           '__about_commit__': 'https://stephenslab.github.io/ipynb-website/notes.html#Note-about-commit-ids',
           'name': 'A Pet Project',
           'repo': 'http://github.com/vatlab/jnbinder',
           'footer': "&copy 2016-2017 Gao Wang at Stephens Lab, University of Chicago",
           'include_dir': [],
           'exclude_file': [],
           'hide_navbar': [],
           'disqus': None,
           'report_style': False,
           'release_message': None,
           'post_release_actions': None,
           'homepage_label': "Overview",
           'source_label': '<span class="fa fa-github"></span>',
           'theme': 'cosmo',
           'homepage': 'Overview.ipynb',
           'skip_homepage': False,
           'font': "Droid Sans",
           'notebook_toc': True,
           'add_commit_info': True,
           'long_autoindex': True,
           'reverse_autoindex': [],
           'remove_whiteline': False,
           'auto_highlight': None,
           'password_file': None,
           'site_url': None
           }
status = 0
try:
  from jnbinder import compare_versions, update_gitignore
except Exception as e:
  if 'jnbinder' in str(e):
     sos_run('setup-jnbinder')
     try:
       from jnbinder import compare_versions, update_gitignore
     except Exception as e:
       status = 1
       fail_if(status, msg = str(e))
  else:
     raise RuntimeError('{}. You can try `pip install` to install the missing module.'.format(e))
if len(glob.glob('.sos/*')) and os.path.isdir(".git"):
   update_gitignore()
if compare_versions(SOS_VERSION, DEFAULT_CONF["__sos_version__"]) < 0:
  sos_run("upgrade-sos")
  status = 1
  fail_if(status, msg = 'Cannot generate documents due to obsolete SoS software. Though we have attempted to upgrade SoS (see above) you need to rerun this script to load the updates.')
#
err_msg = '''
No valid path found in ``include_dir``
Please properly set ``"include_dir"`` in ``config.yml``
'''
import yaml
with open('config.yml', 'r') as f:
    conf = yaml.safe_load(f)
fail_if(conf is None, msg = err_msg)
conf.update({k:v for k,v in DEFAULT_CONF.items() if k not in conf})
conf['jt_theme'] = conf['theme'] if os.path.isfile(f"docs/css/{conf['theme']}.css") else None
for k in ['include_dir', 'hide_navbar', 'exclude_file', 'reverse_autoindex']:
    conf[k] = [os.path.normpath(x) for x in conf[k]]
for item in conf['include_dir']:
    if os.path.basename(item) in ['css', 'fonts', 'js', 'site_libs', 'docs']:
       status = 1
       fail_if(stats, msg = f'Folder name ``{item}`` conflicts with website library paths. Please use a different name.')
conf['remove_whiteline'] = 1 if conf['remove_whiteline'] else 0
if conf['site_url'] is None:
   conf['site_url'] = f"http://{path(conf['repo']):db}.github.io/{path(conf['repo']):b}"
if conf['auto_highlight'] is None:
   conf['auto_highlight'] = ('pack', 'null')
else:
   if os.path.isfile(f'docs/site_libs/highlightjs/highlight.{conf["auto_highlight"]}.js'):
      conf['auto_highlight'] = (conf['auto_highlight'], 'jnbinder')
   else:
      conf['auto_highlight'] = ('pack', 'jnbinder')
binders = [x for x in conf['include_dir'] if os.path.isdir(x)]
# binders = [x for x in next(os.walk('./'))[1] if x not in conf['exclude_dir'] + ['docs'] and not x.startswith('.')]
fail_if(len(binders) == 0, msg = err_msg)
rmd_files = sum([[y for y in glob.glob(f"{x}/*.Rmd") if y not in conf['exclude_file']] for x in binders], [])
notebook_files = sum([[y for y in sorted(glob.glob(f"{x}/*.ipynb")) if y not in [f"{x}/index.ipynb", f"{x}/_index.ipynb", conf['homepage']] + conf['exclude_file']] for x in binders], []) + [x.replace('.Rmd', '.ipynb') for x in rmd_files]
index_files = [f'{x}/index.ipynb' if os.path.isfile(f'{x}/index.ipynb') else f'{x}/_index.ipynb' for x in binders if x not in conf['hide_navbar']]
pipeline_files = sum([[y for y in glob.glob(f"{x}/*.sos") if y not in conf['exclude_file']] for x in binders], [])
generated_files = [f'docs/{x}.html' for x in binders if x not in conf['hide_navbar']] + (['docs/index.html'] if not conf['skip_homepage'] else []) + [f"docs/{path(x):d}/index.html" if f'{path(x):db}' == f'{path(x):bn}' else f"docs/{x.replace('.ipynb', '.html')}" for x in notebook_files] + ['docs/' + x.replace('.sos', '.pipeline.html') for x in pipeline_files] + ['docs/js/docs.js', 'docs/index.tpl'] + [f"docs/{x}.tpl" for x in binders]
if conf['password_file'] is not None and os.path.isfile(conf['password_file']):
   with open(conf['password_file']) as f:
        password = {os.path.normpath(k): v for k, v in yaml.safe_load(f).items()}
   for key in password:
       if not os.path.isdir(key) and not os.path.isfile(key):
          status = 1
          fail_if(status, msg = f"Cannot find ``{key}``. Please comment out or remove it from ``{conf['password_file']}``")
else:
   password = None
for item in rmd_files:
    fail_if(os.path.isfile(os.path.splitext(item)[0] + '.ipynb'), msg = f"Rmarkdown file name ``{item}`` conflicts with existing notebook ``{os.path.splitext(item)[0] + '.ipynb'}``. If the notebook was automatically generated you can use ``rm -f {os.path.splitext(item)[0] + '.ipynb'}`` to remove it.")

[download-jnbinder: provides = 'jnbinder-master/README.md']
download: decompress = True, dest_file = 'jnbinder-master.zip'
  https://github.com/vatlab/jnbinder/archive/master.zip

[setup-jnbinder]
depends: 'jnbinder-master/README.md'
run:
  yes | cp -rf jnbinder-master/docs ./
  cp jnbinder-master/config.yml config.default.yml
  rm -rf jnbinder-master*

[upgrade-jnbinder]
depends: 'jnbinder-master/README.md'
bash:
  mkdir -p docs
  rm -rf docs/{css,fonts,jnbinder.py,js,site_libs}
  cp -r jnbinder-master/docs/{css,fonts,jnbinder.py,js,site_libs} docs
  if [ -f release.sos ]; then
     cp jnbinder-master/release ./release.sos
  else
     cp jnbinder-master/release ./
  fi
  rm -rf jnbinder-master*

[upgrade-sos (SoS upgrader)]
# using latest pip release
try:
  get_output(f'{pip} install --upgrade --upgrade-strategy only-if-needed --no-cache-dir sos')
except:
  get_output(f'{pip} install --upgrade --upgrade-strategy only-if-needed --no-cache-dir --user sos')

[check-link (Broken link checker)]
depends: executable('linkchecker'), sos_variable('conf')
run: expand = True
  linkchecker {conf['site_url']}

[update-tpl (Jupyter HTML template updater)]
parameter: binders = list
parameter: conf = dict
input: None
output: [f"docs/{x}.tpl" for x in binders] + ['docs/index.tpl']
from jnbinder import make_template
make_template(conf, binders, 'docs')

[update-index (Index notebook generator)]
parameter: binders = list
parameter: notebook_files = list
parameter: conf = dict
stop_if(len(notebook_files) == 0)
input: notebook_files
output: [binder + '/_index.ipynb' for binder in binders]
from jnbinder import make_index_nb, make_empty_nb
for binder in binders:
  content = make_index_nb(binder, conf['exclude_file'] + [conf['homepage']], conf['long_autoindex'], binder in conf['reverse_autoindex'])
  with open(f'{binder}/_index.ipynb', 'w') as f:
    f.write(content)
if not os.path.isfile(conf['homepage']):
  with open(conf['homepage'], 'w') as f:
    f.write(make_empty_nb(conf['name']))

[update-hp (Index HTML updater)]
parameter: index_files = list
parameter: conf = dict
parameter: password = str
depends: sos_step("update-tpl"), executable('jupyter'), executable('sed'), executable('perl'), Py_Module('jupyter_contrib_nbextensions', autoinstall=True), path(conf['homepage']).with_suffix('.ipynb')
input_files = index_files + ([path(conf['homepage']).with_suffix('.ipynb')] if not conf['skip_homepage'] else [])
output_files = [f'docs/{x}.html' for x in binders if x not in conf['hide_navbar']] + (['docs/index.html'] if not conf['skip_homepage'] else [])
input: dynamic(input_files), group_by = 1
output: output_files, group_by = 1
bash: expand = "${ }", stderr = False
  jupyter nbconvert ${_input:e} --output ${_output:ae} --template docs/index.tpl --no-input
  # sed -i 's/<div class="prompt input_prompt">In&nbsp;\[[0-9]\]:<\/div>//g' ${_output:e}
  if [[ ${conf['remove_whiteline']} -gt 0 ]]; then
     perl -i -ne 'print if /\S/' ${_output:e}
  fi

python: expand = "${ }", input = 'docs/jnbinder.py'
  get_sha1_files([(${_input:r}, ${_output:r})], [], ${password}, write = True)

[update-nb (Notebook to HTML converter)]
# test: --execute --ExecutePreprocessor.timeout=-1
parameter: notebook_files = list
parameter: rmd_files = list
parameter: conf = dict
parameter: password = str
stop_if(len(notebook_files) == 0)
depends: sos_step("update-tpl"), executable('jupyter'), executable('sed'), executable('perl'), Py_Module('jupyter_contrib_nbextensions', autoinstall=True), paths(notebook_files)
from jnbinder import get_commit_info
nb_info = [get_commit_info(x[:-5] + 'Rmd' if path(x).with_suffix(".Rmd").exists() else x, conf) for x in notebook_files]
input: notebook_files, group_by = 1, paired_with = ['nb_info']
output: [f"docs/{path(x):d}/index.html" if f'{path(x):db}' == f'{path(x):bn}' else f"docs/{x.replace('.ipynb', '.html')}" for x in notebook_files], group_by = 1
bash: expand = "${ }", stderr = False
  jupyter nbconvert ${_input:e} --output ${_output:ae} --template docs/${_input:d}.tpl --no-input ${"--execute" if str(_input.with_suffix(".Rmd")) in rmd_files else ""}
  num=`grep -n -Fx '''${conf['footer']}''' ${_output:e} | tail -1 | sed 's/\([0-9]*\).*/\1/'`
  perl -i -lpe 'print "${_nb_info[0]}" if $. == '"$num"'+1' ${_output:e}
  if [[ ${conf['remove_whiteline']} -gt 0 ]]; then
     perl -i -ne 'print if /\S/' ${_output:e}
  fi

python: expand = "${ }", input = 'docs/jnbinder.py'
  get_sha1_files([], [(${_input:r}, ${_output:r})], ${password}, write = True)

[update-wf (Pipeline to HTML converter)]
parameter: pipeline_files = list
stop_if(len(pipeline_files) == 0)
input: pipeline_files, group_by = 1
output: [f"docs/{x.replace('.sos', '.pipeline.html')}" for x in pipeline_files], group_by = 1
run: expand = "${ }", stdout = False, stderr = False
  sos convert ${_input} ${_output}

[update-toc (TOC Javascript updater)]
parameter: notebook_files = list
parameter: index_files = list
parameter: binders = list
parameter: conf = dict
input: dynamic(notebook_files + index_files)
output: 'docs/js/docs.js'
from jnbinder import get_toc
out = [get_toc(x, conf['exclude_file'] + [conf['homepage']]) for x in binders]
with open(_output, 'w') as f:
  f.write('\n'.join(['\n'.join(x) for x in out]))

[update-search (Search Javescript updater)]
parameter: generated_files = list
parameter: conf = dict()
input: generated_files
output: 'docs/site_libs/tipuesearch/tipuesearch_content.js'
from jnbinder import generate_tipue_content
generate_tipue_content(generated_files, conf['site_url'], 'docs/')
with open('docs/search.html', 'w') as f:
  f.write(open('docs/site_libs/tipuesearch/search.html').read())

[remove-obsolete (Obsolete files remover)]
parameter: conf = dict()
parameter: generated_files = list
parameter: password = str
output: 'docs/MANIFEST'
from jnbinder import get_sha1_files
if os.path.isfile(_output):
  old_files = [x.strip() for x in open(_output).readlines()]
  all_f1 = [(x, y) for x, y in zip(index_files + [conf['homepage']], [f'docs/{x}.html' for x in binders if x not in conf['hide_navbar']] + (['docs/index.html'] if not conf['skip_homepage'] else []))]
  all_f2 = [(x, y) for x, y in zip(notebook_files,  [f"docs/{x.replace('.ipynb', '.html')}" for x in notebook_files])]
  sha1_files = [f"docs/{item.rsplit('_', 1)[-1][:-5]}/{path(item.rsplit('_', 1)[0]):b}.html" if item.rsplit('_', 1)[-1][:-5] != 'docs' else f"docs/{path(item.rsplit('_', 1)[0]):b}.html" for item in get_sha1_files(all_f1, all_f2, password)]
  for item in [x for x in old_files if not x in generated_files + sha1_files]:
    if item == 'docs/index.html':
      continue
    try:
      os.remove(item)
    except OSError:
      pass
else:
  sha1_files = []
with open(_output, 'w') as f:
  f.write('\n'.join(sorted(generated_files + sha1_files)))

[release-website (gh-pages uploader)]
parameter: conf = dict()
stop_if(conf['release_message'] is None)
try:
  get_output(f'''cd docs && git add . && git commit --no-verify -m "{conf['release_message']}" && git push --no-verify''')
  get_output(conf['post_release_actions'])
except:
  pass

[clean]
# remove generated files
depends: 'docs/MANIFEST'
for item in [x.strip() for x in open('docs/MANIFEST').readlines()]:
  try:
    os.remove(item)
  except OSError:
    pass

[ipyrmd (Rmarkdown to ipynb): provides = '{data}.ipynb']
depends: Py_Module("ipyrmd", autoinstall=True), R_library('IRkernel'), R_library('IRdisplay')
input: f'{data}.Rmd'
output: f'{data}.ipynb'
run: expand = "${ }", stdout = False, stderr = False
  ipyrmd --to ipynb --from Rmd -y -o ${_output} ${_input}

[default_2 (Build website)]
# build ipynb website ...
err_msg = '''
To properly add in version info, all changes made to source notebooks must be committed.
Please use ``git commit`` to commit changes and run this command again. Or you may
set ``add_commit_info: False`` in ``config.yml`` if you want to disable this feature.
'''
stop_if(status)
changed_list = get_output('git status --untracked-files=no --porcelain')
fail_if(os.path.isdir('.git') and ('ipynb' in changed_list or 'Rmd' in changed_list) and conf['add_commit_info'], msg = err_msg)
sos_run('update-index', conf=conf, binders=binders, notebook_files=notebook_files)
sos_run('update-hp + update-nb + update-wf + update-toc + update-search + remove-obsolete', conf=conf, binders=binders, notebook_files=notebook_files, rmd_files=rmd_files, index_files=index_files, pipeline_files=pipeline_files, generated_files=generated_files, password=password)
sos_run('release-website', conf=conf)

[default_4]
# remove intermediate files
stop_if(status)
hp = [path(conf["homepage"]).with_suffix(".ipynb")] if conf["homepage"].endswith(".Rmd") else []
for item in [binder + '/_index.ipynb' for binder in binders] + [x.replace('.Rmd', '.ipynb') for x in rmd_files] + hp:
  try:
    os.remove(item)
  except OSError:
    pass