Merge pull request #86 from Temigo/temigo

Documentation

.gitignore

@@ -78,6 +78,8 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/_static/
+docs/_templates/
 
 # PyBuilder
 target/

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally build your docs in additional formats such as PDF
+#formats:
+#   - pdf
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+   version: 3.7
+   install:
+   - requirements: docs/requirements_rtd.txt

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = ./source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.md

@@ -0,0 +1,41 @@
+# Documentation
+
+[![Documentation Status](https://readthedocs.org/projects/lartpc-mlreco3d/badge/?version=latest)](https://lartpc-mlreco3d.readthedocs.io/en/latest/?badge=latest)
+
+We use Sphinx to generate the documentation, and Readthedocs.io to host it at https://lartpc-mlreco3d.readthedocs.io/en/latest/.
+In theory the online documentation gets built and updated automatically every time the source branch changes (for now, this is set to `Temigo/lartpc_mlreco3d`, branch `temigo`).
+
+## Writing docstrings
+If possible, let us try to consistently use NumPy style. See [Napoleon](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/index.html) and [NumPy](https://numpydoc.readthedocs.io/en/latest/format.html) style guides.
+
+```
+def func(arg1, arg2):
+    """Summary line.
+
+    Extended description of function.
+
+    Parameters
+    ----------
+    arg1 : int
+        Description of arg1
+    arg2 : str
+        Description of arg2
+
+    Returns
+    -------
+    bool
+        Description of return value
+
+    """
+    return True
+```
+
+## Building the documentation locally
+If you would like to build it yourself on your local computer:
+```
+$ cd docs/
+$ pip install -r requirements.txt
+$ sphinx-apidoc -f -o ./source ../mlreco/ ../mlreco/models/arxiv/
+$ make html
+```
+Then open the file `docs/_build/html/index.html` in your favorite browser.

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,8 @@
+sphinx-copybutton==0.3.1
+sphinx-rtd-theme==0.5.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4

docs/requirements_rtd.txt

@@ -0,0 +1,14 @@
+numpy
+torch
+plotly
+scikit-learn
+pandas
+PyYAML
+sphinx-copybutton==0.3.1
+sphinx-rtd-theme==0.5.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4

docs/source/Configuration.rst

@@ -0,0 +1,187 @@
+Configuration
+=============
+
+High-level overview
+-------------------
+Configuration files are written in the YAML format.
+Some examples are distributed in the `config/` folder.
+This page is a reference for the various configuration
+keys and options that are generic. Module- or network-
+specific configuration can be found in the corresponding
+documentation.
+
+There are up to four top-level sections in a config file:
+
+- `iotool`
+- `model`
+- `trainval`
+- `post_processing` (optional)
+
+``iotool`` section
+------------------
+
+
+..  rubric:: batch_size (default: 1)
+
+..  rubric:: shuffle (default: True)
+
+Whether to randomize the dataset sampling.
+
+..  rubric:: num_workers (default: 1)
+
+How many workers should be processing the
+dataset in parallel.
+
+..  rubric:: collate_fn (default: None)
+
+How to collate data from different events
+into a single batch.
+Can be `None`, `CollateSparse`, `CollateDense`.
+
+..  rubric:: sampler (batch_size, name)
+
+The sampler defines how events are picked in
+the dataset. For training it is better to use
+something like :any:`RandomSequenceSampler`. For
+inference time you can omit this field and it
+will fall back to the default, a sequential
+sampling of the dataset. Available samplers
+are in :any:`mlreco.iotools.samplers`.
+
+An example of sampler config looks like this:
+
+..  code-block:: yaml
+
+    sampler:
+      batch_size: 32
+      name: RandomSequenceSampler
+
+.. note:: The batch size should match the one specified above.
+
+..  rubric:: dataset
+
+An example of `dataset` config looks like this:
+
+..  code-block:: yaml
+    :linenos:
+
+      dataset:
+        name: LArCVDataset
+        data_keys:
+          - /gpfs/slac/staas/fs1/g/neutrino/kterao/data/wire_mpvmpr_2020_04/train_*.root
+          #- /gpfs/slac/staas/fs1/g/neutrino/kvtsang/data/pdune/mpvmpr/train/larcv*.root
+        limit_num_files: 10
+        schema:
+          input_data:
+            - parse_sparse3d_scn
+            - sparse3d_reco
+
+``model`` section
+-----------------
+
+..  rubric:: name
+
+Name of the model that you want to run
+(typically one of the models under `mlreco/models`).
+
+..  rubric:: modules
+
+An example of `modules` looks like this for the model
+`full_chain`:
+
+..  code-block:: yaml
+
+    modules:
+      chain:
+        enable_uresnet: True
+        enable_ppn: True
+        enable_cnn_clust: True
+        enable_gnn_shower: True
+        enable_gnn_track: True
+        enable_gnn_particle: False
+        enable_gnn_inter: True
+        enable_gnn_kinematics: False
+        enable_cosmic: False
+        enable_ghost: True
+        use_ppn_in_gnn: True
+      some_module:
+        ... config of the module ...
+
+..  rubric:: network_input
+
+..  rubric:: loss_input
+
+
+``trainval`` section
+--------------------
+
+..  rubric:: seed
+
+Integer to use as random seed.
+
+..  rubric:: unwrapper
+
+Can be `unwrap_3d_scn` or `unwrap_2d_scn`.
+
+.. rubric:: concat_result
+
+Typically looks like this:
+
+.. code-block:: yaml
+
+    concat_result: ['seediness', 'margins', 'embeddings', 'fragments', 'fragments_seg', 'shower_fragments', 'shower_edge_index','shower_edge_pred','shower_node_pred','shower_group_pred','track_fragments', 'track_edge_index', 'track_node_pred', 'track_edge_pred', 'track_group_pred', 'particle_fragments', 'particle_edge_index', 'particle_node_pred', 'particle_edge_pred', 'particle_group_pred', 'particles','inter_edge_index', 'inter_node_pred', 'inter_edge_pred', 'node_pred_p', 'node_pred_type', 'flow_edge_pred', 'kinematics_particles', 'kinematics_edge_index', 'clust_fragments', 'clust_frag_seg', 'interactions', 'inter_cosmic_pred', 'node_pred_vtx', 'total_num_points', 'total_nonghost_points']
+
+.. rubric:: gpus
+
+If empty string, use CPU. Otherwise string
+containing one or more GPU ids.
+
+..  rubric:: weight_prefix
+
+Path to folder where weights will be saved.
+Includes the weights file prefix, e.g.
+`/path/to/snapshot-` for weights that will be
+named `snapshot-0000.ckpt`, etc.
+
+..  rubric:: iterations
+
+..  rubric:: report_step
+
+How often to print in the console log.
+
+.. rubric:: checkpoint_step
+
+How often to save the weights in a
+checkpoint file.
+
+.. rubric:: model_path
+
+Can be empty string. Otherwise, path to a
+checkpoint file to load for the whole model.
+
+.. rubric:: log_dir
+
+Path to a folder where logs will be stored.
+
+..  rubric:: train
+
+Boolean, whether to use train or inference mode.
+
+..  rubric:: debug
+
+..  rubric:: minibatch_size (default: -1)
+
+..  rubric:: optimizer
+
+Can look like this:
+
+..  code-block:: yaml
+
+    optimizer:
+      name: Adam
+      args:
+        lr: 0.001
+
+``post_processing`` section
+---------------------------
+Coming soon.

docs/source/GettingStarted.rst

@@ -0,0 +1,20 @@
+Getting started
+===============
+
+``lartpc_mlreco3d`` is a machine learning pipeline for LArTPC data.
+
+Basic example
+--------------
+
+.. code-block:: python
+   :linenos:
+
+   # assume that lartpc_mlreco3d folder is on python path
+   from mlreco.main_funcs import process_config, train
+   import yaml
+   # Load configuration file
+   with open('lartpc_mlreco3d/config/test_uresnet.cfg', 'r') as f:
+       cfg = yaml.load(f, Loader=yaml.Loader)
+   process_config(cfg)
+   # train a model based on configuration
+   train(cfg)