🐛(tests) use LMS settings in tests (#9)

** FIX DETAILS **

* configure & test DRF API versioning
* use lms settings in tests
* get rid of tox in our CI:

Using tox within docker containers seems overkill; let's make things a
little bit "simpler" by dropping it once for all.

* Prevent root user creation:

when trying to build the base container with no user or group as build
arguments, docker tries to create the root user. We prevent this
behavior by only creating a user when uid:gid are distinct from root
user ids.

* Update volume endpoints:

fundocker/edxapp now use the same application tree as edx, we need to
stick to it to make it works.

** MISC **

* fix broken comparison in ci script
* use super() in test cases setUp method
* remove unused CMS service
* improve CI step switch/case

.travis.yml

@@ -5,10 +5,10 @@ cache:
   - "$HOME/.cache/db"
   - node_modules
 env:
-- TOXENV=quality
-- TOXENV=docs
-- TOXENV=django18
-- API_SPEC=1 OPENEDX_RELEASE=ginkgo.1
+- CI_STEP=quality
+- CI_STEP=docs
+- CI_STEP=test
+- CI_STEP=test-spec OPENEDX_RELEASE=ginkgo.1
 before_install:
 - bin/ci before-install
 install:
@@ -23,7 +23,7 @@ deploy:
   distributions: sdist bdist_wheel
   on:
     tags: true
-    condition: "$TOXENV = quality"
+    condition: "$CI_STEP = quality"
   # Encrypted password set via "travis encrypt --add deploy.password"; for
   # details, see https://docs.travis-ci.com/user/deployment/pypi
   password:

Dockerfile

@@ -8,9 +8,11 @@ ENV DOCKERIZE_VERSION v0.6.0
 ARG user=0
 ARG group=0
 
-# Add a non-privileged user to run the application
-RUN groupadd --gid $group app && \
-    useradd --uid $user --gid $group --home /app --create-home app
+# Add a non-privileged user to run the application if given as a build argument
+RUN if [ ${user} -ne 0 -a ${group} -ne 0 ]; then \
+        groupadd --gid $group app ; \
+        useradd --uid $user --gid $group --home /app app ; \
+    fi
 
 # Install dockerize
 RUN curl -L \

Makefile

@@ -1,9 +1,6 @@
-.PHONY: clean compile_translations coverage docs dummy_translations \
-	extract_translations fake_translations help pull_translations push_translations \
-	quality requirements run selfcheck stop test test-all validate
-
-.DEFAULT_GOAL := help
-
+#
+# GLOBALS
+#
 define BROWSER_PYSCRIPT
 import os, webbrowser, sys
 try:
@@ -16,83 +13,105 @@ endef
 export BROWSER_PYSCRIPT
 BROWSER := python -c "$$BROWSER_PYSCRIPT"
 
-# Auth
-USER_ID  = $(shell id -u)
-GROUP_ID = $(shell id -g)
-
 # Dockerized tools
 # -- Docker
-COMPOSE            = docker-compose
-COMPOSE_BUILD      = $(COMPOSE) build --build-arg user=$(USER_ID) --build-arg group=$(GROUP_ID)
-COMPOSE_RUN        = $(COMPOSE) run --rm --user=$(USER_ID):$(GROUP_ID)
+COMPOSE            = bin/compose
+COMPOSE_BUILD      = bin/build
+COMPOSE_RUN        = bin/run
 COMPOSE_RUN_LMS    = $(COMPOSE_RUN) lms
 COMPOSE_RUN_FONZIE = $(COMPOSE_RUN) fonzie
 COMPOSE_RUN_NODE   = $(COMPOSE_RUN) node
 
 # -- Python
-COVERAGE    = $(COMPOSE_RUN_FONZIE) coverage
+COVERAGE    = $(COMPOSE_RUN_LMS) coverage
 DIFF-COVER  = $(COMPOSE_RUN_FONZIE) diff-cover
-PIP         = $(COMPOSE_RUN_FONZIE) pip
-PIP-COMPILE = $(COMPOSE_RUN_FONZIE) pip-compile
-PIP-SYNC    = $(COMPOSE_RUN_FONZIE) pip-sync
-PYTEST      = $(COMPOSE_RUN_FONZIE) pytest
-TOX         = $(COMPOSE_RUN_FONZIE) tox
+PYTEST      = bin/pytest
 
 # -- Node
 YARN = $(COMPOSE_RUN_NODE) yarn
 
+#
+# RULES
+#
+default: help
 
 help: ## display this help message
 	@echo "Please use \`make <target>' where <target> is one of"
 	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+.PHONY: help
+
+bootstrap: build ## bootstrap the project
+	$(COMPOSE_RUN_LMS) python manage.py lms migrate
+.PHONY: bootstrap
+
+build: ## build project containers
+	$(COMPOSE_BUILD) lms
+	$(COMPOSE_RUN) --no-deps node yarn install
+.PHONY: build
 
 clean: ## remove generated byte code, coverage reports, and build artifacts
 	$(COMPOSE_RUN_FONZIE) find . -name '__pycache__' -exec rm -rf {} +
 	$(COMPOSE_RUN_FONZIE) find . -name '*.pyc' -exec rm -f {} +
 	$(COMPOSE_RUN_FONZIE) find . -name '*.pyo' -exec rm -f {} +
 	$(COMPOSE_RUN_FONZIE) find . -name '*~' -exec rm -f {} +
-	$(COVERAGE) erase
+	$(COMPOSE_RUN) --no-deps lms coverage erase
 	$(COMPOSE_RUN_FONZIE) rm -fr build/
 	$(COMPOSE_RUN_FONZIE) rm -fr dist/
 	$(COMPOSE_RUN_FONZIE) rm -fr *.egg-info
+.PHONY: clean
 
 coverage: clean ## generate and view HTML coverage report
 	$(PYTEST) --cov-report html
 	$(BROWSER) htmlcov/index.html
+.PHONY: coverage
+
+diff_cover: test
+	$(DIFF-COVER) edx-platform/reports/coverage.xml
+.PHONY: diff_cover
 
 docs: ## generate Sphinx HTML documentation, including API docs
-	$(TOX) -e docs
+	$(COMPOSE_RUN_FONZIE) doc8 --ignore-path docs/_build README.rst docs
+	rm -f docs/fonzie.rst
+	rm -f docs/modules.rst
+	$(COMPOSE_RUN_FONZIE) make -C docs clean
+	$(COMPOSE_RUN_FONZIE) make -C docs html
+	$(COMPOSE_RUN_FONZIE) python setup.py check --restructuredtext --strict
 	$(BROWSER) docs/_build/html/index.html
+.PHONY: docs
 
 quality: ## check coding style with pycodestyle and pylint
-	$(TOX) -e quality
-
-requirements: ## install development environment requirements
-	$(COMPOSE_BUILD) lms
-	$(COMPOSE_BUILD) fonzie
-	$(YARN) install
+	touch tests/__init__.py
+	$(COMPOSE_RUN_FONZIE) pylint fonzie tests test_utils
+	$(COMPOSE_RUN_FONZIE) pylint --py3k fonzie tests test_utils
+	rm tests/__init__.py
+	$(COMPOSE_RUN_FONZIE) pycodestyle fonzie tests
+	$(COMPOSE_RUN_FONZIE) pydocstyle fonzie tests
+	$(COMPOSE_RUN_FONZIE) isort --check-only --recursive tests test_utils fonzie manage.py setup.py
+	${MAKE} selfcheck
+	$(COMPOSE_RUN_FONZIE) pyroma .
+.PHONY: quality
+
+run: ## start development server
+	$(COMPOSE) up -d
+.PHONY: run
 
-test: clean ## run tests in the current virtualenv
-	$(PYTEST)
+selfcheck: ## check that the Makefile is well-formed
+	@echo "The Makefile is well-formed."
+.PHONY: selfcheck
 
-diff_cover: test
-	$(DIFF-COVER) coverage.xml
+stop:  ## stop development server
+	$(COMPOSE) stop
+.PHONY: stop
 
-test-all: ## run tests on every supported Python/Django combination
-	$(TOX) -e quality
-	$(TOX)
+test: clean ## run python tests suite
+	$(PYTEST)
+.PHONY: test
 
 test-spec: ## run tests on API specification (API blueprint)
 	$(COMPOSE) up -d lms
+	$(COMPOSE_RUN_LMS) dockerize -wait tcp://mysql:3306 -timeout 60s
 	$(YARN) dredd
+.PHONY: test-spec
 
 validate: quality test ## run tests and quality checks
-
-selfcheck: ## check that the Makefile is well-formed
-	@echo "The Makefile is well-formed."
-
-run: ## start develppment server
-	$(COMPOSE) up -d
-
-stop:  ## stop development server
-	$(COMPOSE) stop
+.PHONY: validate

bin/ci

@@ -48,7 +48,6 @@ function install_dependencies() {
     echo "Building fonzie containers..."
 
     _dc_build lms
-    _dc_build fonzie
     FONZIE_UNSET_USER=1 _dc_run --no-deps node yarn install --no-progress
 }
 
@@ -59,49 +58,49 @@ function install_dependencies() {
 function run_test_suite() {
     echo "Starting the test suite..."
 
-    if [ -z $TOXENV ]; then
-        echo "Running API spec test suite..."
+    _docker_compose --no-ansi up -d lms
 
-        _docker_compose --no-ansi up -d lms
+    # Wait for mysql database to accept connections
+    _dc_run fonzie dockerize -wait tcp://mysql:3306 -timeout 60s
 
-        # Wait for mysql database to accept connections
-        _dc_run fonzie dockerize -wait tcp://mysql:3306 -timeout 60s
+    local sql="$HOME/.cache/db/$OPENEDX_RELEASE.sql"
+    local database="edxapp"
 
-        local sql="$HOME/.cache/db/$OPENEDX_RELEASE.sql"
-        local database="edxapp"
+    if [ -e $sql ]; then
+        echo "Stack status:"
+        _docker_compose ps
 
-        if [ -e $sql ]; then
-            echo "Stack status:"
-            _docker_compose ps
+        echo "MySQL container logs:"
+        _docker_compose logs mysql
 
-            echo "MySQL container logs:"
-            _docker_compose logs mysql
+        echo "MySQL container ID:"
+        _docker_compose ps -q mysql
 
-            echo "MySQL container ID:"
-            _docker_compose ps -q mysql
-
-            echo "Loading database schema from cache: $sql"
-            # FIXME: docker-compose is having issues with huge files streamed
-            # via a pty/stdin, so we bypass compose to use raw docker exec
-            # command.
-            docker exec -i $(_docker_compose ps -q mysql) mysql -u root $database < $sql
-        else
-            # Perform database migrations
-            echo "Starting database migration"
-            _dc_run lms python ./manage.py lms migrate
-
-            # Dump database
-            echo "Dump database schema to: $sql"
-            _dc_exec mysql mysqldump -u root $database > $sql
-        fi
-
-        # Run API implementation tests
-        _dc_run node yarn dredd
+        echo "Loading database schema from cache: $sql"
+        # FIXME: docker-compose is having issues with huge files streamed
+        # via a pty/stdin, so we bypass compose to use raw docker exec
+        # command.
+        docker exec -i $(_docker_compose ps -q mysql) mysql -u root $database < $sql
     else
-        echo "Running tox:$TOXENV test suite..."
+        # Perform database migrations
+        echo "Starting database migration"
+        _dc_run lms python ./manage.py lms migrate
 
-        _dc_run -e TOXENV fonzie tox
+        # Dump database
+        echo "Dump database schema to: $sql"
+        _dc_exec mysql mysqldump -u root $database > $sql
     fi
+
+    case "$CI_STEP" in
+        quality|docs|test)
+            make ${CI_STEP}
+            ;;
+        test-spec)
+            echo "Running API spec test suite..."
+            _dc_run node yarn dredd
+            ;;
+        *)
+    esac
 }
 
 

bin/tox → bin/pytest

@@ -2,4 +2,4 @@
 
 source "$(dirname "${BASH_SOURCE[0]}")/config.sh"
 
-_dc_run -e TOXENV fonzie tox "$@"
+_dc_run lms pytest -c /app/fonzie/setup.cfg "$@"

docker-compose.yml

@@ -30,7 +30,8 @@ services:
       - "8072:8000"
     volumes:
       - ./edx-platform/config/docker_run_lms.py:/config/docker_run_lms.py
-      - ./edx-platform/lms/urls.py:/app/edx-platform/lms/urls.py
+      - ./edx-platform/lms/urls.py:/edx/app/edxapp/edx-platform/lms/urls.py
+      - ./edx-platform/reports:/edx/app/edxapp/edx-platform/reports
       - .:/app/fonzie
     tmpfs:
       - /data/static/lms
@@ -44,18 +45,6 @@ services:
       - memcached
       - rabbitmq
 
-  cms:
-    image: edxapp-fonzie
-    environment:
-      SERVICE_VARIANT: cms
-      DJANGO_SETTINGS_MODULE: cms.envs.docker_run
-    ports:
-      - "8082:8000"
-    command: >
-      dockerize -wait tcp://mysql:3306 -timeout 60s python manage.py cms runserver 0.0.0.0:8000
-    depends_on:
-      - lms
-
   fonzie:
     image: edxapp-fonzie
     volumes:

docs/conf.py

@@ -49,7 +49,7 @@
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #
-# needs_sphinx = '1.0'
+needs_sphinx = '1.3'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom

edx-platform/config/docker_run_lms.py

@@ -21,3 +21,10 @@
 )
 
 ROOT_URLCONF = 'lms.urls'
+
+# Django Rest Framework (aka DRF)
+REST_FRAMEWORK = {
+    'ALLOWED_VERSIONS': ('1.0', ),
+    'DEFAULT_VERSION': '1.0',
+    'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.URLPathVersioning',
+}

manage.py

@@ -11,7 +11,7 @@
 PWD = os.path.abspath(os.path.dirname(__file__))
 
 if __name__ == '__main__':
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lms.envs.docker_run')
     sys.path.append(PWD)
     try:
         from django.core.management import execute_from_command_line  # pylint: disable=wrong-import-position