From 4432046cefaf541a65de28eeb1c7e56a4432f738 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Tue, 3 Jan 2023 16:03:21 +0530 Subject: [PATCH 1/5] feat: Use poetry everywhere, remove vagrant support - Update ansible to use poetry - Do not generate requirements.txt files from poetry - Upgrade poetry version to 1.3.1 --- .github/workflows/main.yml | 10 ++-- README.md | 28 +---------- hooks/post_gen_project.sh | 6 +-- .../.github/workflows/main.yml | 23 ++++----- {{cookiecutter.github_repository}}/Makefile | 28 +++-------- {{cookiecutter.github_repository}}/README.md | 16 ------- .../Vagrantfile | 48 ------------------- .../compose/dev/django/Dockerfile | 2 +- .../compose/local/Dockerfile | 2 +- .../docs/backend/server_config.md | 1 + .../provisioner/hosts | 19 -------- .../provisioner/roles/common/tasks/main.yml | 15 ------ .../provisioner/roles/nginx/tasks/main.yml | 7 ++- .../roles/nginx/templates/site.80.conf.j2 | 4 +- .../roles/project_data/defaults/main.yml | 1 - .../roles/project_data/tasks/main.yml | 35 ++++++++------ .../provisioner/site.yml | 20 -------- .../provisioner/vars.yml | 1 - .../runtime.txt | 1 - 19 files changed, 54 insertions(+), 213 deletions(-) delete mode 100644 {{cookiecutter.github_repository}}/Vagrantfile delete mode 100644 {{cookiecutter.github_repository}}/runtime.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4e00e7d..feac8db3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,13 +28,9 @@ jobs: with: python-version: "3.9" cache: "pip" - - name: Install cookiecutter + - name: Install cookiecutter and poetry run: | python -m pip install --upgrade pip - pip3 install cookiecutter==1.7.3 - - name: Install poetry - run: | - pip3 install poetry==1.2.0 + pip3 install cookiecutter==2.1.1 poetry==1.3.1 - name: Run tests - run: | - ./run_test.sh + run: bash run_test.sh diff --git a/README.md b/README.md index b8eb5b20..57e43dc1 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ If you opt to setup the project automatically, it will also: - initialize a git repo and bump initial tag and version. - create a virtualenv in the folder `venv` inside the project. - install all the python dependencies inside it. -- create `poetry.lock` file after resolving dependencies and then generate `requirements.txt` and `requirements_dev.txt` for production and dev use respectively, for backward-compatibility. +- create `poetry.lock` file after resolving dependencies. - create a postgres database and run the initial migration against it. then only thing you'll need to do is: @@ -65,32 +65,6 @@ Don't forget to carefully look at the generated README. Awesome, right? You can also explore the [wiki] section for details on advance setup and usages. -## Managing dependencies - -### Poetry - -To guarantee repeatable installations, all project dependencies are managed using [Poetry](https://python-poetry.org/). The project’s direct dependencies are listed in `pyproject.toml`. -Running `poetry lock` generates `poetry.lock` which has all versions pinned. - -You can install Poetry by using `pip install --pre poetry` or by following the official installation guide [here](https://github.com/python-poetry/poetry#installation). - -*Tip:* We recommend that you use this workflow and keep `pyproject.toml` as well as `poetry.lock` under version control to make sure all computers and environments run exactly the same code. - -### Other tools - -For compatibility, `requirements.txt` and `requirements_dev.txt` can be updated by running - -```bash -poetry export -f requirements.txt -o requirements.txt -poetry export -f requirements.txt -o requirements_dev.txt --dev -``` - -or - -```bash -make generate_requirements -``` - ## Articles - [Setting up Django projects in a breeze](https://medium.com/fueled-engineering/setting-up-django-projects-in-a-breeze-36c715cc9a6f) diff --git a/hooks/post_gen_project.sh b/hooks/post_gen_project.sh index af6c0a0a..40a77b5b 100755 --- a/hooks/post_gen_project.sh +++ b/hooks/post_gen_project.sh @@ -10,7 +10,7 @@ echo "${green}[Finished]${reset}" echo "==> Setup project dependencies? It will:" echo " - Create virtualenv at './{{ cookiecutter.github_repository }}/venv/'." -echo " - Install development requirements inside virtualenv." +echo " - Install development requirements using poetry" echo " - Create a postgres database named '{{ cookiecutter.main_module }}'." echo " - Run './manage.py migrate'." echo " - Initialize git." @@ -26,11 +26,11 @@ else fi if echo "{{ cookiecutter.add_heroku }}" | grep -iq "^n"; then - rm -rf uwsgi.ini Procfile runtime.txt bin/post_compile + rm -rf uwsgi.ini Procfile bin/post_compile fi if echo "{{ cookiecutter.add_ansible }}" | grep -iq "^n"; then - rm -rf provisioner Vagrantfile ansible.cfg + rm -rf provisioner ansible.cfg fi if echo "{{ cookiecutter.add_celery }}" | grep -iq "^n"; then diff --git a/{{cookiecutter.github_repository}}/.github/workflows/main.yml b/{{cookiecutter.github_repository}}/.github/workflows/main.yml index 32912a6a..b12d5776 100644 --- a/{{cookiecutter.github_repository}}/.github/workflows/main.yml +++ b/{{cookiecutter.github_repository}}/.github/workflows/main.yml @@ -33,21 +33,22 @@ jobs: sudo apt-get update sudo apt-get install postgresql-13-postgis-3-scripts {%- endif %} + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: '3.9' cache: 'pip' - - name: Install poetry - run: | - pip3 install poetry==1.2.0 - - name: Install requirements + + - name: Install python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip poetry wheel poetry install --with dev - - name: Run tests - run: | - poetry run pytest --cov -v --tb=native - - name: Linting - run: | - make lint + + - name: Run Linting + run: make lint + + - name: Run Python tests + run: poetry run pytest --cov -v --tb=native + + diff --git a/{{cookiecutter.github_repository}}/Makefile b/{{cookiecutter.github_repository}}/Makefile index b5a3e75b..cbc80bf8 100644 --- a/{{cookiecutter.github_repository}}/Makefile +++ b/{{cookiecutter.github_repository}}/Makefile @@ -4,8 +4,10 @@ SHELL := bash PROJECT_NAME={{cookiecutter.main_module}} DB_NAME=$(PROJECT_NAME) +{% if cookiecutter.add_ansible.lower() == 'y' -%} INVENTORY=provisioner/hosts PLAYBOOK=provisioner/site.yml +{%- endif %} ENV_PREFIX=$(shell echo 'poetry run ') @@ -36,32 +38,14 @@ run_all: ## Run all the servers in parallel, requires GNU Make make -j django docs{% if cookiecutter.add_celery == 'y' %} celery{% endif %} redis .PHONY: run_all -virtualenv: ## Create a virtual environment. - @echo "creating virtualenv using poetry..." - @pip install -U pip poetry - @poetry env use python3 - @echo - @echo "!!! Please run 'poetry shell' to enable the environment !!!" - - regenerate: ## Delete and create new database. -dropdb $(DB_NAME) createdb $(DB_NAME) ${ENV_PREFIX}python manage.py migrate .PHONY: regenerate -generate_requirements: - poetry export -f requirements.txt -o requirements_dev.txt --dev - poetry export -f requirements.txt -o requirements.txt - -update_libs: ## update libs + generate new lockfile & requirements - poetry update - make generate_requirements -.PHONY: update-libs - -install: virtualenv ## Install and setup project dependencies - python3 -m pip install --upgrade pip wheel - make generate_requirements +install: ## Install dependencies, create db and migrate + python3 -m pip install --upgrade pip wheel poetry poetry install ${ENV_PREFIX}pre-commit install ifneq ($(CI),True) @@ -99,7 +83,7 @@ djurls: ## Displays all the django urls shell: ## Enter the django shell ${ENV_PREFIX}python manage.py shell_plus -docs: virtualenv ## Start documentation server locally +docs: ## Start documentation server locally ${ENV_PREFIX}mkdocs serve {%- if cookiecutter.add_celery == 'y' %} @@ -110,6 +94,7 @@ celery: install ## Start celery worker redis: ## Start redis server redis-server +{% if cookiecutter.add_ansible.lower() == 'y' %} # Ansible related things # ------------------------------------------------------ # Usages: @@ -138,3 +123,4 @@ deploy_qa: deploy deploy_prod: ENV=prod ## Deploy to production server deploy_prod: deploy +{%- endif %} diff --git a/{{cookiecutter.github_repository}}/README.md b/{{cookiecutter.github_repository}}/README.md index 592b7e8c..4981f93b 100644 --- a/{{cookiecutter.github_repository}}/README.md +++ b/{{cookiecutter.github_repository}}/README.md @@ -45,22 +45,6 @@ You can install Poetry by using `pip install --pre poetry` or by following the o *Tip:* We recommend that you use this workflow and keep `pyproject.toml` as well as `poetry.lock` under version control to make sure all computers and environments run exactly the same code. -### Other tools - -For compatibility, `requirements.txt` and `requirements_dev.txt` can be updated by running - -```bash -poetry export --without-hashes -f requirements.txt -o requirements.txt -``` - -and - -```bash -poetry export --without-hashes -f requirements.txt -o requirements_dev.txt --with dev -``` - -, respectively. - ## Deploying Project diff --git a/{{cookiecutter.github_repository}}/Vagrantfile b/{{cookiecutter.github_repository}}/Vagrantfile deleted file mode 100644 index 6e3e5622..00000000 --- a/{{cookiecutter.github_repository}}/Vagrantfile +++ /dev/null @@ -1,48 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.box = "bento/ubuntu-16.04" - - # Make virtualbox use 1GB RAM. Compilation of libraries like lxml may fail for - # less than 1GB RAM. - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - end - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - config.vm.network :forwarded_port, guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - config.vm.network :private_network, ip: "192.168.33.12" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network :public_network - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - # config.ssh.forward_agent = true - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder "./", "/home/vagrant/{{ cookiecutter.github_repository }}/" - - config.vm.provision "ansible" do |ansible| - ansible.playbook = "provisioner/site.yml" - ansible.host_key_checking = "False" - ansible.inventory_path = "provisioner/hosts" - ansible.verbose = "v" - ansible.limit = "vagrant" - end -end diff --git a/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile b/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile index 72027a2a..016b993d 100644 --- a/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile +++ b/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile @@ -3,7 +3,7 @@ ARG PYTHON_VERSION=3.9-slim-buster # define an alias for the specfic python version used in this file. FROM python:${PYTHON_VERSION} as python -ENV POETRY_VERSION=1.2.0 +ENV POETRY_VERSION=1.3.1 ARG BUILD_ENVIRONMENT=dev ARG APP_HOME=/app diff --git a/{{cookiecutter.github_repository}}/compose/local/Dockerfile b/{{cookiecutter.github_repository}}/compose/local/Dockerfile index 29f5045d..b50cf68f 100644 --- a/{{cookiecutter.github_repository}}/compose/local/Dockerfile +++ b/{{cookiecutter.github_repository}}/compose/local/Dockerfile @@ -6,7 +6,7 @@ FROM python:${PYTHON_VERSION} as python ARG BUILD_ENVIRONMENT=local ARG APP_HOME=/app -ENV POETRY_VERSION=1.2.0 +ENV POETRY_VERSION=1.3.1 ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 ENV BUILD_ENV ${BUILD_ENVIRONMENT} diff --git a/{{cookiecutter.github_repository}}/docs/backend/server_config.md b/{{cookiecutter.github_repository}}/docs/backend/server_config.md index f99012b6..49c9782b 100644 --- a/{{cookiecutter.github_repository}}/docs/backend/server_config.md +++ b/{{cookiecutter.github_repository}}/docs/backend/server_config.md @@ -55,6 +55,7 @@ Run these commands to deploy this project on Heroku (substitue all references of ``` heroku create --ssh-git +heroku buildpacks:add https://github.com/moneymeets/python-poetry-buildpack.git --app= heroku buildpacks:set heroku/python --app= heroku addons:create heroku-postgresql{% if cookiecutter.add_postgis == 'y' %}:standard-0{% endif %} --app= diff --git a/{{cookiecutter.github_repository}}/provisioner/hosts b/{{cookiecutter.github_repository}}/provisioner/hosts index 36decf3c..d2ae7f42 100644 --- a/{{cookiecutter.github_repository}}/provisioner/hosts +++ b/{{cookiecutter.github_repository}}/provisioner/hosts @@ -1,31 +1,12 @@ [all:vars] -vm=0 user=ubuntu project_namespace={% raw %}{{ project_name }}-{{ deploy_env }}{% endraw %} project_path=/home/ubuntu/{% raw %}{{ deploy_env }}{% endraw %}/{{ cookiecutter.github_repository }} venv_path={% raw %}{{ project_path }}/venv{% endraw %} use_letsencrypt={{ 'True' if cookiecutter.letsencrypt.lower() == 'y' else 'False' }} letsencrypt_email={{ cookiecutter.letsencrypt_email }} -django_requirements_file=requirements.txt django_settings="settings.production" -[vagrant] -192.168.33.12 - -[vagrant:vars] -vm=1 -deploy_env=vagrant -user=vagrant -project_path=/home/vagrant/{{ cookiecutter.github_repository }} -venv_path=/home/vagrant/venv -django_requirements_file=requirements_dev.txt -django_settings="settings.development" -use_letsencrypt=False -pg_db={{ cookiecutter.main_module }} -pg_user=vagrant -pg_password=vagrant -domain_name=vagrant.{{ cookiecutter.main_module }}.com - [dev] dev.{{ cookiecutter.main_module }}.com diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml index 7daee0e7..a865304d 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml @@ -1,21 +1,6 @@ {% raw %}--- - -# Uncomment following if ipv6 is not available or not tunelled through ipv4 properly -# Disable ipv6 when running in VM(vagrant) -#- name: Disable ipv6 for all interfaces -# sysctl: name="net.ipv6.conf.all.disable_ipv6" value=1 state=present -# when: vm == 1 -# -#- name: Disable ipv6 for default interface -# sysctl: name="net.ipv6.conf.default.disable_ipv6" value=1 state=present -# when: vm == 1 -# -#- name: Disable ipv6 for local interface -# sysctl: name="net.ipv6.conf.lo.disable_ipv6" value=1 state=present -# when: vm == 1 - name: Set hostname action: shell hostnamectl set-hostname {{ domain_name }} - when: vm == 0 - name: set system locale command: update-locale LC_ALL={{ lc_all }} LANG={{ lc_lang }} LC_CTYPE={{ lc_ctype }} LC_COLLATE={{ lc_collate }} diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml index 5b869eff..260cb393 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml @@ -9,7 +9,7 @@ - name: make sure ssl directory exists file: path={{ ssl_cert_dir }} state=directory - when: vm == 0 and use_letsencrypt + when: use_letsencrypt - name: check {{ letsencrypt_ssl_cert_dir }} exists stat: path={{ letsencrypt_ssl_cert_dir }} @@ -18,7 +18,7 @@ - import_tasks: htpasswd.yml - import_tasks: letsencrypt.yml - when: vm == 0 and use_letsencrypt and letsencrypt_dir.stat.exists == false + when: use_letsencrypt and letsencrypt_dir.stat.exists == false - name: check ssl/nginx.crt exists stat: path={{ ssl_certificate }} @@ -29,11 +29,10 @@ register: nginx_key - fail: msg="Whoops! ssl certificate doesn't exist" - when: (vm == 0 and use_letsencrypt) == true and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) + when: use_letsencrypt and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) - name: generate ssl forward secrecy key command: openssl dhparam -out {{ ssl_forward_secrecy_key_path }} {{ ssl_forward_secrecy_key_length }} creates={{ ssl_forward_secrecy_key_path }} - when: vm == 0 - name: copy base nginx configuration. template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 index 78bc0d98..31cb1b43 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 @@ -2,7 +2,7 @@ server { listen 80; listen [::]:80; - server_name {% if vm %}_{% else %}{{ domain_name }}{% endif %}; + server_name {{ domain_name }}; {% if use_letsencrypt %} location /.well-known/acme-challenge/ { @@ -11,7 +11,7 @@ server { } {% endif %} - {% if vm and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) %} + {% if nginx_cert.stat.exists == false or nginx_key.stat.exists == false %} location / { uwsgi_pass unix:///tmp/uwsgi-{{ project_namespace }}.sock; include /etc/nginx/uwsgi_params; diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml index 67012813..2c02b9c2 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml @@ -3,7 +3,6 @@ pg_hstore: False pg_db: "{{ project_namespace }}" pg_user: dev pg_password: password -django_requirements_file: requirements.txt # uwsgi related variables uwsgi_user: www-data diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml index ec32104e..ff5d9276 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml @@ -3,15 +3,13 @@ git: repo={{ project_repo_url }} dest={{ project_path }} version={{ repo_version }} accept_hostkey=true become: false register: gitresult - when: vm == 0 tags: ['always'] - debug: msg="Git SHA-1 before={{ gitresult.before }} after={{ gitresult.after }}" - when: vm == 0 tags: ['always'] -- name: Ensure python virtualenv folder exist - command: python3 -m venv {{ venv_path }} creates={{ venv_path }} +- name: Ensure poetry is installed + command: python3 -m pip install poetry become: false tags: ['always'] @@ -23,8 +21,10 @@ cache_valid_time: 300 when: pg_gis -- name: Install python dependencies - pip: requirements={{ project_path }}/{{ django_requirements_file }} executable={{ venv_path }}/bin/pip +- name: Install python dependencies with Poetry + command: "poetry install" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] @@ -38,20 +38,26 @@ pkg: gettext state: present -- name: Run collect static - django_manage: command=collectstatic app_path={{ project_path }} virtualenv={{ venv_path }} +- name: Run Django collectstatic + command: "poetry run python manage.py collectstatic --no-input" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] -- name: Run database migrations - django_manage: command=migrate app_path={{ project_path }} virtualenv={{ venv_path }} +- name: Run Django database migrations + command: "poetry run python manage.py migrate" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] - import_tasks: uwsgi-setup.yml - name: Run compilemessages for static translations - django_manage: command=compilemessages app_path={{ project_path }} virtualenv={{ venv_path }} + command: "poetry run python manage.py compilemessages" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] @@ -67,16 +73,15 @@ {% raw %}- name: apt_get install graphviz for db schema generation apt: pkg=graphviz state=present -- name: Generate DB Schema. - shell: "source {{ venv_path }}/bin/activate && python bin/generate_db_schema.py" +- name: Generate DB Schema for documentation + command: "poetry run python bin/generate_db_schema.py" args: chdir: "{{ project_path }}" - executable: /bin/bash become: false tags: ['deploy', 'documentation'] - name: Build documentation for "/docs" url. - command: "{{ venv_path }}/bin/mkdocs build" + command: "poetry run mkdocs build" args: chdir: "{{ project_path }}" become: false diff --git a/{{cookiecutter.github_repository}}/provisioner/site.yml b/{{cookiecutter.github_repository}}/provisioner/site.yml index bfaa2cac..bec93a33 100644 --- a/{{cookiecutter.github_repository}}/provisioner/site.yml +++ b/{{cookiecutter.github_repository}}/provisioner/site.yml @@ -1,24 +1,4 @@ --- -- hosts: vagrant - vars_files: - - vars.yml - gather_facts: true - become: true - become_method: sudo - - roles: - - common - - nginx - - postgresql - - project_data -{%- if cookiecutter.add_celery.lower() == 'y' %} - - redis - - celery -{%- else %} - # - redis - # - celery -{%- endif %} - #= Dev #=================================================== - hosts: dev diff --git a/{{cookiecutter.github_repository}}/provisioner/vars.yml b/{{cookiecutter.github_repository}}/provisioner/vars.yml index b57c1e1e..4598924a 100644 --- a/{{cookiecutter.github_repository}}/provisioner/vars.yml +++ b/{{cookiecutter.github_repository}}/provisioner/vars.yml @@ -6,7 +6,6 @@ repo_version: master pg_gis: {{ 'True' if cookiecutter.add_postgis.lower() == 'y' else 'False' }} ansible_python_interpreter: /usr/bin/python3 django_settings: settings.production -django_requirements_file: requirements.txt python_version: 3.9 postgresql_version: 13 postgis_version: 3 diff --git a/{{cookiecutter.github_repository}}/runtime.txt b/{{cookiecutter.github_repository}}/runtime.txt deleted file mode 100644 index f72c5111..00000000 --- a/{{cookiecutter.github_repository}}/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.9.0 From 9c1ddf65c89489f2df071e04b7f4ab28355b9b5f Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 20 Dec 2023 20:03:45 +0530 Subject: [PATCH 2/5] Add back vagrantfile --- .../Vagrantfile | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 {{cookiecutter.github_repository}}/Vagrantfile diff --git a/{{cookiecutter.github_repository}}/Vagrantfile b/{{cookiecutter.github_repository}}/Vagrantfile new file mode 100644 index 00000000..6e3e5622 --- /dev/null +++ b/{{cookiecutter.github_repository}}/Vagrantfile @@ -0,0 +1,48 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.box = "bento/ubuntu-16.04" + + # Make virtualbox use 1GB RAM. Compilation of libraries like lxml may fail for + # less than 1GB RAM. + config.vm.provider "virtualbox" do |v| + v.memory = 1024 + end + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network :forwarded_port, guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network :private_network, ip: "192.168.33.12" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network :public_network + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + config.vm.synced_folder "./", "/home/vagrant/{{ cookiecutter.github_repository }}/" + + config.vm.provision "ansible" do |ansible| + ansible.playbook = "provisioner/site.yml" + ansible.host_key_checking = "False" + ansible.inventory_path = "provisioner/hosts" + ansible.verbose = "v" + ansible.limit = "vagrant" + end +end From 6b76f9985175a5eeabd9d3d6c324635e362faac1 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 20 Dec 2023 20:18:58 +0530 Subject: [PATCH 3/5] Add back vagrant config --- .github/workflows/main.yml | 2 +- README.md | 2 +- .../provisioner/hosts | 17 ++++++++++++++++ .../provisioner/roles/nginx/tasks/main.yml | 7 ++++--- .../roles/nginx/templates/site.80.conf.j2 | 2 +- .../provisioner/site.yml | 20 +++++++++++++++++++ 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index feac8db3..04cfbced 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,6 @@ jobs: - name: Install cookiecutter and poetry run: | python -m pip install --upgrade pip - pip3 install cookiecutter==2.1.1 poetry==1.3.1 + pip3 install cookiecutter==2.1.1 poetry==1.7.1 - name: Run tests run: bash run_test.sh diff --git a/README.md b/README.md index f13db6e9..8cfb7f95 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - Django 4.1.x - Python 3.9.x -- [Poetry][poetry] Support +- Dependency management via [Poetry][poetry] - Support for [black](https://pypi.org/project/black/)! - [12-Factor][12factor] based settings management via [django-environ], reads settings from `.env` if present. - Supports PostreSQL 13.0 (support of postgis-3.0 is available). diff --git a/{{cookiecutter.github_repository}}/provisioner/hosts b/{{cookiecutter.github_repository}}/provisioner/hosts index d2ae7f42..686bcc0f 100644 --- a/{{cookiecutter.github_repository}}/provisioner/hosts +++ b/{{cookiecutter.github_repository}}/provisioner/hosts @@ -1,4 +1,5 @@ [all:vars] +vm=0 user=ubuntu project_namespace={% raw %}{{ project_name }}-{{ deploy_env }}{% endraw %} project_path=/home/ubuntu/{% raw %}{{ deploy_env }}{% endraw %}/{{ cookiecutter.github_repository }} @@ -7,6 +8,22 @@ use_letsencrypt={{ 'True' if cookiecutter.letsencrypt.lower() == 'y' else 'False letsencrypt_email={{ cookiecutter.letsencrypt_email }} django_settings="settings.production" +[vagrant] +192.168.33.12 + +[vagrant:vars] +vm=1 +deploy_env=vagrant +user=vagrant +project_path=/home/vagrant/{{ cookiecutter.github_repository }} +venv_path=/home/vagrant/venv +django_settings="settings.development" +use_letsencrypt=False +pg_db={{ cookiecutter.main_module }} +pg_user=vagrant +pg_password=vagrant +domain_name=vagrant.{{ cookiecutter.main_module }}.com + [dev] dev.{{ cookiecutter.main_module }}.com diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml index 260cb393..5b869eff 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml @@ -9,7 +9,7 @@ - name: make sure ssl directory exists file: path={{ ssl_cert_dir }} state=directory - when: use_letsencrypt + when: vm == 0 and use_letsencrypt - name: check {{ letsencrypt_ssl_cert_dir }} exists stat: path={{ letsencrypt_ssl_cert_dir }} @@ -18,7 +18,7 @@ - import_tasks: htpasswd.yml - import_tasks: letsencrypt.yml - when: use_letsencrypt and letsencrypt_dir.stat.exists == false + when: vm == 0 and use_letsencrypt and letsencrypt_dir.stat.exists == false - name: check ssl/nginx.crt exists stat: path={{ ssl_certificate }} @@ -29,10 +29,11 @@ register: nginx_key - fail: msg="Whoops! ssl certificate doesn't exist" - when: use_letsencrypt and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) + when: (vm == 0 and use_letsencrypt) == true and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) - name: generate ssl forward secrecy key command: openssl dhparam -out {{ ssl_forward_secrecy_key_path }} {{ ssl_forward_secrecy_key_length }} creates={{ ssl_forward_secrecy_key_path }} + when: vm == 0 - name: copy base nginx configuration. template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 index 81fb96c6..40a4a79c 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 @@ -2,7 +2,7 @@ server { listen 80; listen [::]:80; - server_name {{ domain_name }}; + server_name {% if vm %}_{% else %}{{ domain_name }}{% endif %}; {% if use_letsencrypt %} location /.well-known/acme-challenge/ { diff --git a/{{cookiecutter.github_repository}}/provisioner/site.yml b/{{cookiecutter.github_repository}}/provisioner/site.yml index bec93a33..bfaa2cac 100644 --- a/{{cookiecutter.github_repository}}/provisioner/site.yml +++ b/{{cookiecutter.github_repository}}/provisioner/site.yml @@ -1,4 +1,24 @@ --- +- hosts: vagrant + vars_files: + - vars.yml + gather_facts: true + become: true + become_method: sudo + + roles: + - common + - nginx + - postgresql + - project_data +{%- if cookiecutter.add_celery.lower() == 'y' %} + - redis + - celery +{%- else %} + # - redis + # - celery +{%- endif %} + #= Dev #=================================================== - hosts: dev From 2a7a93b9e28f9711082cb8e617810fa2d845354c Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 20 Dec 2023 20:33:52 +0530 Subject: [PATCH 4/5] fix ansible --- .../provisioner/roles/project_data/defaults/main.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml index 6eac5cc3..30296462 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml @@ -3,17 +3,13 @@ pg_hstore: False pg_db: "{{ project_namespace }}" pg_user: dev pg_password: password +{% endraw %} {%- if cookiecutter.add_asgi.lower() == 'y' %} # asgi related variables asgi_user: www-data asgi_group: www-data asgi_workers: 2 -{% raw %} -asgi_socket: /tmp/django-{{ domain_name }}-asgi.sock -{% endraw %} -asgi_user: www-data -asgi_group: www-data -asgi_workers: 2 +{% raw %}asgi_socket: /tmp/django-{{ domain_name }}-asgi.sock{% endraw %} {% else %} # uwsgi related variables uwsgi_user: www-data @@ -32,8 +28,7 @@ uwsgi_emperor_pid_file: /run/uwsgi-emperor.pid {% raw %} uwsgi_socket: "/tmp/uwsgi-{{ project_namespace }}.sock" uwsgi_pid_file: "/tmp/uwsgi-{{ project_namespace }}.pid" - +{% endraw %} uwsgi_log_dir: /var/log/uwsgi uwsgi_log_file: "{{ uwsgi_log_dir }}/{{ project_namespace }}.log" -{% endraw %} {% endif %} From 40ae7feb2e5304a7170beeb16489fb2d6545fb13 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Wed, 20 Dec 2023 21:28:01 +0530 Subject: [PATCH 5/5] Use poetry for running celery, etc --- hooks/post_gen_project.sh | 2 +- {{cookiecutter.github_repository}}/provisioner/hosts | 2 -- .../provisioner/roles/celery/tasks/main.yml | 10 ---------- .../roles/celery/templates/celery.service.j2 | 6 +++--- .../roles/celery/templates/celerybeat.service.j2 | 2 +- .../provisioner/roles/common/defaults/main.yml | 1 - .../roles/project_data/templates/django.asgi.ini.j2 | 2 +- {{cookiecutter.github_repository}}/pyproject.toml | 4 ++-- 8 files changed, 8 insertions(+), 21 deletions(-) diff --git a/hooks/post_gen_project.sh b/hooks/post_gen_project.sh index f094ec70..3f8279ec 100755 --- a/hooks/post_gen_project.sh +++ b/hooks/post_gen_project.sh @@ -4,7 +4,7 @@ green=`tput setaf 2` reset=`tput sgr0` # Ensure newline at EOF -find . ! -path "*/venv/*" -type f -name "*.py" -exec bash -c "tail -n1 {} | read -r _ || echo >> {}" \; +find . ! -path "*/.venv/*" -type f -name "*.py" -exec bash -c "tail -n1 {} | read -r _ || echo >> {}" \; echo "${green}[Finished]${reset}" diff --git a/{{cookiecutter.github_repository}}/provisioner/hosts b/{{cookiecutter.github_repository}}/provisioner/hosts index 686bcc0f..28548aca 100644 --- a/{{cookiecutter.github_repository}}/provisioner/hosts +++ b/{{cookiecutter.github_repository}}/provisioner/hosts @@ -3,7 +3,6 @@ vm=0 user=ubuntu project_namespace={% raw %}{{ project_name }}-{{ deploy_env }}{% endraw %} project_path=/home/ubuntu/{% raw %}{{ deploy_env }}{% endraw %}/{{ cookiecutter.github_repository }} -venv_path={% raw %}{{ project_path }}/venv{% endraw %} use_letsencrypt={{ 'True' if cookiecutter.letsencrypt.lower() == 'y' else 'False' }} letsencrypt_email={{ cookiecutter.letsencrypt_email }} django_settings="settings.production" @@ -16,7 +15,6 @@ vm=1 deploy_env=vagrant user=vagrant project_path=/home/vagrant/{{ cookiecutter.github_repository }} -venv_path=/home/vagrant/venv django_settings="settings.development" use_letsencrypt=False pg_db={{ cookiecutter.main_module }} diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/celery/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/celery/tasks/main.yml index 6a728443..a5d0a7e6 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/celery/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/celery/tasks/main.yml @@ -9,16 +9,6 @@ file: path={{ celery_log_dir }} state=directory owner={{celery_user}} group={{celery_group}} mode=751 recurse=yes tags: ['configure', 'celery'] -- name: ensure python virtualenv exist - command: python3 -m venv {{ venv_path }} creates={{ venv_path }} - become: false - tags: ['celery'] - -- name: ensure celery package is installed - pip: name=celery state=present executable={{ venv_path }}/bin/pip - become: false - tags: ['celery'] - - name: copy celery service template: src=celery.service.j2 dest=/etc/systemd/system/celery-{{ project_namespace }}.service tags: ['celery'] diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celery.service.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celery.service.j2 index fca7296d..464cdea7 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celery.service.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celery.service.j2 @@ -9,10 +9,10 @@ Group={{ celery_group }} Type=forking Restart=always WorkingDirectory={{ project_path }} -ExecStart={{ venv_path }}/bin/celery -A {{ project_name }} multi start worker-{{ project_namespace }} -l {{ celery_log_level }} \ +ExecStart=poetry run celery -A {{ project_name }} multi start worker-{{ project_namespace }} -l {{ celery_log_level }} \ --logfile={{ celery_log_file }} --pidfile={{ celery_pid_file }} --schedule={{ celerybeat_schedule_file}} -ExecStop={{ venv_path }}/bin/celery multi stopwait worker-{{ project_namespace }} --pidfile={{ celery_pid_file }} -ExecReload={{ venv_path }}/bin/celery -A {{ project_name }} multi restart worker-{{ project_namespace }} -l {{ celery_log_level }} \ +ExecStop=poetry run celery multi stopwait worker-{{ project_namespace }} --pidfile={{ celery_pid_file }} +ExecReload=poetry run celery -A {{ project_name }} multi restart worker-{{ project_namespace }} -l {{ celery_log_level }} \ --logfile={{ celery_log_file }} --pidfile={{ celery_pid_file }} --schedule={{ celerybeat_schedule_file}} [Install] diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celerybeat.service.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celerybeat.service.j2 index 3597b14f..5980d649 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celerybeat.service.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/celery/templates/celerybeat.service.j2 @@ -9,7 +9,7 @@ RuntimeDirectory={{ celery_runtime_dir }} Group={{ celery_group }} Restart=always WorkingDirectory={{ project_path }} -ExecStart={{ venv_path }}/bin/celery -A {{ project_name }} beat -l {{ celery_log_level }} \ +ExecStart=poetry run celery -A {{ project_name }} beat -l {{ celery_log_level }} \ --logfile={{ celerybeat_log_file }} --pidfile={{ celerybeat_pid_file }} --schedule={{ celerybeat_schedule_file}} [Install] diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/common/defaults/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/common/defaults/main.yml index 301089ca..a1ca5fa9 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/common/defaults/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/common/defaults/main.yml @@ -18,7 +18,6 @@ base_ubuntu: - python3.9 - python3.9-dev - python-setuptools - - python3-venv - python3-pip - sysstat - vim diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/templates/django.asgi.ini.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/templates/django.asgi.ini.j2 index 0c84aceb..4b038cdf 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/templates/django.asgi.ini.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/templates/django.asgi.ini.j2 @@ -11,7 +11,7 @@ SyslogIdentifier=gunicorn User={{ asgi_user }} Group={{ asgi_group }} WorkingDirectory={{ project_path }} -ExecStart={{ venv_path }}/bin/gunicorn -w {{ asgi_workers }} --bind unix://{{ asgi_socket }} --access-logfile {{project_log_dir}}/asgi.log --capture-output --error-logfile {{project_log_dir}}/asgi-errors.log -k uvicorn.workers.UvicornWorker asgi:application +ExecStart=poetry run gunicorn -w {{ asgi_workers }} --bind unix://{{ asgi_socket }} --access-logfile {{project_log_dir}}/asgi.log --capture-output --error-logfile {{project_log_dir}}/asgi-errors.log -k uvicorn.workers.UvicornWorker asgi:application [Install] WantedBy=multi-user.target diff --git a/{{cookiecutter.github_repository}}/pyproject.toml b/{{cookiecutter.github_repository}}/pyproject.toml index e2d19637..d285fdbc 100644 --- a/{{cookiecutter.github_repository}}/pyproject.toml +++ b/{{cookiecutter.github_repository}}/pyproject.toml @@ -138,7 +138,7 @@ django-stubs = "^1.12" bump2version = "^1.0" [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.isort] @@ -148,7 +148,7 @@ default_section = "THIRDPARTY" import_heading_firstparty = "{{ cookiecutter.project_name }} Stuff" import_heading_stdlib = "Standard Library" import_heading_thirdparty = "Third Party Stuff" -skip_glob = ["*/migrations/**", "*/venv/**", "*/docs/**"] +skip_glob = ["*/migrations/**", "*/.venv/**", "*/docs/**"] [tool.black] target_version = [ "py38", "py39" ]