From ad7ce15b944ec134912a2c34d7f715ac701cef81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Tue, 21 Feb 2023 21:04:16 +0000 Subject: [PATCH] Make postgres database configurable (#335) --- avancement_cumul.sh | 5 ++++- bano/db.py | 4 ++-- bano/export.py | 2 +- config | 17 +++++++++++++++++ copy_table_from_osm_to_cadastre.sh | 6 +++--- create_base.sh | 8 +++++--- cron_bano.sh | 14 +++++++------- init_base.sh | 10 ++++++---- load_COG.sh | 14 +++++++------- load_code_cadastre.sh | 4 +++- load_codes_postaux.sh | 4 ++-- load_fantoir.sh | 4 ++-- load_osm_france_db.sh | 4 ++-- stats.sh | 4 +++- update_bases_adresses_locales.sh | 4 ++-- update_cadastre_adresses.sh | 4 ++-- update_cadastre_batiments.sh | 4 ++-- update_cadastre_lieux-dits.sh | 2 +- update_table_infos_communes.sh | 2 +- utils/load_csv_export.sh | 7 +++++-- 20 files changed, 77 insertions(+), 46 deletions(-) diff --git a/avancement_cumul.sh b/avancement_cumul.sh index 33fa993..4507aaa 100755 --- a/avancement_cumul.sh +++ b/avancement_cumul.sh @@ -2,7 +2,10 @@ set -e -psql -d cadastre -U cadastre -c "SELECT source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPT_DIR/config + +$pgsql_CADASTRE -c "SELECT source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;" ps -eaf|grep 'bano' ps -eaf|grep cumul diff --git a/bano/db.py b/bano/db.py index 79f08c3..30b7b37 100644 --- a/bano/db.py +++ b/bano/db.py @@ -3,7 +3,7 @@ import psycopg2 import psycopg2.extras -bano = psycopg2.connect(os.environ.get("BANO_PG", "dbname='cadastre' user='cadastre'")) -bano_cache = psycopg2.connect(os.environ.get("BANO_PG_CACHE", "dbname='osm' user='cadastre'")) +bano = psycopg2.connect(os.environ.get("PG_CADASTRE", "dbname='cadastre' user='cadastre'")) +bano_cache = psycopg2.connect(os.environ.get("PG_OSM", "dbname='osm' user='cadastre'")) bano_cache.autocommit = True psycopg2.extras.register_hstore(bano_cache) diff --git a/bano/export.py b/bano/export.py index 93bd0df..d8c8ea3 100644 --- a/bano/export.py +++ b/bano/export.py @@ -146,7 +146,7 @@ def save_as_ttl(self): def save_as_shp(self): - subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', self.get_sas_full_filename('shp'), 'PG:dbname=cadastre user=cadastre', '-sql', f'{self.csv_query}']) + subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', self.get_sas_full_filename('shp'), 'PG:' + os.environ['PG_CADASTRE'], '-sql', f'{self.csv_query}']) def save_as_json(self): with open(self.get_sas_full_filename('json'),'w') as jsonfile: diff --git a/config b/config index c18fabc..48a13a7 100644 --- a/config +++ b/config @@ -16,3 +16,20 @@ export EXPORT_WEB_DIR=${EXPORT_WEB_DIR:-/data/work/bano.openstreetmap.fr/www/web export TMP_DIR=${TMP_DIR:-/data/tmp} export SCHEMA_CIBLE=public export LANG=${LANG:-fr_FR.UTF-8} + + +POSTGRES_USER=${POSTGRES_USER:-cadastre} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-} +POSTGRES_HOST=${POSTGRES_HOST:-localhost} +POSTGRES_PORT=${POSTGRES_PORT:-5432} + +POSTGRES_DB_OSM=${POSTGRES_DB:-osm} +POSTGRES_DB_CADASTRE=${POSTGRES_DB:-cadastre} + +export PGCON_OSM="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_OSM" +export pgsql_OSM="psql postgresql://${PGCON_OSM} -v ON_ERROR_STOP=1" +export PG_OSM="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_OSM' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}" + +export PGCON_CADASTRE="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_CADASTRE" +export pgsql_CADASTRE="psql postgresql://${PGCON_CADASTRE} -v ON_ERROR_STOP=1" +export PG_CADASTRE="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_CADASTRE' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}" diff --git a/copy_table_from_osm_to_cadastre.sh b/copy_table_from_osm_to_cadastre.sh index 12c7721..e9bf796 100755 --- a/copy_table_from_osm_to_cadastre.sh +++ b/copy_table_from_osm_to_cadastre.sh @@ -8,7 +8,7 @@ source $SCRIPT_DIR/config set -e cd $TMP_DIR -pg_dump -t ${1} -U cadastre osm > ${1}_dump.sql -psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS ${1} CASCADE;" -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f ${1}_dump.sql +pg_dump -t ${1} "postgresql://${PGCON_OSM}" > ${1}_dump.sql +$pgsql_CADASTRE -c "DROP TABLE IF EXISTS ${1} CASCADE;" +$pgsql_CADASTRE -f ${1}_dump.sql rm -f ${1}_dump.sql diff --git a/create_base.sh b/create_base.sh index 11bb8bb..e9a017b 100755 --- a/create_base.sh +++ b/create_base.sh @@ -2,6 +2,8 @@ set -e -createuser -s -d cadastre -createdb -E UTF-8 -T template0 -O cadastre cadastre -createdb -E UTF-8 -T template0 -O cadastre osm +source config + +createuser -s -d $POSTGRES_USER +createdb -E UTF-8 -T template0 -O $POSTGRES_USER $POSTGRES_DB_CADASTRE +createdb -E UTF-8 -T template0 -O $POSTGRES_USER $POSTGRES_DB_OSM diff --git a/cron_bano.sh b/cron_bano.sh index a50d83c..60fdee0 100755 --- a/cron_bano.sh +++ b/cron_bano.sh @@ -48,14 +48,14 @@ fi ./copy_table_from_osm_to_cadastre.sh planet_osm_postal_code ./copy_table_from_osm_to_cadastre.sh infos_communes -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_polygones_communes.sql +$pgsql_OSM -f sql/create_table_polygones_communes.sql ./copy_table_from_osm_to_cadastre.sh polygones_insee ./copy_table_from_osm_to_cadastre.sh polygones_insee_geo -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_polygones_postaux.sql +$pgsql_OSM -f sql/create_table_polygones_postaux.sql ./copy_table_from_osm_to_cadastre.sh polygones_postaux ./copy_table_from_osm_to_cadastre.sh ban_odbl -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f sql/post_copie_ban_odbl.sql +$pgsql_CADASTRE -f sql/post_copie_ban_odbl.sql # exports cat deplist.txt | parallel -j 4 bano export {1} @@ -65,7 +65,7 @@ cat deplist.txt | parallel -j 4 bano publish {1} bano publish_aggregate # ménage PostgreSQL -psql -d cadastre -U cadastre -c "VACUUM cumul_adresses;" -psql -d cadastre -U cadastre -c "VACUUM cumul_voies;" -psql -d cadastre -U cadastre -c "VACUUM cumul_places;" -psql -d cadastre -U cadastre -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC"; +$pgsql_CADASTRE -c "VACUUM cumul_adresses;" +$pgsql_CADASTRE -c "VACUUM cumul_voies;" +$pgsql_CADASTRE -c "VACUUM cumul_places;" +$pgsql_CADASTRE -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC"; diff --git a/init_base.sh b/init_base.sh index 2230636..cf4e9e0 100755 --- a/init_base.sh +++ b/init_base.sh @@ -2,7 +2,9 @@ set -e -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/create_base.sql -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_base_bano.sql -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/create_base.sql -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f sql/create_table_base_osm.sql +source config + +$pgsql_CADASTRE -f bano/sql/create_base.sql +$pgsql_CADASTRE -f sql/create_table_base_bano.sql +$pgsql_OSM -f bano/sql/create_base.sql +$pgsql_OSM -f sql/create_table_base_osm.sql diff --git a/load_COG.sh b/load_COG.sh index b8f2691..c1e876d 100755 --- a/load_COG.sh +++ b/load_COG.sh @@ -16,12 +16,12 @@ CSV_ARRONDISSEMENT=arrondissement2021.csv CSV_DEPARTEMENT=departement2021.csv CSV_REGION=region2021.csv -for d in osm cadastre +for psql in "$pgsql_CADASTRE" "$pgsql_OSM" do - psql -d ${d} -U cadastre -f $BANO_DIR/sql/create_table_cog.sql - cat ${CSV_COMMUNE}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_commune; COPY cog_commune FROM stdin WITH csv header delimiter ',';" - cat ${CSV_CANTON}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_canton; COPY cog_canton FROM stdin WITH csv header delimiter ',';" - cat ${CSV_ARRONDISSEMENT}|psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_arrondissement;COPY cog_arrondissement FROM stdin WITH csv header delimiter ',';" - cat ${CSV_DEPARTEMENT}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_departement; COPY cog_departement FROM stdin WITH csv header delimiter ',';" - cat ${CSV_REGION}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_region; COPY cog_region FROM stdin WITH csv header delimiter ',';" + $psql -f $BANO_DIR/sql/create_table_cog.sql + cat "${CSV_COMMUNE}" | $psql -c "TRUNCATE TABLE cog_commune; COPY cog_commune FROM stdin WITH csv header delimiter ',';" + cat "${CSV_CANTON}" | $psql -c "TRUNCATE TABLE cog_canton; COPY cog_canton FROM stdin WITH csv header delimiter ',';" + cat "${CSV_ARRONDISSEMENT}" | $psql -c "TRUNCATE TABLE cog_arrondissement;COPY cog_arrondissement FROM stdin WITH csv header delimiter ',';" + cat "${CSV_DEPARTEMENT}" | $psql -c "TRUNCATE TABLE cog_departement; COPY cog_departement FROM stdin WITH csv header delimiter ',';" + cat "${CSV_REGION}" | $psql -c "TRUNCATE TABLE cog_region; COPY cog_region FROM stdin WITH csv header delimiter ',';" done diff --git a/load_code_cadastre.sh b/load_code_cadastre.sh index ed3059f..a0aaa00 100755 --- a/load_code_cadastre.sh +++ b/load_code_cadastre.sh @@ -2,5 +2,7 @@ set -e +source config + python load_code_cadastre.py -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f bano/sql/load_code_cadastre.sql +$pgsql_CADASTRE -f bano/sql/load_code_cadastre.sql diff --git a/load_codes_postaux.sh b/load_codes_postaux.sh index ab0a2dd..b70736c 100755 --- a/load_codes_postaux.sh +++ b/load_codes_postaux.sh @@ -8,5 +8,5 @@ mkdir -p codes_postaux cd codes_postaux rm -f *csv* wget "https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&use_labels_for_header=true&csv_separator=%3B" -O codes_postaux.csv -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/create_and_load_codes_postaux.sql -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/create_and_load_codes_postaux.sql +$pgsql_OSM -f $BANO_DIR/sql/create_and_load_codes_postaux.sql +$pgsql_CADASTRE -f $BANO_DIR/sql/create_and_load_codes_postaux.sql diff --git a/load_fantoir.sh b/load_fantoir.sh index ed09682..a9e7abb 100755 --- a/load_fantoir.sh +++ b/load_fantoir.sh @@ -10,5 +10,5 @@ rm -rf FANTOIR* && rm -f *.zip wget https://data.economie.gouv.fr/api/datasets/1.0/fichier-fantoir-des-voies-et-lieux-dits/attachments/fichier_national_fantoir_situation_janvier_2023_zip/ -O fantoir.zip unzip fantoir.zip mv FANTOIR* fantoir.txt -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/load_fantoir.sql -psql -d cadastre -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/load_fantoir.sql +$pgsql_OSM -f $BANO_DIR/sql/load_fantoir.sql +$pgsql_CADASTRE -f $BANO_DIR/sql/load_fantoir.sql diff --git a/load_osm_france_db.sh b/load_osm_france_db.sh index f7e9087..36e0c97 100755 --- a/load_osm_france_db.sh +++ b/load_osm_france_db.sh @@ -22,9 +22,9 @@ mkdir -p $DOWNLOAD_DIR cd $DOWNLOAD_DIR wget -NS $PBF_URL imposm import -mapping $BANO_DIR/bano.yml -read $DOWNLOAD_DIR/$PBF_FILE -overwritecache -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR -imposm import -mapping $BANO_DIR/bano.yml -write -connection 'postgis://cadastre@localhost/osm'?prefix=NONE -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR +imposm import -mapping $BANO_DIR/bano.yml -write -connection "postgis://$PGCON_OSM"?prefix=NONE -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/finalisation.sql +$pgsql_OSM -f $BANO_DIR/sql/finalisation.sql cp $DOWNLOAD_DIR/last.state.txt $DOWNLOAD_DIR/state.txt rm ${lockfile} diff --git a/stats.sh b/stats.sh index ed5a5a3..bf30c9d 100755 --- a/stats.sh +++ b/stats.sh @@ -2,4 +2,6 @@ set -e -psql -d cadastre -v ON_ERROR_STOP=1 -f stats.sql -v dept=$1 +source config + +$pgsql_CADASTRE -f stats.sql -v dept=$1 diff --git a/update_bases_adresses_locales.sh b/update_bases_adresses_locales.sh index b9edb11..c67659e 100755 --- a/update_bases_adresses_locales.sh +++ b/update_bases_adresses_locales.sh @@ -12,7 +12,7 @@ do echo 'Département '$dep wget -NS https://adresse.data.gouv.fr/data/adresses-locales/latest/csv/adresses-locales-$dep.csv.gz quoted_dep=\'$dep'%'\' - psql -d osm -U cadastre -c "DELETE FROM bal_locales WHERE commune_code LIKE $quoted_dep;" - gunzip -c $BAL_CACHE_DIR/adresses-locales-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_locales from stdin with csv header delimiter ';'" + $pgsql_OSM -c "DELETE FROM bal_locales WHERE commune_code LIKE $quoted_dep;" + gunzip -c $BAL_CACHE_DIR/adresses-locales-$dep.csv.gz| $pgsql_OSM -c "\copy bal_locales from stdin with csv header delimiter ';'" sleep 1 done diff --git a/update_cadastre_adresses.sh b/update_cadastre_adresses.sh index a844801..b219b0e 100755 --- a/update_cadastre_adresses.sh +++ b/update_cadastre_adresses.sh @@ -10,7 +10,7 @@ do echo 'Département '$dep wget -NS https://adresse.data.gouv.fr/data/adresses-cadastre/latest/csv/adresses-cadastre-$dep.csv.gz quoted_dep=\'$dep'%'\' - psql -d osm -U cadastre -c "DELETE FROM bal_cadastre WHERE commune_code LIKE $quoted_dep;" - gunzip -c $CADASTRE_CACHE_DIR/adresses-cadastre-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_cadastre from stdin with csv header delimiter ';'" + $pgsql_OSM -c "DELETE FROM bal_cadastre WHERE commune_code LIKE $quoted_dep;" + gunzip -c $CADASTRE_CACHE_DIR/adresses-cadastre-$dep.csv.gz| $pgsql_OSM -c "\copy bal_cadastre from stdin with csv header delimiter ';'" sleep 1 done diff --git a/update_cadastre_batiments.sh b/update_cadastre_batiments.sh index 1fab347..2cd73de 100755 --- a/update_cadastre_batiments.sh +++ b/update_cadastre_batiments.sh @@ -17,8 +17,8 @@ do unzip $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip # reconstruction du shapefile avec ogr2ogr car corrompu pour shp2pgsql ogr2ogr -overwrite -f 'ESRI Shapefile' batiments_ok.shp batiments.shp - shp2pgsql -s 2154:4326 -g geometrie -W LATIN1 batiments_ok.shp public.tmp_batiments$dep | psql -d cadastre -q - psql -d cadastre -v ON_ERROR_STOP=1 -f $BANO_DIR/sql/replace_batiments.sql -v schema_cible=$SCHEMA_CIBLE -v dept=$dep + shp2pgsql -s 2154:4326 -g geometrie -W LATIN1 batiments_ok.shp public.tmp_batiments$dep | $pgsql_CADASTRE -q + $pgsql_CADASTRE -f $BANO_DIR/sql/replace_batiments.sql -v schema_cible=$SCHEMA_CIBLE -v dept=$dep zip -mT $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip batiments.* sleep 1 cd .. diff --git a/update_cadastre_lieux-dits.sh b/update_cadastre_lieux-dits.sh index b32f4b3..b046ca8 100755 --- a/update_cadastre_lieux-dits.sh +++ b/update_cadastre_lieux-dits.sh @@ -25,7 +25,7 @@ do gzip -dfk ${ZIPFILE} mv ${JSONFILE} lieux_dits.json # ls -al - psql -d osm -U cadastre -c "DELETE FROM lieux_dits WHERE insee_com LIKE '${DEPT}%';" + $pgsql_OSM -c "DELETE FROM lieux_dits WHERE insee_com LIKE '${DEPT}%';" ogr2ogr -append -nln lieux_dits -fieldmap 1,0,2,3 -f PostgreSQL PG:'user=cadastre dbname=osm' lieux_dits.json rm lieux_dits.json else diff --git a/update_table_infos_communes.sh b/update_table_infos_communes.sh index 296ba81..4ebc5f7 100755 --- a/update_table_infos_communes.sh +++ b/update_table_infos_communes.sh @@ -5,4 +5,4 @@ set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPT_DIR -psql -d osm -U cadastre -v ON_ERROR_STOP=1 -f ./sql/update_table_infos_communes.sql +$pgsql_OSM -f ./sql/update_table_infos_communes.sql diff --git a/utils/load_csv_export.sh b/utils/load_csv_export.sh index b28e252..294d130 100755 --- a/utils/load_csv_export.sh +++ b/utils/load_csv_export.sh @@ -2,7 +2,10 @@ set -e -psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS export_csv CASCADE; +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $SCRIPT_DIR/config + +$pgsql_CADASTRE -c "DROP TABLE IF EXISTS export_csv CASCADE; CREATE TABLE export_csv( id text, numero text, @@ -15,5 +18,5 @@ psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS export_csv CASCADE; for dep in {01..19} 2A 2B {21..95} {971..974} 976 do - cat /data/sas_web/bano-${dep}.csv|psql -d cadastre -U cadastre -c "COPY export_csv FROM STDIN WITH CSV" + cat /data/sas_web/bano-${dep}.csv| $pgsql_CADASTRE -c "COPY export_csv FROM STDIN WITH CSV" done