Merge pull request #1213 from resource-watch/cit_002

skip ene.036.rw0 and ene.037.rw0
resource-watch · Sep 24, 2024 · 6c1d9fc · 6c1d9fc
2 parents aecce54 + 975dd46
commit 6c1d9fc
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 36 deletions.
diff --git a/upload_worldbank_data/Dockerfile b/upload_worldbank_data/Dockerfile
@@ -8,10 +8,10 @@ RUN pip install -U pip
 
 # install application libraries
 RUN pip install pandas==2.2.2
-RUN pip install boto3==1.28.9
-RUN pip install requests==2.31.0
+RUN pip install boto3==1.35.20
+RUN pip install requests==2.32.3
 RUN pip install -e git+https://github.com/resource-watch/cartosql.py.git#egg=cartosql
-RUN pip install carto==1.11.3
+RUN pip install carto==1.11.2
 RUN pip install LMIPy==0.6.2
 RUN pip install cartoframes==1.2.5
 

diff --git a/upload_worldbank_data/README.md b/upload_worldbank_data/README.md
@@ -14,6 +14,9 @@ Each time this script is run, it updates all Carto tables for these datasets and
 
 Please see the [Python script](https://github.com/resource-watch/nrt-scripts/blob/master/upload_worldbank_data/contents/main.py) for more details on this processing.
 
+###### Note: Pump Price for Diesel Fuel (ene.036.rw0) and Pump Price for Gasoline (ene.037.rw0) are no longer available on World Bank website. Marked as skipped since September 2024.
+
+
 **Adding new World Bank datasets to Resource Watch**
 
 Any time a new World Bank dataset is added to Resource Watch, it should be added using this script, if at all possible. To add a new dataset to this script, you can follow these steps:

diff --git a/upload_worldbank_data/WB_RW_dataset_names_ids.csv b/upload_worldbank_data/WB_RW_dataset_names_ids.csv
@@ -1,31 +1,31 @@
-WRI ID,Public Title,Time Slider,Dataset ID,Carto Table,Carto Column,wb_indicators,wb_units
-cit.020,Air Quality: Population Exposed to Unhealthy Levels of Fine Particulate Matter (PM2.5),Yes,6d3163f5-4e08-4830-84f1-2c5d76570a82,cit_020_nat_pop_exposed_high_levels_pm_2_5_edit,yr_data,EN.ATM.PM25.MC.ZS,% of population
-cit.025,Urban Population,Yes,d38d0d5c-31b1-47f4-9d2e-d8fba4c7d083,cit_025_urban_population_edit,yr_data,SP.URB.TOTL.IN.ZS,% of total population
-cit.028,Urban Population Living in Slums,Yes,35ce2b98-adbb-4873-b334-d7b1cc542de7,cit_028_percent_urban_slums_edit,yr_data,EN.POP.SLUM.UR.ZS,% of urban population
-com.006,"Current Account Balance (BoP, current US$)",Yes,2e31a1f3-576b-46b4-84f0-3f0cc399f887,com_006_national_current_accounts_edit,yr_data,BN.CAB.XOKA.CD,"BoP, current US$"
-com.010,"Gross Domestic Product (PPP, Current International $)",Yes,52c55378-0484-48c3-92fc-3ee94d21c716,com_010_gdp_ppp_usd_edit,yr_data,NY.GDP.MKTP.CD,current US$
-com.036,Unemployment Rate,Yes,90432312-35be-46c4-bd79-42f606d5611e,com_036_unemployment_edit,yr_data,SL.UEM.TOTL.ZS,% of total labor force
-ene.004,Percentage of Energy Consumption from Renewable Sources,Yes,bced4001-425a-4fad-8c22-8214d9340ea4,ene_004_renewable_energy_share_of_total_energy_consumption_edit,renewable_energy_share_of_total_energy_consumption_p,EG.FEC.RNEW.ZS,% of total final energy consumption
-ene.012a;ene.012b;ene.012c,Access to Electricity (National);Access to Electricity (Urban);Access to Electricity (Rural),Yes,c29aee50-a46f-4c10-a8b1-152e4821be81;1ad53858-f5da-47cb-8006-5b4aa5aad589;ebcf4c98-a07d-479b-a27c-6d0c5b444548,ene_012_electricity_access_edit,total;urban;rural,EG.ELC.ACCS.ZS;EG.ELC.ACCS.UR.ZS;EG.ELC.ACCS.RU.ZS,% of population;% of urban population;% of rural population
-ene.021a,Renewable Energy Consumption,Yes,431f7c9e-7ef5-4f00-a5dc-8845fcb3a9a9,ene_021_se4all_country_indicators_edit,yr_data,3.1_RE.CONSUMPTION,Renewable energy consumption (TJ)
-ene.028,Access to Clean Cooking Fuels,Yes,c665f519-eef9-4f67-a8bf-7e3e6dc8bfcd,ene_028_access_clean_cooking_fuels_edit,yr_data,EG.CFT.ACCS.ZS,% of population
-ene.029a,Energy Intensity,Yes,2c444596-2be3-4786-bdfc-24010f99b21e,ene_029a_energy_intensity_edit,energy_intensity,6.1_PRIMARY.ENERGY.INTENSITY,MJ/2011 USD PPP
-ene.036.rw0,Pump Price for Diesel Fuel,Yes,3b77d3d6-e2ec-42c5-a2b4-04a3ebbd90f7,ene_036_rw0_pump_price_for_diesel_fuel_edit,yr_data,EP.PMP.DESL.CD,US$ per litter
-ene.037.rw0,Pump Price for Gasoline,Yes,b8b2cdde-27ec-48c7-be63-e7cdcbec76d1,ene_037_rw0_pump_price_for_gasoline_edit,yr_data,EP.PMP.SGAS.CD,US$ per litter
-foo.043,"Agriculture, Value Added",Yes,95b013a3-389a-4367-83b7-c9d68c28c406,foo_043_agriculture_value_added_edit,yr_data,NV.AGR.TOTL.ZS,percent of GDP
-soc.008,"Gross Domestic Product Per Capita",Yes,00abb46f-34e2-4bf7-be30-1fb0b1de022f,soc_008_gdp_per_capita_edit,yr_data,NY.GDP.PCAP.PP.CD,current international $
-soc.015,Adult Literacy Rate,Yes,e8f53f73-d77c-485a-a2a6-1c47ea4aead9,soc_015_adult_literacy_rate_edit,yr_data;female_data;male_data,SE.ADT.LITR.ZS;SE.ADT.LITR.FE.ZS;SE.ADT.LITR.MA.ZS,% of population 15+;% of females 15+;% of males 15+
-soc.020,Gini Index,Yes,f8d3e79c-c3d0-4f9a-9b68-9c5ad1f025e4,soc_020_gini_edit,yr_data,SI.POV.GINI,GINI index (World Bank estimate)
-soc.029,Worldwide Governance Indicators,No,7793f46c-a48a-466f-a8ce-ca1a87b7aeed,soc_029_worldwide_governance_indicators_edit,government_effectiveness_data;regulatory_quality_data;political_stability_data;rule_of_law_data;voice_accountability_data;control_of_corruption_data,GE.EST;RQ.EST;PV.EST;RL.EST;VA.EST;CC.EST,estimate;estimate;estimate;estimate;estimate;estimate
-soc.036,Life Expectancy at Birth,Yes,8671f536-1979-4b6f-a147-70152fcb44ed,soc_036_life_expectancy_at_birth_edit,yr_data,SP.DYN.LE00.IN,years
-soc.040,Access to Improved Sanitation Facilities,No,37d04efc-0ab2-4499-a891-54dca1013c74,soc_040_improved_sanitation_edit,total_data;urban_data;rural_data,SH.STA.SMSS.ZS;SH.STA.SMSS.UR.ZS;SH.STA.SMSS.RU.ZS,% of population;% of urban pop;% of rural pop
-soc.066,Population Below Income Poverty Line ($1.90),Yes,4d7ce999-1e37-418f-b8a6-1816b29e901a,soc_066_population_below_poverty_line_edit,yr_data,SI.POV.DDAY,% of population
-soc.074,Employment in Agriculture,Yes,cc354f7f-2622-44cb-91c1-73559373de72,soc_074_employment_in_agriculture_edit,yr_data,SL.AGR.EMPL.ZS,% of population
-soc.076,National Populations,Yes,2a8ca4f7-7285-4aed-9ef5-ba1f9c4b653d,soc_076_country_population_edit,yr_data,SP.POP.TOTL,people
-soc.079,Fertility Rates,Yes,80f951a7-edd5-472a-8b46-744eb93fd7ef,soc_079_total_fertility_rate_edit,total_fertility_rate,SP.DYN.TFRT.IN,births per woman
-soc.081,National Infant Mortality,Yes,6cfd6255-609e-4922-8709-5aaad0db9fae,soc_081_mortality_rate_edit,mortality_rate,SP.DYN.IMRT.IN,deaths per 1000 live births
-soc.082,Percent of Population Using the Internet,Yes,daaa9f12-c0ef-499a-b2d8-4bceaa9b95fa,soc_082_individuals_using_the_internet_edit,yr_data,IT.NET.USER.ZS,% of population
-soc.090,Population Growth Rates,Yes,5bc62559-3ae3-472c-9517-823437539a3f,soc_090_pop_growth_rate_edit,yr_data,SP.POP.GROW,percent
-soc.101,Population in Poverty ($3.20/day),Yes,8711d841-8421-4fcb-8ee2-881e573856e0,soc_101_poverty320_edit,yr_data,SI.POV.LMIC,% of population
-soc.102,Inflation of Consumer Prices,Yes,4996cdb1-6169-46b7-9a74-0adf0e2ef1ce,soc_102_inflation_edit,yr_data,FP.CPI.TOTL.ZG,annual %
-wat.005a;wat.005b;wat.005c,Access to an Improved Water Source (National);Access to an Improved Water Source (Urban);Access to an Improved Water Source (Rural),Yes,e32bdaba-5a72-487c-b640-668adaea2976;c29473e4-6606-49c9-a248-df1f2b21c2b5;3598f43d-0ae9-445b-a8da-e59ff2cf43d9,wat_005a_improved_water_access_edit,total;urban;rural,SH.H2O.SMDW.ZS;SH.H2O.SMDW.UR.ZS;SH.H2O.SMDW.RU.ZS,% of population;% of urban pop;% of rural pop
+WRI ID,Public Title,Time Slider,Dataset ID,Carto Table,Carto Column,wb_indicators,wb_units,skip
+cit.020,Air Quality: Population Exposed to Unhealthy Levels of Fine Particulate Matter (PM2.5),Yes,6d3163f5-4e08-4830-84f1-2c5d76570a82,cit_020_nat_pop_exposed_high_levels_pm_2_5_edit,yr_data,EN.ATM.PM25.MC.ZS,% of population,
+cit.025,Urban Population,Yes,d38d0d5c-31b1-47f4-9d2e-d8fba4c7d083,cit_025_urban_population_edit,yr_data,SP.URB.TOTL.IN.ZS,% of total population,
+cit.028,Urban Population Living in Slums,Yes,35ce2b98-adbb-4873-b334-d7b1cc542de7,cit_028_percent_urban_slums_edit,yr_data,EN.POP.SLUM.UR.ZS,% of urban population,
+com.006,"Current Account Balance (BoP, current US$)",Yes,2e31a1f3-576b-46b4-84f0-3f0cc399f887,com_006_national_current_accounts_edit,yr_data,BN.CAB.XOKA.CD,"BoP, current US$",
+com.010,"Gross Domestic Product (PPP, Current International $)",Yes,52c55378-0484-48c3-92fc-3ee94d21c716,com_010_gdp_ppp_usd_edit,yr_data,NY.GDP.MKTP.CD,current US$,
+com.036,Unemployment Rate,Yes,90432312-35be-46c4-bd79-42f606d5611e,com_036_unemployment_edit,yr_data,SL.UEM.TOTL.ZS,% of total labor force,
+ene.004,Percentage of Energy Consumption from Renewable Sources,Yes,bced4001-425a-4fad-8c22-8214d9340ea4,ene_004_renewable_energy_share_of_total_energy_consumption_edit,renewable_energy_share_of_total_energy_consumption_p,EG.FEC.RNEW.ZS,% of total final energy consumption,
+ene.012a;ene.012b;ene.012c,Access to Electricity (National);Access to Electricity (Urban);Access to Electricity (Rural),Yes,c29aee50-a46f-4c10-a8b1-152e4821be81;1ad53858-f5da-47cb-8006-5b4aa5aad589;ebcf4c98-a07d-479b-a27c-6d0c5b444548,ene_012_electricity_access_edit,total;urban;rural,EG.ELC.ACCS.ZS;EG.ELC.ACCS.UR.ZS;EG.ELC.ACCS.RU.ZS,% of population;% of urban population;% of rural population,
+ene.021a,Renewable Energy Consumption,Yes,431f7c9e-7ef5-4f00-a5dc-8845fcb3a9a9,ene_021_se4all_country_indicators_edit,yr_data,3.1_RE.CONSUMPTION,Renewable energy consumption (TJ),
+ene.028,Access to Clean Cooking Fuels,Yes,c665f519-eef9-4f67-a8bf-7e3e6dc8bfcd,ene_028_access_clean_cooking_fuels_edit,yr_data,EG.CFT.ACCS.ZS,% of population,
+ene.029a,Energy Intensity,Yes,2c444596-2be3-4786-bdfc-24010f99b21e,ene_029a_energy_intensity_edit,energy_intensity,6.1_PRIMARY.ENERGY.INTENSITY,MJ/2011 USD PPP,
+ene.036.rw0,Pump Price for Diesel Fuel,Yes,3b77d3d6-e2ec-42c5-a2b4-04a3ebbd90f7,ene_036_rw0_pump_price_for_diesel_fuel_edit,yr_data,EP.PMP.DESL.CD,US$ per litter,TRUE
+ene.037.rw0,Pump Price for Gasoline,Yes,b8b2cdde-27ec-48c7-be63-e7cdcbec76d1,ene_037_rw0_pump_price_for_gasoline_edit,yr_data,EP.PMP.SGAS.CD,US$ per litter,TRUE
+foo.043,"Agriculture, Value Added",Yes,95b013a3-389a-4367-83b7-c9d68c28c406,foo_043_agriculture_value_added_edit,yr_data,NV.AGR.TOTL.ZS,percent of GDP,
+soc.008,Gross Domestic Product Per Capita,Yes,00abb46f-34e2-4bf7-be30-1fb0b1de022f,soc_008_gdp_per_capita_edit,yr_data,NY.GDP.PCAP.PP.CD,current international $,
+soc.015,Adult Literacy Rate,Yes,e8f53f73-d77c-485a-a2a6-1c47ea4aead9,soc_015_adult_literacy_rate_edit,yr_data;female_data;male_data,SE.ADT.LITR.ZS;SE.ADT.LITR.FE.ZS;SE.ADT.LITR.MA.ZS,% of population 15+;% of females 15+;% of males 15+,
+soc.020,Gini Index,Yes,f8d3e79c-c3d0-4f9a-9b68-9c5ad1f025e4,soc_020_gini_edit,yr_data,SI.POV.GINI,GINI index (World Bank estimate),
+soc.029,Worldwide Governance Indicators,No,7793f46c-a48a-466f-a8ce-ca1a87b7aeed,soc_029_worldwide_governance_indicators_edit,government_effectiveness_data;regulatory_quality_data;political_stability_data;rule_of_law_data;voice_accountability_data;control_of_corruption_data,GE.EST;RQ.EST;PV.EST;RL.EST;VA.EST;CC.EST,estimate;estimate;estimate;estimate;estimate;estimate,
+soc.036,Life Expectancy at Birth,Yes,8671f536-1979-4b6f-a147-70152fcb44ed,soc_036_life_expectancy_at_birth_edit,yr_data,SP.DYN.LE00.IN,years,
+soc.040,Access to Improved Sanitation Facilities,No,37d04efc-0ab2-4499-a891-54dca1013c74,soc_040_improved_sanitation_edit,total_data;urban_data;rural_data,SH.STA.SMSS.ZS;SH.STA.SMSS.UR.ZS;SH.STA.SMSS.RU.ZS,% of population;% of urban pop;% of rural pop,
+soc.066,Population Below Income Poverty Line ($1.90),Yes,4d7ce999-1e37-418f-b8a6-1816b29e901a,soc_066_population_below_poverty_line_edit,yr_data,SI.POV.DDAY,% of population,
+soc.074,Employment in Agriculture,Yes,cc354f7f-2622-44cb-91c1-73559373de72,soc_074_employment_in_agriculture_edit,yr_data,SL.AGR.EMPL.ZS,% of population,
+soc.076,National Populations,Yes,2a8ca4f7-7285-4aed-9ef5-ba1f9c4b653d,soc_076_country_population_edit,yr_data,SP.POP.TOTL,people,
+soc.079,Fertility Rates,Yes,80f951a7-edd5-472a-8b46-744eb93fd7ef,soc_079_total_fertility_rate_edit,total_fertility_rate,SP.DYN.TFRT.IN,births per woman,
+soc.081,National Infant Mortality,Yes,6cfd6255-609e-4922-8709-5aaad0db9fae,soc_081_mortality_rate_edit,mortality_rate,SP.DYN.IMRT.IN,deaths per 1000 live births,
+soc.082,Percent of Population Using the Internet,Yes,daaa9f12-c0ef-499a-b2d8-4bceaa9b95fa,soc_082_individuals_using_the_internet_edit,yr_data,IT.NET.USER.ZS,% of population,
+soc.090,Population Growth Rates,Yes,5bc62559-3ae3-472c-9517-823437539a3f,soc_090_pop_growth_rate_edit,yr_data,SP.POP.GROW,percent,
+soc.101,Population in Poverty ($3.20/day),Yes,8711d841-8421-4fcb-8ee2-881e573856e0,soc_101_poverty320_edit,yr_data,SI.POV.LMIC,% of population,
+soc.102,Inflation of Consumer Prices,Yes,4996cdb1-6169-46b7-9a74-0adf0e2ef1ce,soc_102_inflation_edit,yr_data,FP.CPI.TOTL.ZG,annual %,
+wat.005a;wat.005b;wat.005c,Access to an Improved Water Source (National);Access to an Improved Water Source (Urban);Access to an Improved Water Source (Rural),Yes,e32bdaba-5a72-487c-b640-668adaea2976;c29473e4-6606-49c9-a248-df1f2b21c2b5;3598f43d-0ae9-445b-a8da-e59ff2cf43d9,wat_005a_improved_water_access_edit,total;urban;rural,SH.H2O.SMDW.ZS;SH.H2O.SMDW.UR.ZS;SH.H2O.SMDW.RU.ZS,% of population;% of urban pop;% of rural pop,
diff --git a/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py b/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py
@@ -135,7 +135,7 @@ def fetch_wb_data(table):
             try:
                 # fetch data for this indicator (only the first 10,000 entries will be returned)
                 res = requests.get(
-                    "https://api.worldbank.org/v2/countries/all/indicators/{}?format=json&per_page=10000".format(indicator))
+                    f"https://api.worldbank.org/v2/countries/all/indicators/{indicator}?format=json&per_page=10000")
                 # check how many pages of data there are for this indicator
                 pages = int(res.json()[0]['pages'])
                 break
@@ -253,6 +253,8 @@ def main():
     if not os.path.exists(data_dir):
         os.mkdir(data_dir)
 
+    wb_rw_table = wb_rw_table[wb_rw_table['skip']!=True]
+
     # process each Carto table for World Bank datasets one at a time
     for table_name, info in wb_rw_table.iterrows():
         # get the dataset name (table name without the '_edit' at the end of the table_name
@@ -370,4 +372,5 @@ def main():
     # Delete local files in Docker container
     delete_local()
 
-    logging.info('SUCCESS')
+    logging.info('SUCCESS')
+