From 975dd464401e903541b323f6145201e3271908db Mon Sep 17 00:00:00 2001 From: weiqi-tori Date: Tue, 24 Sep 2024 18:44:45 +0800 Subject: [PATCH] skip ene.036.rw0 and ene.037.rw0 --- upload_worldbank_data/Dockerfile | 6 +- upload_worldbank_data/README.md | 3 + .../WB_RW_dataset_names_ids.csv | 62 +++++++++---------- .../src/update_worldbank_data_on_carto.py | 7 ++- 4 files changed, 42 insertions(+), 36 deletions(-) diff --git a/upload_worldbank_data/Dockerfile b/upload_worldbank_data/Dockerfile index 06037be9..24371566 100644 --- a/upload_worldbank_data/Dockerfile +++ b/upload_worldbank_data/Dockerfile @@ -8,10 +8,10 @@ RUN pip install -U pip # install application libraries RUN pip install pandas==2.2.2 -RUN pip install boto3==1.28.9 -RUN pip install requests==2.31.0 +RUN pip install boto3==1.35.20 +RUN pip install requests==2.32.3 RUN pip install -e git+https://github.com/resource-watch/cartosql.py.git#egg=cartosql -RUN pip install carto==1.11.3 +RUN pip install carto==1.11.2 RUN pip install LMIPy==0.6.2 RUN pip install cartoframes==1.2.5 diff --git a/upload_worldbank_data/README.md b/upload_worldbank_data/README.md index a0908582..ffe90e26 100644 --- a/upload_worldbank_data/README.md +++ b/upload_worldbank_data/README.md @@ -14,6 +14,9 @@ Each time this script is run, it updates all Carto tables for these datasets and Please see the [Python script](https://github.com/resource-watch/nrt-scripts/blob/master/upload_worldbank_data/contents/main.py) for more details on this processing. +###### Note: Pump Price for Diesel Fuel (ene.036.rw0) and Pump Price for Gasoline (ene.037.rw0) are no longer available on World Bank website. Marked as skipped since September 2024. + + **Adding new World Bank datasets to Resource Watch** Any time a new World Bank dataset is added to Resource Watch, it should be added using this script, if at all possible. To add a new dataset to this script, you can follow these steps: diff --git a/upload_worldbank_data/WB_RW_dataset_names_ids.csv b/upload_worldbank_data/WB_RW_dataset_names_ids.csv index 5875ffbe..17befec1 100644 --- a/upload_worldbank_data/WB_RW_dataset_names_ids.csv +++ b/upload_worldbank_data/WB_RW_dataset_names_ids.csv @@ -1,31 +1,31 @@ -WRI ID,Public Title,Time Slider,Dataset ID,Carto Table,Carto Column,wb_indicators,wb_units -cit.020,Air Quality: Population Exposed to Unhealthy Levels of Fine Particulate Matter (PM2.5),Yes,6d3163f5-4e08-4830-84f1-2c5d76570a82,cit_020_nat_pop_exposed_high_levels_pm_2_5_edit,yr_data,EN.ATM.PM25.MC.ZS,% of population -cit.025,Urban Population,Yes,d38d0d5c-31b1-47f4-9d2e-d8fba4c7d083,cit_025_urban_population_edit,yr_data,SP.URB.TOTL.IN.ZS,% of total population -cit.028,Urban Population Living in Slums,Yes,35ce2b98-adbb-4873-b334-d7b1cc542de7,cit_028_percent_urban_slums_edit,yr_data,EN.POP.SLUM.UR.ZS,% of urban population -com.006,"Current Account Balance (BoP, current US$)",Yes,2e31a1f3-576b-46b4-84f0-3f0cc399f887,com_006_national_current_accounts_edit,yr_data,BN.CAB.XOKA.CD,"BoP, current US$" -com.010,"Gross Domestic Product (PPP, Current International $)",Yes,52c55378-0484-48c3-92fc-3ee94d21c716,com_010_gdp_ppp_usd_edit,yr_data,NY.GDP.MKTP.CD,current US$ -com.036,Unemployment Rate,Yes,90432312-35be-46c4-bd79-42f606d5611e,com_036_unemployment_edit,yr_data,SL.UEM.TOTL.ZS,% of total labor force -ene.004,Percentage of Energy Consumption from Renewable Sources,Yes,bced4001-425a-4fad-8c22-8214d9340ea4,ene_004_renewable_energy_share_of_total_energy_consumption_edit,renewable_energy_share_of_total_energy_consumption_p,EG.FEC.RNEW.ZS,% of total final energy consumption -ene.012a;ene.012b;ene.012c,Access to Electricity (National);Access to Electricity (Urban);Access to Electricity (Rural),Yes,c29aee50-a46f-4c10-a8b1-152e4821be81;1ad53858-f5da-47cb-8006-5b4aa5aad589;ebcf4c98-a07d-479b-a27c-6d0c5b444548,ene_012_electricity_access_edit,total;urban;rural,EG.ELC.ACCS.ZS;EG.ELC.ACCS.UR.ZS;EG.ELC.ACCS.RU.ZS,% of population;% of urban population;% of rural population -ene.021a,Renewable Energy Consumption,Yes,431f7c9e-7ef5-4f00-a5dc-8845fcb3a9a9,ene_021_se4all_country_indicators_edit,yr_data,3.1_RE.CONSUMPTION,Renewable energy consumption (TJ) -ene.028,Access to Clean Cooking Fuels,Yes,c665f519-eef9-4f67-a8bf-7e3e6dc8bfcd,ene_028_access_clean_cooking_fuels_edit,yr_data,EG.CFT.ACCS.ZS,% of population -ene.029a,Energy Intensity,Yes,2c444596-2be3-4786-bdfc-24010f99b21e,ene_029a_energy_intensity_edit,energy_intensity,6.1_PRIMARY.ENERGY.INTENSITY,MJ/2011 USD PPP -ene.036.rw0,Pump Price for Diesel Fuel,Yes,3b77d3d6-e2ec-42c5-a2b4-04a3ebbd90f7,ene_036_rw0_pump_price_for_diesel_fuel_edit,yr_data,EP.PMP.DESL.CD,US$ per litter -ene.037.rw0,Pump Price for Gasoline,Yes,b8b2cdde-27ec-48c7-be63-e7cdcbec76d1,ene_037_rw0_pump_price_for_gasoline_edit,yr_data,EP.PMP.SGAS.CD,US$ per litter -foo.043,"Agriculture, Value Added",Yes,95b013a3-389a-4367-83b7-c9d68c28c406,foo_043_agriculture_value_added_edit,yr_data,NV.AGR.TOTL.ZS,percent of GDP -soc.008,"Gross Domestic Product Per Capita",Yes,00abb46f-34e2-4bf7-be30-1fb0b1de022f,soc_008_gdp_per_capita_edit,yr_data,NY.GDP.PCAP.PP.CD,current international $ -soc.015,Adult Literacy Rate,Yes,e8f53f73-d77c-485a-a2a6-1c47ea4aead9,soc_015_adult_literacy_rate_edit,yr_data;female_data;male_data,SE.ADT.LITR.ZS;SE.ADT.LITR.FE.ZS;SE.ADT.LITR.MA.ZS,% of population 15+;% of females 15+;% of males 15+ -soc.020,Gini Index,Yes,f8d3e79c-c3d0-4f9a-9b68-9c5ad1f025e4,soc_020_gini_edit,yr_data,SI.POV.GINI,GINI index (World Bank estimate) -soc.029,Worldwide Governance Indicators,No,7793f46c-a48a-466f-a8ce-ca1a87b7aeed,soc_029_worldwide_governance_indicators_edit,government_effectiveness_data;regulatory_quality_data;political_stability_data;rule_of_law_data;voice_accountability_data;control_of_corruption_data,GE.EST;RQ.EST;PV.EST;RL.EST;VA.EST;CC.EST,estimate;estimate;estimate;estimate;estimate;estimate -soc.036,Life Expectancy at Birth,Yes,8671f536-1979-4b6f-a147-70152fcb44ed,soc_036_life_expectancy_at_birth_edit,yr_data,SP.DYN.LE00.IN,years -soc.040,Access to Improved Sanitation Facilities,No,37d04efc-0ab2-4499-a891-54dca1013c74,soc_040_improved_sanitation_edit,total_data;urban_data;rural_data,SH.STA.SMSS.ZS;SH.STA.SMSS.UR.ZS;SH.STA.SMSS.RU.ZS,% of population;% of urban pop;% of rural pop -soc.066,Population Below Income Poverty Line ($1.90),Yes,4d7ce999-1e37-418f-b8a6-1816b29e901a,soc_066_population_below_poverty_line_edit,yr_data,SI.POV.DDAY,% of population -soc.074,Employment in Agriculture,Yes,cc354f7f-2622-44cb-91c1-73559373de72,soc_074_employment_in_agriculture_edit,yr_data,SL.AGR.EMPL.ZS,% of population -soc.076,National Populations,Yes,2a8ca4f7-7285-4aed-9ef5-ba1f9c4b653d,soc_076_country_population_edit,yr_data,SP.POP.TOTL,people -soc.079,Fertility Rates,Yes,80f951a7-edd5-472a-8b46-744eb93fd7ef,soc_079_total_fertility_rate_edit,total_fertility_rate,SP.DYN.TFRT.IN,births per woman -soc.081,National Infant Mortality,Yes,6cfd6255-609e-4922-8709-5aaad0db9fae,soc_081_mortality_rate_edit,mortality_rate,SP.DYN.IMRT.IN,deaths per 1000 live births -soc.082,Percent of Population Using the Internet,Yes,daaa9f12-c0ef-499a-b2d8-4bceaa9b95fa,soc_082_individuals_using_the_internet_edit,yr_data,IT.NET.USER.ZS,% of population -soc.090,Population Growth Rates,Yes,5bc62559-3ae3-472c-9517-823437539a3f,soc_090_pop_growth_rate_edit,yr_data,SP.POP.GROW,percent -soc.101,Population in Poverty ($3.20/day),Yes,8711d841-8421-4fcb-8ee2-881e573856e0,soc_101_poverty320_edit,yr_data,SI.POV.LMIC,% of population -soc.102,Inflation of Consumer Prices,Yes,4996cdb1-6169-46b7-9a74-0adf0e2ef1ce,soc_102_inflation_edit,yr_data,FP.CPI.TOTL.ZG,annual % -wat.005a;wat.005b;wat.005c,Access to an Improved Water Source (National);Access to an Improved Water Source (Urban);Access to an Improved Water Source (Rural),Yes,e32bdaba-5a72-487c-b640-668adaea2976;c29473e4-6606-49c9-a248-df1f2b21c2b5;3598f43d-0ae9-445b-a8da-e59ff2cf43d9,wat_005a_improved_water_access_edit,total;urban;rural,SH.H2O.SMDW.ZS;SH.H2O.SMDW.UR.ZS;SH.H2O.SMDW.RU.ZS,% of population;% of urban pop;% of rural pop \ No newline at end of file +WRI ID,Public Title,Time Slider,Dataset ID,Carto Table,Carto Column,wb_indicators,wb_units,skip +cit.020,Air Quality: Population Exposed to Unhealthy Levels of Fine Particulate Matter (PM2.5),Yes,6d3163f5-4e08-4830-84f1-2c5d76570a82,cit_020_nat_pop_exposed_high_levels_pm_2_5_edit,yr_data,EN.ATM.PM25.MC.ZS,% of population, +cit.025,Urban Population,Yes,d38d0d5c-31b1-47f4-9d2e-d8fba4c7d083,cit_025_urban_population_edit,yr_data,SP.URB.TOTL.IN.ZS,% of total population, +cit.028,Urban Population Living in Slums,Yes,35ce2b98-adbb-4873-b334-d7b1cc542de7,cit_028_percent_urban_slums_edit,yr_data,EN.POP.SLUM.UR.ZS,% of urban population, +com.006,"Current Account Balance (BoP, current US$)",Yes,2e31a1f3-576b-46b4-84f0-3f0cc399f887,com_006_national_current_accounts_edit,yr_data,BN.CAB.XOKA.CD,"BoP, current US$", +com.010,"Gross Domestic Product (PPP, Current International $)",Yes,52c55378-0484-48c3-92fc-3ee94d21c716,com_010_gdp_ppp_usd_edit,yr_data,NY.GDP.MKTP.CD,current US$, +com.036,Unemployment Rate,Yes,90432312-35be-46c4-bd79-42f606d5611e,com_036_unemployment_edit,yr_data,SL.UEM.TOTL.ZS,% of total labor force, +ene.004,Percentage of Energy Consumption from Renewable Sources,Yes,bced4001-425a-4fad-8c22-8214d9340ea4,ene_004_renewable_energy_share_of_total_energy_consumption_edit,renewable_energy_share_of_total_energy_consumption_p,EG.FEC.RNEW.ZS,% of total final energy consumption, +ene.012a;ene.012b;ene.012c,Access to Electricity (National);Access to Electricity (Urban);Access to Electricity (Rural),Yes,c29aee50-a46f-4c10-a8b1-152e4821be81;1ad53858-f5da-47cb-8006-5b4aa5aad589;ebcf4c98-a07d-479b-a27c-6d0c5b444548,ene_012_electricity_access_edit,total;urban;rural,EG.ELC.ACCS.ZS;EG.ELC.ACCS.UR.ZS;EG.ELC.ACCS.RU.ZS,% of population;% of urban population;% of rural population, +ene.021a,Renewable Energy Consumption,Yes,431f7c9e-7ef5-4f00-a5dc-8845fcb3a9a9,ene_021_se4all_country_indicators_edit,yr_data,3.1_RE.CONSUMPTION,Renewable energy consumption (TJ), +ene.028,Access to Clean Cooking Fuels,Yes,c665f519-eef9-4f67-a8bf-7e3e6dc8bfcd,ene_028_access_clean_cooking_fuels_edit,yr_data,EG.CFT.ACCS.ZS,% of population, +ene.029a,Energy Intensity,Yes,2c444596-2be3-4786-bdfc-24010f99b21e,ene_029a_energy_intensity_edit,energy_intensity,6.1_PRIMARY.ENERGY.INTENSITY,MJ/2011 USD PPP, +ene.036.rw0,Pump Price for Diesel Fuel,Yes,3b77d3d6-e2ec-42c5-a2b4-04a3ebbd90f7,ene_036_rw0_pump_price_for_diesel_fuel_edit,yr_data,EP.PMP.DESL.CD,US$ per litter,TRUE +ene.037.rw0,Pump Price for Gasoline,Yes,b8b2cdde-27ec-48c7-be63-e7cdcbec76d1,ene_037_rw0_pump_price_for_gasoline_edit,yr_data,EP.PMP.SGAS.CD,US$ per litter,TRUE +foo.043,"Agriculture, Value Added",Yes,95b013a3-389a-4367-83b7-c9d68c28c406,foo_043_agriculture_value_added_edit,yr_data,NV.AGR.TOTL.ZS,percent of GDP, +soc.008,Gross Domestic Product Per Capita,Yes,00abb46f-34e2-4bf7-be30-1fb0b1de022f,soc_008_gdp_per_capita_edit,yr_data,NY.GDP.PCAP.PP.CD,current international $, +soc.015,Adult Literacy Rate,Yes,e8f53f73-d77c-485a-a2a6-1c47ea4aead9,soc_015_adult_literacy_rate_edit,yr_data;female_data;male_data,SE.ADT.LITR.ZS;SE.ADT.LITR.FE.ZS;SE.ADT.LITR.MA.ZS,% of population 15+;% of females 15+;% of males 15+, +soc.020,Gini Index,Yes,f8d3e79c-c3d0-4f9a-9b68-9c5ad1f025e4,soc_020_gini_edit,yr_data,SI.POV.GINI,GINI index (World Bank estimate), +soc.029,Worldwide Governance Indicators,No,7793f46c-a48a-466f-a8ce-ca1a87b7aeed,soc_029_worldwide_governance_indicators_edit,government_effectiveness_data;regulatory_quality_data;political_stability_data;rule_of_law_data;voice_accountability_data;control_of_corruption_data,GE.EST;RQ.EST;PV.EST;RL.EST;VA.EST;CC.EST,estimate;estimate;estimate;estimate;estimate;estimate, +soc.036,Life Expectancy at Birth,Yes,8671f536-1979-4b6f-a147-70152fcb44ed,soc_036_life_expectancy_at_birth_edit,yr_data,SP.DYN.LE00.IN,years, +soc.040,Access to Improved Sanitation Facilities,No,37d04efc-0ab2-4499-a891-54dca1013c74,soc_040_improved_sanitation_edit,total_data;urban_data;rural_data,SH.STA.SMSS.ZS;SH.STA.SMSS.UR.ZS;SH.STA.SMSS.RU.ZS,% of population;% of urban pop;% of rural pop, +soc.066,Population Below Income Poverty Line ($1.90),Yes,4d7ce999-1e37-418f-b8a6-1816b29e901a,soc_066_population_below_poverty_line_edit,yr_data,SI.POV.DDAY,% of population, +soc.074,Employment in Agriculture,Yes,cc354f7f-2622-44cb-91c1-73559373de72,soc_074_employment_in_agriculture_edit,yr_data,SL.AGR.EMPL.ZS,% of population, +soc.076,National Populations,Yes,2a8ca4f7-7285-4aed-9ef5-ba1f9c4b653d,soc_076_country_population_edit,yr_data,SP.POP.TOTL,people, +soc.079,Fertility Rates,Yes,80f951a7-edd5-472a-8b46-744eb93fd7ef,soc_079_total_fertility_rate_edit,total_fertility_rate,SP.DYN.TFRT.IN,births per woman, +soc.081,National Infant Mortality,Yes,6cfd6255-609e-4922-8709-5aaad0db9fae,soc_081_mortality_rate_edit,mortality_rate,SP.DYN.IMRT.IN,deaths per 1000 live births, +soc.082,Percent of Population Using the Internet,Yes,daaa9f12-c0ef-499a-b2d8-4bceaa9b95fa,soc_082_individuals_using_the_internet_edit,yr_data,IT.NET.USER.ZS,% of population, +soc.090,Population Growth Rates,Yes,5bc62559-3ae3-472c-9517-823437539a3f,soc_090_pop_growth_rate_edit,yr_data,SP.POP.GROW,percent, +soc.101,Population in Poverty ($3.20/day),Yes,8711d841-8421-4fcb-8ee2-881e573856e0,soc_101_poverty320_edit,yr_data,SI.POV.LMIC,% of population, +soc.102,Inflation of Consumer Prices,Yes,4996cdb1-6169-46b7-9a74-0adf0e2ef1ce,soc_102_inflation_edit,yr_data,FP.CPI.TOTL.ZG,annual %, +wat.005a;wat.005b;wat.005c,Access to an Improved Water Source (National);Access to an Improved Water Source (Urban);Access to an Improved Water Source (Rural),Yes,e32bdaba-5a72-487c-b640-668adaea2976;c29473e4-6606-49c9-a248-df1f2b21c2b5;3598f43d-0ae9-445b-a8da-e59ff2cf43d9,wat_005a_improved_water_access_edit,total;urban;rural,SH.H2O.SMDW.ZS;SH.H2O.SMDW.UR.ZS;SH.H2O.SMDW.RU.ZS,% of population;% of urban pop;% of rural pop, diff --git a/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py b/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py index cf24d1ba..f28c7238 100755 --- a/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py +++ b/upload_worldbank_data/contents/src/update_worldbank_data_on_carto.py @@ -135,7 +135,7 @@ def fetch_wb_data(table): try: # fetch data for this indicator (only the first 10,000 entries will be returned) res = requests.get( - "https://api.worldbank.org/v2/countries/all/indicators/{}?format=json&per_page=10000".format(indicator)) + f"https://api.worldbank.org/v2/countries/all/indicators/{indicator}?format=json&per_page=10000") # check how many pages of data there are for this indicator pages = int(res.json()[0]['pages']) break @@ -253,6 +253,8 @@ def main(): if not os.path.exists(data_dir): os.mkdir(data_dir) + wb_rw_table = wb_rw_table[wb_rw_table['skip']!=True] + # process each Carto table for World Bank datasets one at a time for table_name, info in wb_rw_table.iterrows(): # get the dataset name (table name without the '_edit' at the end of the table_name @@ -370,4 +372,5 @@ def main(): # Delete local files in Docker container delete_local() - logging.info('SUCCESS') \ No newline at end of file + logging.info('SUCCESS') + \ No newline at end of file