Skip to content

WP1.8: Add informal settlements proportion indicator #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FlorianNachtigall merged 1 commit into from
Oct 9, 2026
Merged

WP1.8: Add informal settlements proportion indicator #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@ dist-ssr
*.njsproj
*.sln
*.sw?
source-data/
207 changes: 164 additions & 43 deletions preprocess.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,33 +460,135 @@
},
{
"cell_type": "code",
"execution_count": 44,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ghg_sector_cols = [\n",
" 'eu_ghg_emissions_energy_industry',\n",
" 'eu_ghg_emissions_residential',\n",
" 'eu_ghg_emissions_transport',\n",
" 'eu_ghg_emissions_waste',\n",
" 'eu_ghg_emissions_agriculture',\n",
"# --- SDG 11.1.1: Informal settlements proportion ---\n",
"import unicodedata\n",
"\n",
"slum_metric_key = 'informal_settlements_proportion'\n",
"slum_metric_column = 'Proportion of urban population living in slums or informal settlements (%) (a)'\n",
"slum_year_column = 'Data Reference Year'\n",
"\n",
"sdg_source_candidates = [\n",
" Path('source-data/SDG_11-1-1.xlsx'),\n",
" Path('SDG_11-1-1.xlsx'),\n",
"]\n",
"available_ghg_cols = [col for col in ghg_sector_cols if col in cities.columns]\n",
"if available_ghg_cols:\n",
" ghg_total = cities[available_ghg_cols].sum(axis=1, min_count=1)\n",
" share_name_map = {\n",
" 'eu_ghg_emissions_energy_industry': 'eu_ghg_share_energy_industry',\n",
" 'eu_ghg_emissions_residential': 'eu_ghg_share_residential',\n",
" 'eu_ghg_emissions_transport': 'eu_ghg_share_transport',\n",
" 'eu_ghg_emissions_waste': 'eu_ghg_share_waste',\n",
" 'eu_ghg_emissions_agriculture': 'eu_ghg_share_agriculture',\n",
"sdg_source_path = next((path for path in sdg_source_candidates if path.exists()), None)\n",
"if sdg_source_path is None:\n",
" raise FileNotFoundError(\n",
" 'SDG_11-1-1.xlsx not found. Place the file in source-data/SDG_11-1-1.xlsx (preferred) or project root.'\n",
" )\n",
"\n",
"slum_raw = pd.read_excel(sdg_source_path, sheet_name='SDG 11-1-1 2024')\n",
"\n",
"slum_country_candidates = [\n",
" 'Country or Territory Name_1',\n",
" 'Country or Territory Name.1',\n",
" 'Country or Territory Name',\n",
"]\n",
"slum_country_column = next((col for col in slum_country_candidates if col in slum_raw.columns), None)\n",
"if slum_country_column is None:\n",
" raise KeyError(f'Missing expected country column; checked: {slum_country_candidates}')\n",
"\n",
"required_slum_columns = [slum_country_column, slum_metric_column, slum_year_column]\n",
"missing_slum_columns = [col for col in required_slum_columns if col not in slum_raw.columns]\n",
"if missing_slum_columns:\n",
" raise KeyError(f'Missing expected SDG columns: {missing_slum_columns}')\n",
"\n",
"\n",
"def normalize_country_name(value):\n",
" text = str(value or '').strip()\n",
" if not text:\n",
" return ''\n",
" text = unicodedata.normalize('NFD', text)\n",
" text = ''.join(ch for ch in text if unicodedata.category(ch) != 'Mn')\n",
" text = text.replace('’', '').replace(\"'\", '').replace('`', '')\n",
" text = ''.join(ch if ch.isalnum() else ' ' for ch in text)\n",
" return ' '.join(text.split()).lower()\n",
"\n",
"\n",
"country_aliases = {\n",
" 'united states': 'united states of america',\n",
" 'united kingdom': 'united kingdom of great britain and northern ireland',\n",
" 'russia': 'russian federation',\n",
" 'turkey': 'turkiye',\n",
" 'tanzania': 'united republic of tanzania',\n",
" 'palestine': 'state of palestine',\n",
" 'syria': 'syrian arab republic',\n",
" 'iran': 'iran islamic republic of',\n",
" 'laos': 'lao peoples democratic republic',\n",
" 'vietnam': 'viet nam',\n",
" 'brunei': 'brunei darussalam',\n",
" 'republic of the congo': 'congo',\n",
"}\n",
"\n",
"slum_clean = slum_raw[[slum_country_column, slum_metric_column, slum_year_column]].copy()\n",
"slum_clean = slum_clean.rename(\n",
" columns={\n",
" slum_country_column: 'country_raw',\n",
" slum_metric_column: 'value',\n",
" slum_year_column: 'year',\n",
" }\n",
" for source_col, target_col in share_name_map.items():\n",
" if source_col in cities.columns:\n",
" cities[target_col] = np.where(\n",
" ghg_total > 0,\n",
" (cities[source_col] / ghg_total) * 100,\n",
" np.nan,\n",
")\n",
"slum_clean['value'] = pd.to_numeric(slum_clean['value'], errors='coerce')\n",
"slum_clean['year'] = pd.to_numeric(slum_clean['year'], errors='coerce')\n",
"slum_clean['country_norm'] = slum_clean['country_raw'].map(normalize_country_name)\n",
"slum_clean = slum_clean.dropna(subset=['value', 'year'])\n",
"slum_clean = slum_clean[slum_clean['country_norm'] != '']\n",
"\n",
"# Keep latest non-null year per country\n",
"slum_latest = slum_clean.sort_values(['country_norm', 'year']).groupby('country_norm', as_index=False).tail(1)\n",
"slum_country_values = slum_latest.groupby('country_norm', as_index=False)['value'].mean()\n",
"slum_value_map = dict(zip(slum_country_values['country_norm'], slum_country_values['value']))\n",
"\n",
"city_country_norm = cities['country'].map(normalize_country_name)\n",
"city_country_norm = city_country_norm.map(lambda key: country_aliases.get(key, key))\n",
"cities[slum_metric_key] = city_country_norm.map(slum_value_map)\n",
"\n",
"print(\n",
" 'Informal settlements metric coverage:',\n",
" int(cities[slum_metric_key].notna().sum()),\n",
" '/',\n",
" len(cities),\n",
")\n",
"print(\n",
" 'Source year range used (latest available per country):',\n",
" int(slum_latest['year'].min()),\n",
" '-',\n",
" int(slum_latest['year'].max()),\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [],
"source": [
"global_metric_keys = [\n",
" 'population',\n",
" 'population_growth',\n",
" 'population_density',\n",
" 'population_density_growth',\n",
" 'population_age',\n",
" 'gdp_ppp',\n",
" 'gdp_ppp_growth',\n",
" 'critical_infrastructure',\n",
" 'hdd',\n",
" 'cdd',\n",
" 'hdi',\n",
" 'female_gender_index',\n",
" 'informal_settlements_proportion',\n",
" 'emissions',\n",
"]\n",
"\n",
"if 'characteristics' in cities.columns:\n",
" for key in global_metric_keys:\n",
" if key not in cities.columns:\n",
" cities[key] = cities['characteristics'].map(\n",
" lambda d: d.get(key) if isinstance(d, dict) else np.nan\n",
" )\n",
"\n",
"eu_metric_keys = [\n",
Expand Down Expand Up @@ -516,6 +618,39 @@
" 'decentralization_index_fiscal',\n",
"]\n",
"\n",
"if 'eu_characteristics' in cities.columns:\n",
" for key in eu_metric_keys:\n",
" col = f'eu_{key}'\n",
" if col not in cities.columns:\n",
" cities[col] = cities['eu_characteristics'].map(\n",
" lambda d: d.get(key) if isinstance(d, dict) else np.nan\n",
" )\n",
"\n",
"ghg_sector_cols = [\n",
" 'eu_ghg_emissions_energy_industry',\n",
" 'eu_ghg_emissions_residential',\n",
" 'eu_ghg_emissions_transport',\n",
" 'eu_ghg_emissions_waste',\n",
" 'eu_ghg_emissions_agriculture',\n",
"]\n",
"available_ghg_cols = [col for col in ghg_sector_cols if col in cities.columns]\n",
"if available_ghg_cols:\n",
" ghg_total = cities[available_ghg_cols].sum(axis=1, min_count=1)\n",
" share_name_map = {\n",
" 'eu_ghg_emissions_energy_industry': 'eu_ghg_share_energy_industry',\n",
" 'eu_ghg_emissions_residential': 'eu_ghg_share_residential',\n",
" 'eu_ghg_emissions_transport': 'eu_ghg_share_transport',\n",
" 'eu_ghg_emissions_waste': 'eu_ghg_share_waste',\n",
" 'eu_ghg_emissions_agriculture': 'eu_ghg_share_agriculture',\n",
" }\n",
" for source_col, target_col in share_name_map.items():\n",
" if source_col in cities.columns:\n",
" cities[target_col] = np.where(\n",
" ghg_total > 0,\n",
" (cities[source_col] / ghg_total) * 100,\n",
" np.nan,\n",
" )\n",
"\n",
"for key in eu_metric_keys:\n",
" col = f'eu_{key}'\n",
" if col in cities.columns:\n",
Expand Down Expand Up @@ -548,8 +683,8 @@
" },\n",
" axis=1,\n",
")\n",
"cities.loc[cities['eu_characteristics'].map(lambda d: len(d) == 0), 'eu_characteristics'] = None\n",
"cities.loc[cities['eu_percentiles'].map(lambda d: len(d) == 0), 'eu_percentiles'] = None\n",
"cities.loc[cities['eu_characteristics'].map(lambda d: not isinstance(d, dict) or len(d) == 0), 'eu_characteristics'] = None\n",
"cities.loc[cities['eu_percentiles'].map(lambda d: not isinstance(d, dict) or len(d) == 0), 'eu_percentiles'] = None\n",
"\n",
"info_cols = [\n",
" 'id',\n",
Expand All @@ -575,21 +710,7 @@
" 'lon',\n",
"]\n",
"\n",
"metric_cols = [\n",
" 'population',\n",
" 'population_growth',\n",
" 'population_density',\n",
" 'population_density_growth',\n",
" 'population_age',\n",
" 'gdp_ppp',\n",
" 'gdp_ppp_growth',\n",
" 'critical_infrastructure',\n",
" 'hdd',\n",
" 'cdd',\n",
" 'hdi',\n",
" 'female_gender_index',\n",
" 'emissions',\n",
"]\n",
"metric_cols = global_metric_keys\n",
"\n",
"for col in metric_cols:\n",
" cities[f\"{col}_pct\"] = (cities[col].rank(pct=True) * 100).round()\n",
Expand All @@ -611,11 +732,11 @@
" },\n",
" axis=1,\n",
")\n",
"cities.loc[cities['characteristics'].map(lambda d: len(d) == 0), 'characteristics'] = None\n",
"cities.loc[cities['percentiles'].map(lambda d: len(d) == 0), 'percentiles'] = None\n",
"cities.loc[cities['characteristics'].map(lambda d: not isinstance(d, dict) or len(d) == 0), 'characteristics'] = None\n",
"cities.loc[cities['percentiles'].map(lambda d: not isinstance(d, dict) or len(d) == 0), 'percentiles'] = None\n",
"\n",
"available_info_cols = [col for col in info_cols if col in cities.columns]\n",
"cities[available_info_cols].to_json('public/cities.json', orient='records')"
"cities[available_info_cols].to_json('public/cities.json', orient='records')\n"
]
},
{
Expand Down Expand Up @@ -753,4 +874,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
2 changes: 1 addition & 1 deletion public/cities.json
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions src/components/InfoPanel.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,9 @@ const InfoPanel = ({ selectedSample, samples, setSelectedSample, setSearchValue
<p>
Percentile ranks indicate the share of cities worldwide that have lower values, showing how each city compares to its global peers for each characteristic.
</p>
<p>
The indicator &ldquo;Informal settlements proportion&rdquo; describes the proportion of urban population living in slums or informal settlements. It uses UN-Habitat SDG 11.1.1 at national level, mapped to cities by country, using the latest non-null country value (typically 2022, earlier if 2022 is unavailable).
</p>
</MethodologySection>
<div
style={{
Expand Down
1 change: 1 addition & 0 deletions src/utils/metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export const metricList = [
{ key: "critical_infrastructure", label: "Critical infrastructure index", decimals: 2 },
{ key: "female_gender_index", label: "Female gender index", decimals: 2 },
{ key: "hdi", label: "Human development index", decimals: 2 },
{ key: "informal_settlements_proportion", label: "Informal settlements proportion", unit: "%", decimals: 1 },
{ key: "emissions", label: "GHG emissions", unit: "tCO₂e/cap", decimals: 2 },
];

Expand Down
Loading