Align Bosnia fetcher with design docs

Author: thiagovmdon

rivretrieve/bosnia_herzegovina.py

@@ -14,17 +14,26 @@
 
 
 class BosniaHerzegovinaFetcher(base.RiverDataFetcher):
-    """Fetches river gauge data from vodostaji.voda.ba.
-
-    Data Source: [Federal Hydrometeorological Institute portal](https://vodostaji.voda.ba/#2031)
-
-    Supported Variables:
-        - ``constants.DISCHARGE_DAILY_MEAN`` (m3/s)
-        - ``constants.DISCHARGE_INSTANT`` (m3/s)
-        - ``constants.STAGE_DAILY_MEAN`` (m)
-        - ``constants.STAGE_INSTANT`` (m)
-        - ``constants.WATER_TEMPERATURE_DAILY_MEAN`` (degC)
-        - ``constants.WATER_TEMPERATURE_INSTANT`` (degC)
+    """Fetches river gauge data from the Federal Hydrometeorological Institute portal.
+
+    Data source:
+        - https://vodostaji.voda.ba/
+
+    Supported variables:
+        - constants.DISCHARGE_DAILY_MEAN (m³/s)
+        - constants.DISCHARGE_INSTANT (m³/s)
+        - constants.STAGE_DAILY_MEAN (m)
+        - constants.STAGE_INSTANT (m)
+        - constants.WATER_TEMPERATURE_DAILY_MEAN (°C)
+        - constants.WATER_TEMPERATURE_INSTANT (°C)
+
+    Data description and API:
+        - live station metadata snapshot: https://vodostaji.voda.ba/data/internet/layers/20/index.json
+        - annual station workbooks:
+          https://vodostaji.voda.ba/data/internet/stations/<group>/<gauge_id>/<parameter>/<file>
+
+    Terms of use:
+        - see https://vodostaji.voda.ba/
     """
 
     METADATA_URL = "https://vodostaji.voda.ba/data/internet/layers/20/index.json"
@@ -52,6 +61,16 @@ class BosniaHerzegovinaFetcher(base.RiverDataFetcher):
         },
     }
 
+    @staticmethod
+    def _empty_result(variable: str) -> pd.DataFrame:
+        """Returns an empty standardized RivRetrieve time series frame."""
+        return pd.DataFrame(columns=[constants.TIME_INDEX, variable]).set_index(constants.TIME_INDEX)
+
+    @staticmethod
+    def _empty_metadata() -> pd.DataFrame:
+        """Returns an empty metadata frame indexed by gauge ID."""
+        return pd.DataFrame(columns=[constants.GAUGE_ID]).set_index(constants.GAUGE_ID)
+
     @staticmethod
     def get_cached_metadata() -> pd.DataFrame:
         """Retrieves cached Bosnia and Herzegovina gauge metadata."""
@@ -62,7 +81,11 @@ def get_available_variables() -> tuple[str, ...]:
         return tuple(BosniaHerzegovinaFetcher.VARIABLE_MAP.keys())
 
     def get_metadata(self) -> pd.DataFrame:
-        """Downloads and normalizes station metadata from the live JSON snapshot."""
+        """Downloads and normalizes station metadata from the live JSON snapshot.
+
+        Keeps provider-specific metadata columns, standardizes the key RivRetrieve
+        metadata fields, and returns a DataFrame indexed by ``constants.GAUGE_ID``.
+        """
         session = utils.requests_retry_session()
 
         try:
@@ -77,7 +100,7 @@ def get_metadata(self) -> pd.DataFrame:
             raise
 
         if not isinstance(data, list) or not data:
-            return pd.DataFrame().set_index(constants.GAUGE_ID)
+            return self._empty_metadata()
 
         df = pd.json_normalize(data)
         rename_map = {
@@ -163,21 +186,29 @@ def _parse_data(
         """Parses the Excel bytes into the standard RivRetrieve data frame layout."""
         content, station_group = raw_data
         if not content:
-            return pd.DataFrame(columns=[constants.TIME_INDEX, variable]).set_index(constants.TIME_INDEX)
+            return self._empty_result(variable)
 
         try:
-            df = pd.read_excel(BytesIO(content), skiprows=8, names=[constants.TIME_INDEX, variable])
+            df = pd.read_excel(
+                BytesIO(content),
+                skiprows=8,
+                header=None,
+                names=[constants.TIME_INDEX, variable],
+            )
         except Exception as exc:
             logger.error(f"Failed to parse Bosnia and Herzegovina data for {gauge_id}: {exc}")
-            return pd.DataFrame(columns=[constants.TIME_INDEX, variable]).set_index(constants.TIME_INDEX)
+            return self._empty_result(variable)
 
         if df.empty:
-            return pd.DataFrame(columns=[constants.TIME_INDEX, variable]).set_index(constants.TIME_INDEX)
+            return self._empty_result(variable)
 
         df[constants.TIME_INDEX] = pd.to_datetime(df[constants.TIME_INDEX], dayfirst=True, errors="coerce")
         df[variable] = pd.to_numeric(df[variable], errors="coerce")
         df = df.dropna(subset=[constants.TIME_INDEX, variable])
 
+        if variable in {constants.STAGE_DAILY_MEAN, constants.STAGE_INSTANT}:
+            df[variable] = df[variable] / 100.0
+
         if variable in {
             constants.DISCHARGE_DAILY_MEAN,
             constants.STAGE_DAILY_MEAN,
@@ -199,18 +230,44 @@ def get_data(
         start_date: Optional[str] = None,
         end_date: Optional[str] = None,
     ) -> pd.DataFrame:
-        """Fetches and parses time series data for a specific gauge and variable."""
+        """Fetches and parses time series data for a specific gauge and variable.
+
+        This method retrieves the requested data from the provider's workbook endpoint,
+        parses it, and returns it in a standardized pandas DataFrame format.
+
+        Args:
+            gauge_id: The site-specific identifier for the gauge.
+            variable: The variable to fetch. Must be one of the strings listed
+                in the fetcher's ``get_available_variables()`` output.
+                These are typically defined in ``rivretrieve.constants``.
+            start_date: Optional start date for the data retrieval in 'YYYY-MM-DD' format.
+                If None, data is fetched from the earliest available date in the workbook.
+            end_date: Optional end date for the data retrieval in 'YYYY-MM-DD' format.
+                If None, data is fetched up to the latest available date in the workbook.
+
+        Returns:
+            pd.DataFrame: A pandas DataFrame indexed by datetime objects (``constants.TIME_INDEX``)
+            with a single column named after the requested ``variable``. The DataFrame
+            will be empty if no data is found for the given parameters.
+
+        Raises:
+            ValueError: If the requested ``variable`` is not supported by this fetcher.
+        """
         start_date = utils.format_start_date(start_date)
         end_date = utils.format_end_date(end_date)
 
         if variable not in self.get_available_variables():
             raise ValueError(f"Unsupported variable: {variable}")
 
-        raw_data = self._download_data(gauge_id, variable, start_date, end_date)
-        df = self._parse_data(gauge_id, raw_data, variable)
+        try:
+            raw_data = self._download_data(gauge_id, variable, start_date, end_date)
+            df = self._parse_data(gauge_id, raw_data, variable)
+        except Exception as exc:
+            logger.error(f"Failed to fetch Bosnia and Herzegovina data for {gauge_id} ({variable}): {exc}")
+            return self._empty_result(variable)
 
         if df.empty:
-            return df
+            return self._empty_result(variable)
 
         start_dt = pd.to_datetime(start_date)
         end_dt = pd.to_datetime(end_date) + pd.Timedelta(days=1)

tests/test_bosnia_herzegovina.py

@@ -1,5 +1,4 @@
 import json
-import os
 import unittest
 from pathlib import Path
 from unittest.mock import MagicMock, patch
@@ -14,111 +13,161 @@
 class TestBosniaHerzegovinaFetcher(unittest.TestCase):
     def setUp(self):
         self.fetcher = BosniaHerzegovinaFetcher()
-        self.test_data_dir = Path(os.path.dirname(__file__)) / "test_data"
+        self.test_data_dir = Path(__file__).parent / "test_data"
 
     def _load_json(self, filename):
-        with open(self.test_data_dir / filename, "r", encoding="utf-8") as f:
-            return json.load(f)
+        with (self.test_data_dir / filename).open("r", encoding="utf-8") as file_handle:
+            return json.load(file_handle)
+
+    def _load_bytes(self, filename):
+        return (self.test_data_dir / filename).read_bytes()
+
+    @staticmethod
+    def _build_response(status_code=200, content=b"", json_data=None):
+        response = MagicMock()
+        response.status_code = status_code
+        response.content = content
+        response.json.return_value = json_data
+        response.raise_for_status = MagicMock()
+        return response
 
     @patch("rivretrieve.utils.requests_retry_session")
     def test_get_metadata(self, mock_requests_session):
         mock_session = MagicMock()
         mock_requests_session.return_value = mock_session
 
-        mock_response = MagicMock()
-        mock_response.json.return_value = self._load_json("bosnia_herzegovina_metadata_sample.json")
-        mock_response.raise_for_status = MagicMock()
+        mock_response = self._build_response(
+            json_data=self._load_json("bosnia_herzegovina_metadata_sample.json")
+        )
         mock_session.get.return_value = mock_response
 
         result_df = self.fetcher.get_metadata()
 
+        self.assertEqual(result_df.index.name, constants.GAUGE_ID)
         self.assertEqual(list(result_df.index), ["4510", "4121"])
         self.assertEqual(result_df.loc["4510", constants.STATION_NAME], "HS Kaloševići")
         self.assertEqual(result_df.loc["4510", constants.RIVER], "Usora")
         self.assertAlmostEqual(result_df.loc["4510", constants.LATITUDE], 44.64680728070949)
         self.assertAlmostEqual(result_df.loc["4510", constants.LONGITUDE], 17.90406242892678)
+        self.assertIn("metadata_station_carteasting", result_df.columns)
+        self.assertIn("catchment", result_df.columns)
         self.assertEqual(result_df.loc["4510", constants.COUNTRY], "Bosnia and Herzegovina")
         self.assertEqual(result_df.loc["4510", constants.SOURCE], "vodostaji.voda.ba")
         self.assertAlmostEqual(result_df.loc["4121", constants.AREA], 123.4)
+        mock_session.get.assert_called_once_with(self.fetcher.METADATA_URL, timeout=30)
 
-    @patch("pandas.read_excel")
     @patch("rivretrieve.utils.requests_retry_session")
-    def test_get_data_instant_discharge_detects_station_group(self, mock_requests_session, mock_read_excel):
+    def test_get_data_instant_discharge_detects_station_group(self, mock_requests_session):
         mock_session = MagicMock()
         mock_requests_session.return_value = mock_session
-        mock_read_excel.return_value = pd.DataFrame(
-            {
-                constants.TIME_INDEX: [
-                    "01.01.2025 00:00",
-                    "01.01.2025 01:00",
-                    "01.01.2025 02:00",
-                    "02.01.2025 00:00",
-                ],
-                constants.DISCHARGE_INSTANT: [1.0, 2.0, 3.0, 4.0],
-            }
-        )
 
-        missing_response = MagicMock(status_code=404, content=b"")
-        success_response = MagicMock(status_code=200, content=b"fake-xlsx-content")
-        mock_session.get.side_effect = [missing_response, missing_response, success_response]
+        missing_response = self._build_response(status_code=404)
+        success_response = self._build_response(
+            status_code=200,
+            content=self._load_bytes("bosnia_herzegovina_4510_discharge_20250323.xlsx"),
+        )
+        mock_session.get.side_effect = [missing_response, missing_response, missing_response, success_response]
 
         result_df = self.fetcher.get_data(
             gauge_id="4510",
             variable=constants.DISCHARGE_INSTANT,
-            start_date="2025-01-01",
-            end_date="2025-01-01",
+            start_date="2025-03-23",
+            end_date="2025-03-23",
         )
 
         expected_df = pd.DataFrame(
             {
-                constants.TIME_INDEX: pd.to_datetime(
-                    ["2025-01-01 00:00:00", "2025-01-01 01:00:00", "2025-01-01 02:00:00"]
-                ),
-                constants.DISCHARGE_INSTANT: [1.0, 2.0, 3.0],
+                constants.TIME_INDEX: pd.date_range("2025-03-23 00:00:00", periods=24, freq="h"),
+                constants.DISCHARGE_INSTANT: [
+                    8.304,
+                    7.958,
+                    8.105,
+                    8.007,
+                    7.909,
+                    7.762,
+                    7.958,
+                    7.665,
+                    7.713,
+                    8.205,
+                    8.007,
+                    7.328,
+                    7.860,
+                    8.105,
+                    7.568,
+                    7.811,
+                    7.958,
+                    7.762,
+                    7.665,
+                    7.280,
+                    7.568,
+                    7.472,
+                    7.472,
+                    7.280,
+                ],
             }
         ).set_index(constants.TIME_INDEX)
 
-        assert_frame_equal(result_df, expected_df)
-        self.assertEqual(result_df.attrs["station_group"], 3)
-        self.assertEqual(mock_session.get.call_count, 3)
+        assert_frame_equal(result_df, expected_df, check_dtype=False)
+        self.assertEqual(result_df.index.name, constants.TIME_INDEX)
+        self.assertEqual(result_df.attrs["station_group"], 4)
+        self.assertEqual(mock_session.get.call_count, 4)
         self.assertIn("/1/4510/Q/Q_1Y.xlsx", mock_session.get.call_args_list[0].args[0])
-        self.assertIn("/3/4510/Q/Q_1Y.xlsx", mock_session.get.call_args_list[2].args[0])
+        self.assertIn("/4/4510/Q/Q_1Y.xlsx", mock_session.get.call_args_list[3].args[0])
+        self.assertTrue(all(call.kwargs["timeout"] == 20 for call in mock_session.get.call_args_list))
 
-    @patch("pandas.read_excel")
     @patch("rivretrieve.utils.requests_retry_session")
-    def test_get_data_daily_temperature(self, mock_requests_session, mock_read_excel):
+    def test_get_data_daily_stage_converts_centimeters_to_meters(self, mock_requests_session):
         mock_session = MagicMock()
         mock_requests_session.return_value = mock_session
-        mock_read_excel.return_value = pd.DataFrame(
-            {
-                constants.TIME_INDEX: [
-                    "01.01.2025 00:00",
-                    "01.01.2025 12:00",
-                    "02.01.2025 00:00",
-                    "02.01.2025 12:00",
-                ],
-                constants.WATER_TEMPERATURE_DAILY_MEAN: [10.0, 11.0, 12.0, 12.0],
-            }
+        missing_response = self._build_response(status_code=404)
+        success_response = self._build_response(
+            status_code=200,
+            content=self._load_bytes("bosnia_herzegovina_4510_stage_20250323.xlsx"),
         )
-
-        success_response = MagicMock(status_code=200, content=b"fake-xlsx-content")
-        mock_session.get.return_value = success_response
+        mock_session.get.side_effect = [missing_response, missing_response, missing_response, success_response]
 
         result_df = self.fetcher.get_data(
             gauge_id="4510",
-            variable=constants.WATER_TEMPERATURE_DAILY_MEAN,
-            start_date="2025-01-01",
-            end_date="2025-01-02",
+            variable=constants.STAGE_DAILY_MEAN,
+            start_date="2025-03-23",
+            end_date="2025-03-24",
         )
 
         expected_df = pd.DataFrame(
             {
-                constants.TIME_INDEX: pd.to_datetime(["2025-01-01", "2025-01-02"]),
-                constants.WATER_TEMPERATURE_DAILY_MEAN: [10.5, 12.0],
+                constants.TIME_INDEX: pd.to_datetime(["2025-03-23", "2025-03-24"]),
+                constants.STAGE_DAILY_MEAN: [0.8113333333333334, 0.9504166666666667],
             }
         ).set_index(constants.TIME_INDEX)
 
-        assert_frame_equal(result_df, expected_df)
+        assert_frame_equal(result_df, expected_df, check_dtype=False)
+        self.assertIn("/4/4510/H/H_1Y.xlsx", mock_session.get.call_args_list[3].args[0])
+
+    @patch("rivretrieve.utils.requests_retry_session")
+    def test_get_data_returns_standardized_empty_frame_for_empty_temperature_workbook(self, mock_requests_session):
+        mock_session = MagicMock()
+        mock_requests_session.return_value = mock_session
+        missing_response = self._build_response(status_code=404)
+        success_response = self._build_response(
+            status_code=200,
+            content=self._load_bytes("bosnia_herzegovina_4510_water_temperature_20250323.xlsx"),
+        )
+        mock_session.get.side_effect = [missing_response, missing_response, missing_response, success_response]
+
+        result_df = self.fetcher.get_data(
+            gauge_id="4510",
+            variable=constants.WATER_TEMPERATURE_INSTANT,
+            start_date="2025-03-23",
+            end_date="2025-03-23",
+        )
+
+        expected_df = pd.DataFrame(
+            columns=[constants.TIME_INDEX, constants.WATER_TEMPERATURE_INSTANT]
+        ).set_index(constants.TIME_INDEX)
+
+        assert_frame_equal(result_df, expected_df, check_dtype=False)
+        self.assertEqual(result_df.index.name, constants.TIME_INDEX)
+        self.assertIn("/4/4510/WT/Tvode_1Y.xlsx", mock_session.get.call_args_list[3].args[0])
 
 
 if __name__ == "__main__":