diff --git a/.env b/.env index 4db5e2c..8e491ff 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -SOURCE_FILE=./resources/samples/input4_LatLon.pickle +SOURCE_FILE=./resources/samples/input4_LatLon.pickle.gz # development settings CONFIGURATION_FILE=./app-configuration.json PRINT_CONFIGURATION=yes \ No newline at end of file diff --git a/CHANGELOG_SDK.md b/CHANGELOG_SDK.md index 850d7b0..b1b816d 100644 --- a/CHANGELOG_SDK.md +++ b/CHANGELOG_SDK.md @@ -1,5 +1,24 @@ # Changelog SDK +## 2026-09 `v3.1.0` + +- the template now pins `python=3.14` (was 3.11). + +**This does not change your App.** `environment.yml` is excluded from the template sync, so +your repository keeps its own. If you want the newer Python, edit your `environment.yml` +yourself and rebuild - and check what your dependencies resolve to afterwards, because conda +resolves them against today's conda-forge, not against the day your App was first built. + +## 2026-07 `v3.0.0` + +**Breaking change.** The App output is now always written as a gzip-compressed pickle. Apps built +against an earlier SDK cannot read it, so every Python App has to be rebuilt against this SDK. + +- the App output is written gzip-compressed, typically around a tenth of its previous size. +- App input is read whether it is compressed or not - the SDK detects which, so your App does not + have to care. +- shipped sample and test input files are now gzip-compressed (`.pickle.gz`). + ## 2025-07 `v2.3.0` - introduce app-setting-type `SECRET` diff --git a/developer_README.md b/developer_README.md index b5a3a28..6679235 100644 --- a/developer_README.md +++ b/developer_README.md @@ -23,7 +23,7 @@ Here you find an overview of the files and their function in the SDK: 1. `.env`: adjust for App testing. defining the SDK Runtime environment, see below. See [Step 5](https://docs.moveapps.org/#/create_py_app?id=step-5-test-your-app-locally) 1. `./resources/**`: use for App testing. Resources of the SDK 1. `auxiliary/**`: Simulates the usage of [*auxiliary files*](https://docs.moveapps.org/#/auxiliary). You can put files into this folder to simulate an App run with provided/user-uploaded files. - 1. `output/**`: The output data (`output.pickle`) that will be passed on to the next App in a Workflow and other output files (artifacts) that your App may produce will be stored here. See [*producing artifacts*](https://docs.moveapps.org/#/copilot-python-sdk?id=producing-artifacts) for more information. + 1. `output/**`: The output data (`output.pickle.gz`) that will be passed on to the next App in a Workflow and other output files (artifacts) that your App may produce will be stored here. See [*producing artifacts*](https://docs.moveapps.org/#/copilot-python-sdk?id=producing-artifacts) for more information. 1. `samples/**`: Collection of sample App input data. You can use these data to test an App run with real input. @@ -31,7 +31,9 @@ Here you find an overview of the files and their function in the SDK: Critical parts of the SDK can be adjusted by `environment variables`. Keep in mind that these variables are only changeable during App development and not during an App run on MoveApps. They are predefined with sensible defaults - they should work for you as they are. While testing your App you will want to modify the SOURCE_FILE variable to either call the different example data sets provided in the template or other data sets that you want to use to test your App. -- `SOURCE_FILE`: path to the input file for your App. +- `SOURCE_FILE`: path to the input file for your App. Gzip-compressed (`.pickle.gz`) or uncompressed - the SDK detects which from the file content, not from the file name. +- `OUTPUT_FILE`: path to the output file of your App, ie the data that is passed on to the next App in a Workflow (default: `resources/output/output.pickle.gz`). The SDK always writes it gzip-compressed. +- `ERROR_FILE`: path to the file the SDK writes the error message to in case your App raises an exception (default: `resources/output/error.txt`). - `CONFIGURATION_FILE`: path to the configuration/settings file of your App (in [JSON](https://www.w3schools.com/js/js_json_intro.asp) format - must correspondent with the `settings` of your `appspec.json`, see [MoveApps parameters](https://docs.moveapps.org/#/copilot-python-sdk.md#moveapps-parameters) for an example of the `app-configuration.json` file). - `PRINT_CONFIGURATION`: prints the configuration your App receives (`yes|no`) - `MASK_SETTING_IDS`: A comma-separated list of setting IDs whose values will be hidden in the SDK logs diff --git a/environment.yml b/environment.yml index dfd0c38..7505b90 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: moveapps-python-sdk channels: - conda-forge dependencies: - - python=3.11 + - python=3.14 - pandas - geopandas - movingpandas diff --git a/resources/samples/README.txt b/resources/samples/README.txt index c01d73e..28f8b41 100644 --- a/resources/samples/README.txt +++ b/resources/samples/README.txt @@ -14,16 +14,16 @@ Set of input data to test apps. - data are provided in "lat/long" (EPSG:4326) and projected to "Mollweide" (ESRI:54009) in order to test your app accordingly for not projected and projected data. If your app does not allow projected data or only can deal with projected data, document and either build a automatic transformation in the app or make it fail with an informative error message. The app "Change projection" can be refered to for the user to change the projection of the data acordingly previous to your app. *File names* -input1_LatLon.pickle -input1_Mollweide.pickle +input1_LatLon.pickle.gz +input1_Mollweide.pickle.gz -input2_LatLon.pickle -input2_Mollweide.pickle +input2_LatLon.pickle.gz +input2_Mollweide.pickle.gz -input3_LatLon.pickle -input3_Mollweide.pickle +input3_LatLon.pickle.gz +input3_Mollweide.pickle.gz -input4_LatLon.pickle -input4_Mollweide.pickle +input4_LatLon.pickle.gz +input4_Mollweide.pickle.gz diff --git a/resources/samples/input1_LatLon.pickle b/resources/samples/input1_LatLon.pickle deleted file mode 100644 index b2f71f2..0000000 Binary files a/resources/samples/input1_LatLon.pickle and /dev/null differ diff --git a/resources/samples/input1_LatLon.pickle.gz b/resources/samples/input1_LatLon.pickle.gz new file mode 100644 index 0000000..bca7bb7 Binary files /dev/null and b/resources/samples/input1_LatLon.pickle.gz differ diff --git a/resources/samples/input1_Mollweide.pickle b/resources/samples/input1_Mollweide.pickle deleted file mode 100644 index 2d570f8..0000000 Binary files a/resources/samples/input1_Mollweide.pickle and /dev/null differ diff --git a/resources/samples/input1_Mollweide.pickle.gz b/resources/samples/input1_Mollweide.pickle.gz new file mode 100644 index 0000000..f3ad157 Binary files /dev/null and b/resources/samples/input1_Mollweide.pickle.gz differ diff --git a/resources/samples/input2_LatLon.pickle b/resources/samples/input2_LatLon.pickle deleted file mode 100644 index 6e83690..0000000 Binary files a/resources/samples/input2_LatLon.pickle and /dev/null differ diff --git a/resources/samples/input2_LatLon.pickle.gz b/resources/samples/input2_LatLon.pickle.gz new file mode 100644 index 0000000..b2c354e Binary files /dev/null and b/resources/samples/input2_LatLon.pickle.gz differ diff --git a/resources/samples/input2_Mollweide.pickle b/resources/samples/input2_Mollweide.pickle deleted file mode 100644 index 676dca4..0000000 Binary files a/resources/samples/input2_Mollweide.pickle and /dev/null differ diff --git a/resources/samples/input2_Mollweide.pickle.gz b/resources/samples/input2_Mollweide.pickle.gz new file mode 100644 index 0000000..f827075 Binary files /dev/null and b/resources/samples/input2_Mollweide.pickle.gz differ diff --git a/resources/samples/input3_LatLon.pickle b/resources/samples/input3_LatLon.pickle deleted file mode 100644 index 36cdbcb..0000000 Binary files a/resources/samples/input3_LatLon.pickle and /dev/null differ diff --git a/resources/samples/input3_LatLon.pickle.gz b/resources/samples/input3_LatLon.pickle.gz new file mode 100644 index 0000000..83c8759 Binary files /dev/null and b/resources/samples/input3_LatLon.pickle.gz differ diff --git a/resources/samples/input3_Mollweide.pickle b/resources/samples/input3_Mollweide.pickle deleted file mode 100644 index 029d52c..0000000 Binary files a/resources/samples/input3_Mollweide.pickle and /dev/null differ diff --git a/resources/samples/input3_Mollweide.pickle.gz b/resources/samples/input3_Mollweide.pickle.gz new file mode 100644 index 0000000..b045470 Binary files /dev/null and b/resources/samples/input3_Mollweide.pickle.gz differ diff --git a/resources/samples/input4_LatLon.pickle b/resources/samples/input4_LatLon.pickle deleted file mode 100644 index c445535..0000000 Binary files a/resources/samples/input4_LatLon.pickle and /dev/null differ diff --git a/resources/samples/input4_LatLon.pickle.gz b/resources/samples/input4_LatLon.pickle.gz new file mode 100644 index 0000000..2256d21 Binary files /dev/null and b/resources/samples/input4_LatLon.pickle.gz differ diff --git a/resources/samples/input4_Mollweide.pickle b/resources/samples/input4_Mollweide.pickle deleted file mode 100644 index 8d4a6e4..0000000 Binary files a/resources/samples/input4_Mollweide.pickle and /dev/null differ diff --git a/resources/samples/input4_Mollweide.pickle.gz b/resources/samples/input4_Mollweide.pickle.gz new file mode 100644 index 0000000..eee012f Binary files /dev/null and b/resources/samples/input4_Mollweide.pickle.gz differ diff --git a/sdk/moveapps_execution.py b/sdk/moveapps_execution.py index 4d0f854..20d6a98 100644 --- a/sdk/moveapps_execution.py +++ b/sdk/moveapps_execution.py @@ -1,4 +1,3 @@ -import pandas as pd import json import os import logging @@ -6,6 +5,8 @@ from dotenv import load_dotenv from dataclasses import dataclass +from sdk.moveapps_pickle import MoveAppsPickle + @dataclass class Environment: @@ -35,7 +36,7 @@ def execute(self): def __load_environment(self): self.env = Environment( source_file=os.environ.get('SOURCE_FILE'), - output_file=os.environ.get('OUTPUT_FILE', 'resources/output/output.pickle'), + output_file=os.environ.get('OUTPUT_FILE', 'resources/output/output.pickle.gz'), error_file=os.environ.get('ERROR_FILE', 'resources/output/error.txt'), app_configuration=self.__load_config() ) @@ -49,7 +50,7 @@ def __configure_logging(): ) def __load_input(self): - return pd.read_pickle(self.env.source_file) + return MoveAppsPickle.read(path=self.env.source_file) @staticmethod def __load_config(): @@ -77,7 +78,7 @@ def __load_config(): def __store_output(self, data): logging.info(f'storing output: {data}') - pd.to_pickle(data, self.env.output_file) + MoveAppsPickle.write(data=data, path=self.env.output_file) def __store_error(self, error: Exception): logging.info(f'storing error to {self.env.error_file}') diff --git a/sdk/moveapps_pickle.py b/sdk/moveapps_pickle.py new file mode 100644 index 0000000..b9ed7a5 --- /dev/null +++ b/sdk/moveapps_pickle.py @@ -0,0 +1,52 @@ +import logging + +import pandas as pd + +GZIP_MAGIC = b'\x1f\x8b' + + +class MoveAppsPickle: + """ + Reads and writes the pickle files that carry the data from one App to the next. + + MoveApps dictates the input and output paths and they carry no file extension, so pandas cannot + infer the compression from the file name. Writing therefore states the compression explicitly, + and reading determines it from the file content. + """ + + @staticmethod + def read(path: str): + """ + Reads a pickle, compressed or not. + + Apps built against an SDK before `v3` write uncompressed pickles, and such output can still + reach a current App as its input. Both flavours therefore have to stay readable. + + :param path: path to the pickle to read + :return: the unpickled data + """ + compression = MoveAppsPickle.detect_compression(path=path) + logging.info(f'reading pickle \'{path}\' (compression: {compression})') + return pd.read_pickle(path, compression=compression) + + @staticmethod + def write(data, path: str) -> None: + """ + Writes a gzip-compressed pickle, whatever the file is named. + + :param data: the data to pickle + :param path: path to write the pickle to + """ + pd.to_pickle(data, path, compression='gzip') + + @staticmethod + def detect_compression(path: str) -> str | None: + """ + Determines the compression of a pickle from its first bytes instead of from its name. + + :param path: path to the pickle to inspect + :return: `'gzip'` for a gzip-compressed file, otherwise `None`, which is how pandas is told + not to decompress at all + """ + with open(path, 'rb') as probe: + return 'gzip' if probe.read(len(GZIP_MAGIC)) == GZIP_MAGIC else None diff --git a/tests/app/test_app.py b/tests/app/test_app.py index cb0ac74..2cf7fc3 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -15,7 +15,7 @@ def setUp(self) -> None: def test_app_runs(self): # prepare - data: mpd.TrajectoryCollection = pd.read_pickle(os.path.join(ROOT_DIR, 'tests/resources/app/input4_LatLon.pickle')) + data: mpd.TrajectoryCollection = pd.read_pickle(os.path.join(ROOT_DIR, 'tests/resources/app/input4_LatLon.pickle.gz')) config: dict = { "year": 2014 } @@ -124,7 +124,7 @@ def test_year_not_present(self): # Use this test if the App should return the input data def test_app_returns_input(self): # prepare - expected: mpd.TrajectoryCollection = pd.read_pickle(os.path.join(ROOT_DIR, 'tests/resources/app/input2_LatLon.pickle')) + expected: mpd.TrajectoryCollection = pd.read_pickle(os.path.join(ROOT_DIR, 'tests/resources/app/input2_LatLon.pickle.gz')) config: dict = {} # execute diff --git a/tests/resources/app/input2_LatLon.pickle b/tests/resources/app/input2_LatLon.pickle deleted file mode 100644 index dcfae12..0000000 Binary files a/tests/resources/app/input2_LatLon.pickle and /dev/null differ diff --git a/tests/resources/app/input2_LatLon.pickle.gz b/tests/resources/app/input2_LatLon.pickle.gz new file mode 100644 index 0000000..8fbbc8f Binary files /dev/null and b/tests/resources/app/input2_LatLon.pickle.gz differ diff --git a/tests/resources/app/input4_LatLon.pickle b/tests/resources/app/input4_LatLon.pickle deleted file mode 100644 index c445535..0000000 Binary files a/tests/resources/app/input4_LatLon.pickle and /dev/null differ diff --git a/tests/resources/app/input4_LatLon.pickle.gz b/tests/resources/app/input4_LatLon.pickle.gz new file mode 100644 index 0000000..0db4a7e Binary files /dev/null and b/tests/resources/app/input4_LatLon.pickle.gz differ diff --git a/tests/sdk/test_moveapps_pickle.py b/tests/sdk/test_moveapps_pickle.py new file mode 100644 index 0000000..d6795e9 --- /dev/null +++ b/tests/sdk/test_moveapps_pickle.py @@ -0,0 +1,106 @@ +import os +import tempfile +from unittest import TestCase + +import pandas as pd + +from sdk.moveapps_pickle import MoveAppsPickle +from tests.config.definitions import ROOT_DIR + + +class TestMoveAppsPickle(TestCase): + """ + MoveApps hands the App its input and output paths without a file extension, so none of these + cases may rely on the file name to tell compressed from uncompressed data. + """ + + def setUp(self) -> None: + self.sut = MoveAppsPickle() + self.tmp = tempfile.TemporaryDirectory() + self.data = pd.DataFrame({'id': range(5_000), 'note': ['a note about this record'] * 5_000}) + + def tearDown(self) -> None: + self.tmp.cleanup() + + def path(self, file_name: str) -> str: + return os.path.join(self.tmp.name, file_name) + + def test_writes_compressed_although_the_file_name_has_no_extension(self): + # prepare + target = self.path('output_file') + + # execute + self.sut.write(data=self.data, path=target) + + # verify + with open(target, 'rb') as written: + actual = written.read(2) + self.assertEqual(b'\x1f\x8b', actual) + + def test_written_output_is_substantially_smaller_than_uncompressed(self): + # prepare + compressed = self.path('compressed') + uncompressed = self.path('uncompressed') + pd.to_pickle(self.data, uncompressed, compression=None) + + # execute + self.sut.write(data=self.data, path=compressed) + + # verify + actual = os.path.getsize(compressed) + self.assertLess(actual, os.path.getsize(uncompressed) / 2) + + def test_reads_back_what_it_wrote(self): + # prepare + target = self.path('output_file') + self.sut.write(data=self.data, path=target) + + # execute + actual = self.sut.read(path=target) + + # verify + pd.testing.assert_frame_equal(self.data, actual) + + def test_reads_an_uncompressed_pickle_written_by_an_older_sdk(self): + # prepare: Apps built against an SDK before v3 wrote uncompressed pickles, and such output + # can still reach a current App as its input. + legacy = self.path('output_file') + pd.to_pickle(self.data, legacy, compression=None) + + # execute + actual = self.sut.read(path=legacy) + + # verify + pd.testing.assert_frame_equal(self.data, actual) + + def test_reads_a_sample_file_carrying_a_gz_extension(self): + # prepare: during App development the SDK is pointed at the shipped samples via `SOURCE_FILE` + sample = os.path.join(ROOT_DIR, 'tests/resources/app/input4_LatLon.pickle.gz') + + # execute + actual = self.sut.read(path=sample) + + # verify + self.assertIsNotNone(actual) + + def test_detects_gzip_by_content(self): + # prepare + compressed = self.path('compressed') + pd.to_pickle(self.data, compressed, compression='gzip') + + # execute + actual = self.sut.detect_compression(path=compressed) + + # verify + self.assertEqual('gzip', actual) + + def test_detects_absence_of_compression_by_content(self): + # prepare + uncompressed = self.path('uncompressed') + pd.to_pickle(self.data, uncompressed, compression=None) + + # execute + actual = self.sut.detect_compression(path=uncompressed) + + # verify: `None` is what pandas expects for "do not decompress" + self.assertIsNone(actual) diff --git a/utils/input_converter.py b/utils/input_converter.py index 46b6b1a..1beddfc 100644 --- a/utils/input_converter.py +++ b/utils/input_converter.py @@ -63,5 +63,5 @@ def write_result(self, file_name, data): file = f'input{i}' converter.csv_to_pickle( csv_path=f'./resources/input/{file}', - result_file_name=f'{ROOT_DIR}/resources/samples/{file}.pickle' + result_file_name=f'{ROOT_DIR}/resources/samples/{file}.pickle.gz' )