diff --git a/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/binding_worker.err b/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/binding_worker.err new file mode 100644 index 00000000..e69de29b diff --git a/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/binding_worker.out b/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/binding_worker.out new file mode 100644 index 00000000..e69de29b diff --git a/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/worker_localhost.err b/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/worker_localhost.err new file mode 100644 index 00000000..e69de29b diff --git a/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/worker_localhost.out b/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/worker_localhost.out new file mode 100644 index 00000000..961bc30f --- /dev/null +++ b/examples/dds/.COMPSsWorker/57ea0678-a813-4399-b18f-b0bfaa5ac187/localhost/log/worker_localhost.out @@ -0,0 +1,3 @@ +[(202)(2025-02-17 09:05:14,059) Worker] @ - No valid hostID provided to the tracing system. Provided ID: localhost +[(204)(2025-02-17 09:05:14,061) Worker] @init - No streaming backend passed +[(205)(2025-02-17 09:05:14,062) Worker] @init - No storage configuration file passed diff --git a/examples/dds/cfg b/examples/dds/cfg new file mode 100644 index 00000000..4c5c1832 --- /dev/null +++ b/examples/dds/cfg @@ -0,0 +1 @@ +{"working_dir":"/home/dcanocan/Desktop/Curro/git/dataclay/examples/dds","resources":"","project":""} \ No newline at end of file diff --git a/examples/dds/client.py b/examples/dds/client.py new file mode 100644 index 00000000..c1ce4265 --- /dev/null +++ b/examples/dds/client.py @@ -0,0 +1,36 @@ +import sys +print("Python executable:", sys.executable) +print("Python version:", sys.version) +print("Python path:", sys.path) + + + +from pycompss.dds import DDS +#from dataclay import Client +import dataclay +#from pycompss.api.task import task + +#@task() +def _filter(dds): + print("##Filter numbers##") + even_numbers = dds.filter( lambda x : x % 2 == 0 ).collect() + return even_numbers + +def main(): + print(dir(dataclay)) + #client = Client(host="127.0.0.1", username="testuser", password="s3cret", dataset="testdata") + #client.start() + + print("##Creating DDS object and making it persistent##") + data = range(10) + dds = DDS().load(data) + #dds.make_persistent("dds") + + even_numbers=_filter(dds) + + print("##Print##") + print(even_numbers) + +if __name__ =='__main__': + main() + \ No newline at end of file diff --git a/examples/dds/docker-compose.yml b/examples/dds/docker-compose.yml new file mode 100644 index 00000000..e83cc9d4 --- /dev/null +++ b/examples/dds/docker-compose.yml @@ -0,0 +1,41 @@ +services: + + redis: + image: redis:latest + ports: + - 6379:6379 + + metadata-service: + image: "ghcr.io/bsc-dom/dataclay:dev" + depends_on: + - redis + ports: + - 16587:16587 + environment: + - DATACLAY_KV_HOST=redis + - DATACLAY_KV_PORT=6379 + - DATACLAY_ID + - DATACLAY_PASSWORD=s3cret + - DATACLAY_USERNAME=testuser + - DATACLAY_DATASET=testdata + - DATACLAY_METADATA_PORT=16587 + - DATACLAY_LOGLEVEL=DEBUG + command: python -m dataclay.metadata + + backend: + image: "ghcr.io/bsc-dom/dataclay:dev" + depends_on: + - redis + ports: + - 6867:6867 + environment: + - DATACLAY_KV_HOST=redis + - DATACLAY_KV_PORT=6379 + - DATACLAY_BACKEND_ID + - DATACLAY_BACKEND_NAME + - DATACLAY_BACKEND_PORT=6867 + - DATACLAY_LOGLEVEL=DEBUG + command: python -m dataclay.backend + volumes: + #- ./model:/workdir/model:ro + - ./pycompss/:/workdir/pycompss/:ro \ No newline at end of file diff --git a/examples/dds/master/client.py b/examples/dds/master/client.py new file mode 100644 index 00000000..f47eda80 --- /dev/null +++ b/examples/dds/master/client.py @@ -0,0 +1,28 @@ +from pycompss.dds import DDS +#from dataclay import Client +import dataclay +from pycompss.api.task import task + +@task(returns=1) +def _filter(dds): + print("##Filter numbers##") + even_numbers = dds.filter( lambda x : x % 2 == 0 ).collect() + return even_numbers + +def main(): + print(dir(dataclay)) + #client = Client(host="127.0.0.1", username="testuser", password="s3cret", dataset="testdata") + #client.start() + + print("##Creating DDS object and making it persistent##") + data = range(10) + dds = DDS().load(data) + dds.make_persistent("dds") + + even_numbers=_filter(dds) + + print("##Print##") + print(even_numbers) + +if __name__ =='__main__': + main() \ No newline at end of file diff --git a/examples/dds/master/docker-compose.yml b/examples/dds/master/docker-compose.yml new file mode 100644 index 00000000..a6cec428 --- /dev/null +++ b/examples/dds/master/docker-compose.yml @@ -0,0 +1,40 @@ +services: + + redis: + image: redis:latest + ports: + - 6379:6379 + + metadata-service: + image: "ghcr.io/bsc-dom/dataclay:dev" + depends_on: + - redis + ports: + - 16587:16587 + environment: + - DATACLAY_KV_HOST=redis + - DATACLAY_KV_PORT=6379 + - DATACLAY_ID + - DATACLAY_PASSWORD=s3cret + - DATACLAY_USERNAME=testuser + - DATACLAY_DATASET=testdata + - DATACLAY_METADATA_PORT=16587 + - DATACLAY_LOGLEVEL=DEBUG + command: python -m dataclay.metadata + + backend: + image: "ghcr.io/bsc-dom/dataclay:dev" + depends_on: + - redis + ports: + - 6867:6867 + environment: + - DATACLAY_KV_HOST=redis + - DATACLAY_KV_PORT=6379 + - DATACLAY_BACKEND_ID + - DATACLAY_BACKEND_NAME + - DATACLAY_BACKEND_PORT=6867 + - DATACLAY_LOGLEVEL=DEBUG + command: python -m dataclay.backend + volumes: + - ./model:/workdir/model:ro diff --git a/examples/dds/model/dds.py b/examples/dds/model/dds.py new file mode 100644 index 00000000..65693ed4 --- /dev/null +++ b/examples/dds/model/dds.py @@ -0,0 +1,1064 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - API. + +This file contains the DDS interface. +""" + +import heapq +import bisect +import itertools +import functools +import os +from collections import defaultdict +from collections import deque +from dataclay import DataClayObject#,activemethod + +from pycompss.api.api import compss_wait_on +from pycompss.api.api import compss_delete_object +from pycompss.api.api import compss_barrier +from pycompss.dds.partition_generators import IPartitionGenerator +from pycompss.dds.partition_generators import BasicDataLoader +from pycompss.dds.partition_generators import IteratorLoader +from pycompss.dds.partition_generators import WorkerFileLoader +from pycompss.dds.partition_generators import PickleLoader +from pycompss.dds.partition_generators import read_in_chunks +from pycompss.dds.partition_generators import read_lines +from pycompss.dds.tasks import map_partition +from pycompss.dds.tasks import distribute_partition +from pycompss.dds.tasks import reduce_dicts +from pycompss.dds.tasks import task_dict_to_list +from pycompss.dds.tasks import reduce_multiple +from pycompss.dds.tasks import task_collect_samples +from pycompss.dds.tasks import map_and_save_text_file +from pycompss.dds.tasks import map_and_save_pickle +from pycompss.dds.tasks import MARKER +from pycompss.dds.utils import default_hash +from pycompss.util.tracing.helpers import EventMaster + + +#class DDS(): # pylint: disable=too-many-public-methods +class DDS(DataClayObject): # pylint: disable=too-many-public-methods + """Distributed Data Set object.""" + + #@activemethod + def __init__(self): + """Create a new DDS object.""" + super().__init__() + self.partitions = [] + self.func = None + + # Partition As A Collection + # True if partitions are not Future Objects but list of Future Objects + self.paac = False + + #@activemethod + def load(self, iterator, num_of_parts=10, paac=False): + """Load and distribute the iterator on partitions. + + :param iterator: Partitions iterator. + :param num_of_parts: Number of parts. + :param paac: Partition as a collection. + :returns: Self. + """ + self.paac = paac + if num_of_parts == -1: + self.partitions = iterator + return self + + total = len(iterator) + if not total: + return self + + chunk_sizes = [(total // num_of_parts)] * num_of_parts + extras = total % num_of_parts + for i in range(extras): + chunk_sizes[i] += 1 + + start = 0 + for size in chunk_sizes: + end = start + size + _partition_loader = IteratorLoader(iterator, start, end) + self.partitions.append(_partition_loader) + start = end + + return self + + #@activemethod + def load_file(self, file_path, chunk_size=1024, worker_read=False): + """Read file in chunks and save it onto partitions. + + Usage sample: + >>> with open("test.file", "w") as testFile: + ... _ = testFile.write("Hello world!") + >>> DDS().load_file("test.file", 6).collect() + ['Hello ', 'world!'] + + :param file_path: A path to a file to be loaded. + :param chunk_size: Size of chunks in bytes. + :param worker_read: If reading the file in the worker + (skips first bytes). + :return: Self. + """ + if worker_read: + with open(file_path) as file_path_fd: # pylint: disable=W1514 + file_path_fd.seek(0, 2) + total = file_path_fd.tell() + parsed = 0 + while parsed < total: + _partition_loader = WorkerFileLoader( + [file_path], + single_file=True, + start=parsed, + chunk_size=chunk_size, + ) + self.partitions.append(_partition_loader) + parsed += chunk_size + else: + with open(file_path, "r") as file_path_fd: # pylint: disable=W1514 + chunk = file_path_fd.read(chunk_size) + while chunk: + _partition_loader = BasicDataLoader(chunk) + self.partitions.append(_partition_loader) + chunk = file_path_fd.read(chunk_size) + + return self + + #@activemethod + def load_text_file( + self, file_name, chunk_size=1024, in_bytes=True, strip=True + ): + r"""Load a text file into partitions with 'chunk_size' lines on each. + + Usage sample: + >>> with open("test.txt", "w") as testFile: + ... _ = testFile.write("First Line! \n") + ... _ = testFile.write("Second Line! \n") + >>> DDS().load_text_file("test.txt").collect() + ['First Line! ', 'Second Line! '] + + :param file_name: A path to a file to be loaded. + :param chunk_size: Size of chunks in bytes. + :param in_bytes: If chunk size is in bytes or in number of lines. + :param strip: If line separators should be stripped from lines. + :return: Self. + """ + func = read_in_chunks if in_bytes else read_lines + + for _p in func(file_name, chunk_size, strip=strip): + partition_loader = BasicDataLoader(_p) + self.partitions.append(partition_loader) + + return self + + #@activemethod + def load_files_from_dir(self, dir_path, num_of_parts=-1): + """Read multiple files from a given directory. + + Each file and its content is saved in a tuple in ('file_path', + 'file_content') format. + + :param dir_path: A directory that all files will be loaded from. + :param num_of_parts: Can be set to -1 to create one partition per file. + :return: Self. + """ + files = sorted(os.listdir(dir_path)) + total = len(files) + num_of_parts = total if num_of_parts < 0 else num_of_parts + partition_sizes = [(total // num_of_parts)] * num_of_parts + extras = total % num_of_parts + for i in range(extras): + partition_sizes[i] += 1 + + start = 0 + for size in partition_sizes: + end = start + size + partition_files = [] + for file_name in files[start:end]: + file_path = os.path.join(dir_path, file_name) + partition_files.append(file_path) + + _partition_loader = WorkerFileLoader(partition_files) + self.partitions.append(_partition_loader) + start = end + + return self + + #@activemethod + def load_pickle_files(self, dir_path): + """Load serialized partitions from pickle files. + + :param dir_path: Path to serialized partitions. + :return: Self. + """ + files = sorted(os.listdir(dir_path)) + for _f in files: + file_name = os.path.join(dir_path, _f) + _partition_loader = PickleLoader(file_name) + self.partitions.append(_partition_loader) + + return self + + #@activemethod + def union(self, *args): + """Combine this data set with some other DDS data. + + Usage sample: + >>> first = DDS().load([0, 1, 2, 3, 4], 2) + >>> second = DDS().load([5, 6, 7, 8, 9], 3) + >>> first.union(second).count() + 10 + + :param args: Arbitrary amount of DDS objects. + :return: New DDS object combining two DDS objects. + """ + current = list(self.collect(future_objects=True)) + for dds in args: + temp = list(dds.collect(future_objects=True)) + current.extend(temp) + + return DDS().load(current, num_of_parts=-1) + + #@activemethod + def num_of_partitions(self): + """Get the total amount of partitions. + + Usage sample: + >>> DDS().load(range(10), 5).num_of_partitions() + 5 + + :return: Number of partitions. + """ + return len(self.partitions) + + #@activemethod + def map(self, func, *args, **kwargs): + """Apply the given function to each element of the dataset. + + Usage sample: + >>> dds = DDS().load(range(10), 5).map(lambda x: x * 2) + >>> sorted(dds.collect()) + [0, 2, 4, 6, 8, 10, 12, 14, 16, 18] + + :param func: Function to apply. + :param args: Arguments. + :param kwargs: Keyword arguments. + :returns: New child DDS object. + """ + + def mapper(partition): + results = [] + for element in partition: + results.append(func(element, *args, **kwargs)) + return results + + return _ChildDDS(self, mapper) + + #@activemethod + def map_partitions(self, func): + """Apply a function to each partition of this data set. + + Usage sample: + >>> DDS().load(range(10), 5).map_partitions( + ... lambda x: [sum(x)] + ... ).collect(True) + [[1], [5], [9], [13], [17]] + + :param func: Function to apply. + :returns: New child DDS object. + """ + return _ChildDDS(self, func) + + #@activemethod + def flat_map(self, func, *args, **kwargs): + """Apply a function to each element of the dataset. + + NOTE: Extends the derived element(s) if possible. + + Usage sample: + >>> dds = DDS().load([2, 3, 4]) + >>> sorted(dds.flat_map(lambda x: range(1, x)).collect()) + [1, 1, 1, 2, 2, 3] + + :param func: A function that should return a list, tuple or + another kind of iterable. + :param args: Arguments. + :param kwargs: Keyword arguments. + :returns: New child DDS object. + """ + + def mapper(iterator): + res = [] + for item in iterator: + res.extend(func(item, *args, **kwargs)) + return res + + return self.map_partitions(mapper) + + #@activemethod + def filter(self, func): + """Filter elements of this data set by applying a given function. + + Usage sample: + >>> DDS().load(range(10), 5).filter(lambda x: x % 2).count() + 5 + + :param func: Filtering function. + :returns: New child DDS object filtered. + """ + + def _filter(iterator): + return filter(func, iterator) + + return self.map_partitions(_filter) + + #@activemethod + def reduce(self, func, initial=MARKER, arity=-1): + """Reduce the whole data set. + + Usage sample: + >>> DDS().load(range(10), 5).reduce((lambda b, a: b + a) , 100) + 145 + + :param func: A reduce function which should take two parameters as + inputs and return a single result which will be sent to + itself again. + :param initial: Initial value for reducer which will be used to reduce + the first element with. + :param arity: Tree depth. + :return: Reduced result (inside a DDS if necessary). + """ + + def local_reducer(partition): + """Reduce a partition and retrieve it as a one element partition. + + :param partition: Partition. + :return: One element partition. + """ + iterator = iter(partition) + try: + init = next(iterator) + except StopIteration: + return [] + + return [functools.reduce(func, iterator, init)] + + local_results = self.map_partitions(local_reducer).collect( + future_objects=True + ) + + local_results = deque(local_results) + + # If initial value is set, add it to the list as well + if initial != MARKER: + local_results.append([initial]) + + arity = arity if arity > 0 else len(self.partitions) + branch = [] + + while local_results: + while local_results and len(branch) < arity: + temp = local_results.popleft() + branch.append(temp) + + if len(branch) == 1: + branch = compss_wait_on(branch[0]) + break + + temp = reduce_multiple(func, branch) + local_results.append(temp) + branch = [] + + return branch[0] + + #@activemethod + def distinct(self): + """Get the distinct elements of this data set. + + Usage sample: + >>> test = list(range(10)) + >>> test.extend(list(range(5))) + >>> len(test) + 15 + >>> DDS().load(test, 5).distinct().count() + 10 + + :returns: New child DDS object with distinct elements. + """ + return ( + self.map(lambda x: (x, None)) + .reduce_by_key(lambda x, _: x) + .map(lambda x: x[0]) + ) + + #@activemethod + def count_by_value(self, arity=2, as_dict=True, as_fo=False): + """Amount of each element on this data set. + + Usage sample: + >>> first = DDS().load([0, 1, 2], 2) + >>> second = DDS().load([2, 3, 4], 3) + >>> dict(sorted( + ... first.union(second).count_by_value(as_dict=True).items() + ... )) + {0: 1, 1: 1, 2: 2, 3: 1, 4: 1} + + :param arity: Tree depth. + :param as_dict: As dictionary. + :param as_fo: As future object. + :return: List of tuples (element, number). + """ + + def count_partition(iterator): + counts = defaultdict(int) + for obj in iterator: + counts[obj] += 1 + return counts + + # Count locally and create dictionary partitions + local_results = self.map_partitions(count_partition).collect( + future_objects=True + ) + + # Create a deque from partitions and start reduce + future_objects = deque(local_results) + + branch = [] + while future_objects: + branch = [] + while future_objects and len(branch) < arity: + temp = future_objects.popleft() + branch.append(temp) + + if len(branch) == 1: + break + + first, branch = branch[0], branch[1:] + reduce_dicts(first, branch) + future_objects.append(first) + + if as_dict: + if as_fo: + return branch[0] + branch[0] = compss_wait_on(branch[0]) + return dict(branch[0]) + + length = self.num_of_partitions() + new_partitions = [] + for i in range(length): + new_partitions.append(task_dict_to_list(branch[0], length, i)) + + return DDS().load(new_partitions, -1) + + #@activemethod + def key_by(self, func): + """Create a (key,value) pair for each element where 'key' is f(value). + + Usage sample: + >>> dds = DDS().load(range(3), 2) + >>> dds.key_by(lambda x: str(x)).collect() + [('0', 0), ('1', 1), ('2', 2)] + + :param func: A Key Creator function which takes the element as a + parameter and returns the key. + :return: List of (key, value) pairs. + """ + return self.map(lambda x: (func(x), x)) + + #@activemethod + def sum(self): + """Sum everything up. + + Usage sample: + >>> DDS().load(range(3), 2).sum() + 3 + + :returns: The sum of everything. + """ + return sum(self.map_partitions(lambda x: [sum(x)]).collect()) + + #@activemethod + def count(self): + """Count everything up. + + Usage sample: + >>> DDS().load(range(3), 2).count() + 3 + + :return: Total number of elements. + """ + return self.map_partitions(lambda i: [sum(1 for _ in i)]).sum() + + #@activemethod + def foreach(self, func): + """Apply a function to each element of this data set. + + CAUTION: Does not return anything. + + :param func: A void function. + :returns: None + """ + self.map(func) + # Wait for all the tasks to finish + compss_barrier() + + #@activemethod + def collect( + self, # pylint: disable=R0912 + keep_partitions=False, + future_objects=False, + ): + """Return all elements from all partitions. + + Elements can be grouped by partitions by setting keep_partitions value + as True. + + Usage sample: + >>> dds = DDS().load(range(10), 2) + >>> dds.collect(True) + [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] + >>> DDS().load(range(10), 2).collect() + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + :param keep_partitions: Keep Partitions? + :param future_objects: Future objects? + :return: All elements from all partitions. + """ + processed = [] + if self.func: + if self.paac: + for col in self.partitions: + processed.append(map_partition(self.func, None, col)) + else: + for _p in self.partitions: + processed.append(map_partition(self.func, _p)) + # Reset the function! + self.func = None + else: + for _p in self.partitions: + if isinstance(_p, IPartitionGenerator): + processed.append(_p.retrieve_data()) + else: + processed.append(_p) + + # Future objects cannot be extended for now... + if future_objects: + return processed + + processed = compss_wait_on(processed) + + ret = [] + if not keep_partitions: + for _pp in processed: + ret.extend(_pp) + else: + for _pp in processed: + ret.append(list(_pp)) + return ret + + #@activemethod + def save_as_text_file(self, path): + """Save string representations of DDS elements as text files. + + This saving creates one file per partition. + + :param path: Destination file path. + :return: None. + """ + if self.paac: + for i, _p in enumerate(self.partitions): + map_and_save_text_file(self.func, i, path, None, _p) + compss_delete_object(_p) + else: + for i, _p in enumerate(self.partitions): + map_and_save_text_file(self.func, i, path, _p) + + #@activemethod + def save_as_pickle(self, path): + """Save partitions of this DDS as pickle files. + + Each partition is saved as a separate file for the sake of parallelism. + + :param path:Destination file path. + :return: None. + """ + if self.paac: + for i, _p in enumerate(self.partitions): + map_and_save_pickle(self.func, i, path, None, _p) + else: + for i, _p in enumerate(self.partitions): + map_and_save_pickle(self.func, i, path, _p) + + # ################################################################ # + # ############## Functions for (Key, Value) pairs. ############### # + # ################################################################ # + + #@activemethod + def collect_as_dict(self): + """Get (key,value) as { key: value }. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).collect_as_dict() + {'a': 1, 'b': 1} + + :return: Dict. + """ + return dict(self.collect()) + + #@activemethod + def keys(self): + """Get keys. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).keys().collect() + ['a', 'b'] + + :return: List of keys. + """ + return self.map(lambda x: x[0]) + + #@activemethod + def values(self): + """Get values. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 2)]).values().collect() + [1, 2] + + :return: List of values. + """ + return self.map(lambda x: x[1]) + + #@activemethod + def partition_by( + self, partitioner_func=default_hash, num_of_partitions=-1 + ): + """Create partitions by a Partition Func. + + Usage sample: + >>> dds = DDS().load(range(6)).map(lambda x: (x, x)) + >>> dds.partition_by(num_of_partitions=3).collect(True) + [[(0, 0), (3, 3)], [(1, 1), (4, 4)], [(2, 2), (5, 5)]] + + :param partitioner_func: A Function distribute data on partitions based + on for example, hash function. + :param num_of_partitions: Number of partitions to be created. + :return: Partitions. + """ + + def combine_lists(_partition): + # Elements of the partition are grouped by their + # previous partitions + ret = [] + for _li in _partition: + ret.extend(_li) + return ret + + nop = ( + len(self.partitions) + if num_of_partitions == -1 + else num_of_partitions + ) + + grouped = defaultdict(list) + + if self.paac: + for collection in self.partitions: + col = [[] for _ in range(nop)] + with EventMaster(3002): + distribute_partition( + col, self.func, partitioner_func, None, collection + ) + compss_delete_object(collection) + for _i in range(nop): + grouped[_i].append(col[_i]) + else: + for _part in self.partitions: + col = [[] for _ in range(nop)] + with EventMaster(3002): + distribute_partition( + col, self.func, partitioner_func, _part + ) + for _i in range(nop): + grouped[_i].append(col[_i]) + + future_partitions = [] + for key in sorted(grouped.keys()): + future_partitions.append(grouped[key]) + + return ( + DDS() + .load(future_partitions, -1, True) + .map_partitions(combine_lists) + ) + + #@activemethod + def map_values(self, func): + """Apply a function to each value of (k, v) element of this data set. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).map_values( + ... lambda x: x+1 + ... ).collect() + [('a', 2), ('b', 2)] + + :param func: A function which takes 'value's as parameter. + :return: New DDS. + """ + + def dummy(pair): + return pair[0], func(pair[1]) + + return self.map(dummy) + + #@activemethod + def flatten_by_key(self, func): + """Reverse of combine by key.Flat (k, v) as (k, v1), (k, v2) etc. + + In detail: (key, values) as (key, value1), (key, value2) ... + + Usage sample: + >>> DDS().load([('a',[1, 2]), ('b',[1])]).flatten_by_key( + ... lambda x: x + ... ).collect() + [('a', 1), ('a', 2), ('b', 1)] + + :param func: A function to parse values. + :return: Flattened by key. + """ + + def dummy(key_value): + return ((key_value[0], x) for x in func(key_value[1])) + + return self.flat_map(dummy) + + #@activemethod + def join(self, other, num_of_partitions=-1): + """Join DDS objects. + + Usage sample: + >>> x = DDS().load([("a", 1), ("b", 3)]) + >>> y = DDS().load([("a", 2), ("b", 4)]) + >>> sorted(x.join(y).collect()) + [('a', (1, 2)), ('b', (3, 4))] + + :param other: Another DDS object. + :param num_of_partitions: Number of partitions. + :return: Joined DDS objects. + """ + + def dispatch(seq): + buf_1, buf_2 = [], [] + for num, value in seq: + if num == 1: + buf_1.append(value) + elif num == 2: + buf_2.append(value) + return [(v, w) for v in buf_1 for w in buf_2] + + nop = ( + len(self.partitions) + if num_of_partitions == -1 + else num_of_partitions + ) + + buf_a = self.map_values(lambda v: (1, v)) + buf_b = other.map_values(lambda y: (2, y)) + + return ( + buf_a.union(buf_b) + .group_by_key(num_of_parts=nop) + .flatten_by_key(lambda x: dispatch(iter(x))) + ) + + #@activemethod + def combine_by_key( + self, creator_func, combiner_func, merger_function, total_parts=-1 + ): + """Combine elements of each key. + + :param creator_func: To apply to the first element of the key. Takes + only one argument which is the value from (k, v) + pair. (e.g: v = list(v)). + :param combiner_func: To apply when a new element with the same 'key' + is found. It is used to combine partitions + locally. Takes 2 arguments; first one is the + result of 'creator_func' where the second one + is a 'value' of the same 'key' from the same + partition. (e.g: v1.append(v2)). + :param merger_function: To merge local results. Basically takes two + arguments -both are results of 'combiner_func'. + (e.g: list_1.extend(list_2)). + :param total_parts: Number of partitions after combinations. + :return: Combined by key DDS object. + """ + + def combine_partition(partition): + """Combine partitions. + + :param partition: Dictionary of partitions. + :returns: List of combined partitions. + """ + res = {} + for key, val in partition: + res[key] = ( + combiner_func(res[key], val) + if key in res + else creator_func(val) + ) + return list(res.items()) + + def merge_partition(partition): + """Merge partitions. + + :param partition: Dictionary of partitions. + :returns: List of merged partitions. + """ + res = {} + for key, val in partition: + res[key] = ( + merger_function(res[key], val) if key in res else val + ) + return list(res.items()) + + ret = ( + self.map_partitions(combine_partition) + .partition_by(num_of_partitions=total_parts) + .map_partitions(merge_partition) + ) + + return ret + + #@activemethod + def reduce_by_key(self, func): + """Reduce values for each key. + + Usage sample: + >>> DDS().load([("a",1), ("a",2)]).reduce_by_key( + ... (lambda a, b: a+b) + ... ).collect() + [('a', 3)] + + :param func: a reducer function which takes two parameters and + returns one. + :returns: Reduced values. + """ + return self.combine_by_key((lambda x: x), func, func) + + #@activemethod + def count_by_key(self, as_dict=False): + """Count by key. + + Usage sample: + >>> DDS().load([("a", 100), ("a", 200)]).count_by_key(True) + {'a': 2} + + :param as_dict: See 'as_dict' argument of 'combine_by_key'. + :return: A new DDS with data set of list of tuples + (element, occurrence). + """ + return self.map(lambda x: x[0]).count_by_value(as_dict=as_dict) + + #@activemethod + def sort_by_key( + self, ascending=True, num_of_parts=None, key_func=lambda x: x + ): + """Sort by key. + + :param ascending: Ascending. + :param num_of_parts: Number of parts. + :param key_func: Key function. + :return: Sorted by key DDS object. + """ + if num_of_parts is None: + num_of_parts = len(self.partitions) + + # Collect everything to take samples + col_parts = self.collect(future_objects=True) + samples = [] + for _part in col_parts: + samples.append(task_collect_samples(_part, 20, key_func)) + + samples = sorted( + list(itertools.chain.from_iterable(compss_wait_on(samples))) + ) + + bounds = [ + samples[int(len(samples) * (i + 1) / num_of_parts)] + for i in range(0, num_of_parts - 1) + ] + + def range_partitioner(key): + """Partition a range. + + :param key: Partition key. + :return: Partitioned range. + """ + part = bisect.bisect_left(bounds, key_func(key)) + if ascending: + return part + return num_of_parts - 1 - part + + def sort_partition(iterator): + """Sort a partition locally. + + :param iterator: List iterator. + :return: Sorted partition. + """ + chunk_size = 500 + iterator = iter(iterator) + chunks = [] + while True: + chunk = list(itertools.islice(iterator, chunk_size)) + chunk.sort( + key=lambda kv: key_func(kv[0]), reverse=not ascending + ) + if len(chunk) > 0: + chunks.append(chunk) + if len(chunk) < chunk_size: + break + else: + chunks.append( + chunk.sort( + key=lambda kv: key_func(kv[0]), reverse=not ascending + ) + ) + + if len(chunks) == 1: + return chunks[0] + else: + return heapq.merge( + *chunks, + key=lambda kv: key_func(kv[0]), + reverse=not ascending + ) + + partitioned = DDS().load(col_parts, -1).partition_by(range_partitioner) + return partitioned.map_partitions(sort_partition) + + #@activemethod + def group_by_key(self, num_of_parts=-1): + """Group values of each key in a single list. + + A special and most used case of 'combine_by_key'. + + Usage sample: + >>> x = DDS().load([("a", 1), ("b", 2), ("a", 2), ("b", 4)]) + >>> sorted(x.group_by_key().collect()) + [('a', [1, 2]), ('b', [2, 4])] + + :param num_of_parts: Number of parts. + :returns: Grouped by key DDS object. + """ + + def _create(value): + return [value] + + def _merge(container, value): + container.append(value) + return container + + def _combine(container_a, container_b): + container_a.extend(container_b) + return container_a + + return self.combine_by_key( + _create, _merge, _combine, total_parts=num_of_parts + ) + + #@activemethod + def take(self, num): + """Take the first num elements of DDS. + + :param num: Number of elements to be retrieved. + :return: First elements of DDS. + """ + items = [] + partitions = self.collect(future_objects=True) + taken = 0 + + for part in partitions: + _p = iter(compss_wait_on(part)) + while taken < num: + try: + items.append(next(_p)) + taken += 1 + except StopIteration: + break + if taken >= num: + break + + return items[:num] + + +class _ChildDDS(DDS): + """_ChildDDS class. + + Similar as DDS objects, with the only difference that _ChildDDS objects + inherit the partitions from their parents, and have functions to be mapped + to their partitions. + """ + + #@activemethod + def __init__(self, parent, func): + """Create a new _ChildDDS object. + + :param parent: Parent DDS object. + :param func: Function. + """ + super().__init__() + self.paac = parent.paac + + if not isinstance(parent, _ChildDDS): + self.func = func + if isinstance(parent, DDS): + self.partitions = parent.partitions + else: + self.partitions = parent.partitions + par_func = parent.func + + def wrap_parent_func(partition): + return func(par_func(partition)) + + self.func = wrap_parent_func + + + +def _run_tests(): + """Run tests. + + :returns: None. + """ + import doctest # pylint: disable=C0415 + + doctest.testmod() + + # Clean after testing + to_be_removed = ["test.file", "test.txt"] + for file_name in to_be_removed: + try: + os.remove(file_name) + except OSError: + pass + + +if __name__ == "__main__": + _run_tests() diff --git a/examples/dds/pip b/examples/dds/pip new file mode 100755 index 00000000..dd5ba563 --- /dev/null +++ b/examples/dds/pip @@ -0,0 +1,8 @@ +#!/home/dcanocan/Desktop/Curro/git/dataclay/venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/examples/dds/pycompss/__init__.py b/examples/dds/pycompss/__init__.py new file mode 100644 index 00000000..66cdbc0f --- /dev/null +++ b/examples/dds/pycompss/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the COMPSs Python Binding, known as PyCOMPSs.""" + +from pycompss.util.location import get_binding_location + + +PYCOMPSS_HOME = get_binding_location() diff --git a/examples/dds/pycompss/__main__.py b/examples/dds/pycompss/__main__.py new file mode 100644 index 00000000..a96b509b --- /dev/null +++ b/examples/dds/pycompss/__main__.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Runnable as module. + +Provides the functionality to be run as a module: +e.g. python -m pycompss run -dgt myapp.py +""" + +import argparse +import sys +from subprocess import Popen + +from pycompss.runtime.commons import CONSTANTS +from pycompss.util.typing_helper import typing + +RUN_TAG = "run" +ENQUEUE_TAG = "enqueue" +RUN_EXECUTABLE = "runcompss" +ENQUEUE_EXECUTABLE = "enqueue_compss" +TAGS = [RUN_TAG, ENQUEUE_TAG] + + +class Object: # pylint: disable=too-few-public-methods + """Dummy class to mimic argparse return object.""" + + action = "None" + params = [] + + +def setup_parser() -> argparse.ArgumentParser: + """Argument parser. + + - Argument defining run for runcompss or enqueue for enqueue_compss. + - The rest of the arguments as a list. + + :return: the parser + """ + parser = argparse.ArgumentParser(prog="python -m pycompss") + parser.add_argument( + "action", + choices=TAGS, + nargs="?", + help='Execution mode: "run" for launching an' + + ' execution and "enqueue" for submitting a' + + " job to the queuing system." + + ' Default value: "run"', + ) + parser.add_argument( + "params", + nargs=argparse.REMAINDER, + help="COMPSs and application arguments" + + ' (check "runcompss" or "enqueue_compss"' + + " commands help).", + ) + return parser + + +def run(cmd: typing.List[str]) -> None: + """Execute a command line in a subprocess. + + :param cmd: Command to execute (list of ) + :return: None + """ + with Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) as command_process: + command_process.communicate() + + +def main() -> None: + """Run PyCOMPSs as module. + + SAMPLE: python -m pycompss run myapp.py + + :return: None + """ + _help = ["-h", "--help"] + parser = None # type: typing.Optional[argparse.ArgumentParser] + + # Check params + if ( + len(sys.argv) > 1 + and sys.argv[1] not in TAGS + and sys.argv[1] not in _help + ): + # No action specified. Assume run. + args = Object() + args.action = RUN_TAG + args.params = sys.argv[1:] + else: + parser = setup_parser() + args = parser.parse_args() + + # Check if the user has specified to use a specific python interpreter + if any("--python_interpreter=" in param for param in args.params): + # The user specified explicitly a python interpreter to use + # Do not include the python_interpreter variable in the cmd call + python_interpreter = [] + else: + # Use the same as current + python_interpreter = [ + f"--python_interpreter={CONSTANTS.python_interpreter}" + ] + + # Take an action + if args.action == RUN_TAG: + cmd = [RUN_EXECUTABLE] + python_interpreter + args.params + run(cmd) + elif args.action == ENQUEUE_TAG: + cmd = [ENQUEUE_EXECUTABLE] + python_interpreter + args.params + run(cmd) + else: + # Reachable only when python -m pycompss (and nothing else) + parser.print_usage() + + +if __name__ == "__main__": + main() diff --git a/examples/dds/pycompss/api/IO.py b/examples/dds/pycompss/api/IO.py new file mode 100644 index 00000000..c9fdb19b --- /dev/null +++ b/examples/dds/pycompss/api/IO.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - IO decorator. + +This file contains the IO class, needed for the IO task definition through +the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = set() # type: typing.Set[str] +SUPPORTED_ARGUMENTS = set() # type: typing.Set[str] +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class IO: # pylint: disable=too-few-public-methods + """IO decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on IO task creation. + """ + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", IO.__name__.lower())) + # super(IO, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the IO parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def io_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("IO")) + + if __debug__: + logger.debug("Executing IO_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + io_f.__doc__ = user_function.__doc__ + return io_f + + @staticmethod + def __configure_core_element__( + kwargs: typing.Dict[str, typing.Any] + ) -> None: + """Include the registering info related to @IO. + + IMPORTANT! Updates kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @IO core element.") + + # Resolve @io specific parameters + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_io(True) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_io(True) + kwargs[CORE_ELEMENT_KEY] = core_element + + +# ########################################################################### # +# ###################### IO DECORATOR ALTERNATIVE NAME ###################### # +# ########################################################################### # + +io = IO # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/__init__.py b/examples/dds/pycompss/api/__init__.py new file mode 100644 index 00000000..c5bad3f5 --- /dev/null +++ b/examples/dds/pycompss/api/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs API.""" diff --git a/examples/dds/pycompss/api/api.py b/examples/dds/pycompss/api/api.py new file mode 100644 index 00000000..6981309c --- /dev/null +++ b/examples/dds/pycompss/api/api.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API. + +This file defines the public PyCOMPSs API functions. +It implements the: + - start runtime + - stop runtime + - file exists + - open file + - delete file + - wait on file + - wait on directory + - delete object + - barrier + - barrier group + - snapshot + - wait_on + - get number of resources + - request resources creation + - request resources destruction + - set wall clock + - add logger + - TaskGroup (class) +functions. +Also includes the redirection to the dummy API. + +CAUTION: If the CONTEXT.has not been defined, it will load the dummy API + automatically. +""" + +from pycompss.util.context import CONTEXT + +# Dummy imports +from pycompss.api.dummy.api import ( + compss_start as __dummy_compss_start__, + compss_stop as __dummy_compss_stop__, + compss_file_exists as __dummy_compss_file_exists__, + compss_open as __dummy_compss_open__, + compss_delete_file as __dummy_compss_delete_file__, + compss_wait_on_file as __dummy_compss_wait_on_file__, + compss_wait_on_directory as __dummy_compss_wait_on_directory__, + compss_delete_object as __dummy_compss_delete_object__, + compss_barrier as __dummy_compss_barrier__, + compss_barrier_group as __dummy_compss_barrier_group__, + compss_cancel_group as __dummy_compss_cancel_group__, + compss_snapshot as __dummy_compss_snapshot__, + compss_wait_on as __dummy_compss_wait_on__, + compss_get_number_of_resources as __dummy_compss_get_number_of_resources__, + compss_request_resources as __dummy_compss_request_resources__, + compss_free_resources as __dummy_compss_free_resources__, + compss_set_wall_clock as __dummy_compss_set_wall_clock__, +) +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if CONTEXT.in_pycompss(): + # ################################################################# # + # PyCOMPSs API definitions # + # Any change on this API must be considered within the dummy API. # + # ################################################################# # + + from pycompss.runtime.binding import ( + start_runtime as __start_runtime__, + stop_runtime as __stop_runtime__, + file_exists as __file_exists__, + open_file as __open_file__, + delete_file as __delete_file__, + wait_on_file as __wait_on_file__, + wait_on_directory as __wait_on_directory__, + delete_object as __delete_object__, + barrier as __barrier__, + barrier_group as __barrier_group__, + open_task_group as __open_task_group__, + close_task_group as __close_task_group__, + cancel_task_group as __cancel_task_group__, + snapshot as __snapshot__, + get_number_of_resources as __get_number_of_resources__, + request_resources as __request_resources__, + free_resources as __free_resources__, + set_wall_clock as __set_wall_clock__, + wait_on as __wait_on__, + add_logger as __add_logger__, + ) + from pycompss.api.exceptions import COMPSsException as __COMPSsException__ + + +def compss_start( + log_level: str = "off", + tracing: bool = False, + interactive: bool = False, + disable_external: bool = False, +) -> None: + """Start the COMPSs runtime. + + :param log_level: Log level [ True | False ]. + :param tracing: Activate or disable tracing. + :param interactive: Boolean if interactive (ipython or jupyter). + :param disable_external: To avoid to load compss in external process. + :return: None + """ + if CONTEXT.in_pycompss(): + __start_runtime__(log_level, tracing, interactive, disable_external) + else: + __dummy_compss_start__( + log_level, tracing, interactive, disable_external + ) + + +def compss_stop(code: int = 0, _hard_stop: bool = False) -> None: + """Stop the COMPSs runtime. + + :param code: Stop code. + :param _hard_stop: Stop compss when runtime has died. + :return: None + """ + if CONTEXT.in_pycompss(): + __stop_runtime__(code, _hard_stop) + else: + __dummy_compss_stop__(code, _hard_stop) + + +def compss_file_exists( + *file_name: typing.Union[list, tuple, str] +) -> typing.Union[list, tuple, bool]: + """Check if a file exists. + + If it does not exist, it checks if the given file name has been + accessed before by calling the runtime. + + :param file_name: The file/s name to check. + :return: True either the file exists or has been accessed by the + runtime. False otherwise. + """ + if CONTEXT.in_pycompss(): + return __file_exists__(*file_name) + return __dummy_compss_file_exists__(*file_name) + + +def compss_open(file_name: str, mode: str = "r") -> typing.Any: + """Open a remotely produced file. + + Calls the runtime to bring the file to the master and opens it. + It will wait for the file to be produced. + CAUTION: Remember to close the file after using it with builtin close + function. + + :param file_name: File name. + :param mode: Open mode. Options = [w, r+ or a, r or empty]. + Default = "r" + :return: An object of "file" type. + :raise IOError: If the file can not be opened. + """ + if CONTEXT.in_pycompss(): + compss_name = __open_file__(file_name, mode) + return open(compss_name, mode) # pylint: disable=unspecified-encoding + return __dummy_compss_open__(file_name, mode) + + +def compss_delete_file( + *file_name: typing.Union[list, tuple, str] +) -> typing.Union[list, tuple, bool]: + """Delete one or more files. + + Calls the runtime to delete the file/s everywhere in the infrastructure. + Deletion is asynchronous and will be performed when the file is not + necessary anymore. + + :param file_name: File/s name. + :return: True if success. False otherwise. + """ + if CONTEXT.in_pycompss(): + return __delete_file__(*file_name) + return __dummy_compss_delete_file__(*file_name) + + +def compss_wait_on_file( + *file_name: typing.Union[list, tuple, str] +) -> typing.Union[list, tuple, str]: + """Wait and get one or more file/s. + + Calls the runtime to bring the file/s to the master when possible + and waits until produced. + + :param file_name: File/s name. + :return: The file/s name. + """ + if CONTEXT.in_pycompss(): + return __wait_on_file__(*file_name) + return __dummy_compss_wait_on_file__(*file_name) + + +def compss_wait_on_directory( + *directory_name: typing.Union[list, tuple, str] +) -> typing.Union[list, tuple, str]: + """Wait and get one or more directory/ies. + + Calls the runtime to bring the directory to the master when possible + and waits until produced. + + :param directory_name: Directory/ies name. + :return: The directory/ies name. + """ + if CONTEXT.in_pycompss(): + return __wait_on_directory__(*directory_name) + return __dummy_compss_wait_on_directory__(*directory_name) + + +def compss_delete_object( + *obj: typing.Any, +) -> typing.Union[list, tuple, str, bool]: + """Delete object/s. + + Removes one or more used object from the internal structures and calls the + external python library (that calls the bindings-common) + in order to request its/their corresponding file removal. + + :param obj: Object/s to delete. + :return: True if success. False otherwise. + """ + if CONTEXT.in_pycompss(): + return __delete_object__(*obj) + return __dummy_compss_delete_object__(*obj) + + +def compss_barrier(no_more_tasks: bool = False) -> None: + """Wait for all tasks. + + Perform a barrier waiting until all the submitted tasks have finished. + + :param no_more_tasks: No more tasks boolean. + :return: None. + """ + if CONTEXT.in_pycompss(): + __barrier__(no_more_tasks) + else: + __dummy_compss_barrier__(no_more_tasks) + + +def compss_barrier_group(group_name: str) -> None: + """Perform a barrier to a group. + + Stop until all the tasks of a group have finished. + + :param group_name: Name of the group to wait. + :return: None. + """ + if CONTEXT.in_pycompss(): + exception_message = __barrier_group__(group_name) + if exception_message != "None": + raise __COMPSsException__(exception_message) + else: + __dummy_compss_barrier_group__(group_name) + + +def compss_cancel_group(group_name: str) -> None: + """Cancel a task group. + + Cancel all the non finished tasks of a group. + + :param group_name: Name of the group to wait. + :return: None. + """ + if CONTEXT.in_pycompss(): + exception_message = __cancel_task_group__(group_name) + if exception_message != "None": + raise __COMPSsException__(exception_message) + else: + __dummy_compss_cancel_group__(group_name) + + +def compss_snapshot() -> None: + """Request a snapshot. + + Performs a snapshot of the data of the tasks that have been finished and + not saved already. + + :return: None. + """ + if CONTEXT.in_pycompss(): + __snapshot__() + else: + __dummy_compss_snapshot__() + + +def compss_wait_on(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """Wait for objects. + + Waits on a set of objects defined in args with the options defined in + kwargs. + Kwargs options: + - "mode" Write enable? [ 'r' | 'rw' ] Default = 'rw' + + :param args: Objects to wait on. + :param kwargs: Options dictionary. + :return: List with the final values (or a single element if only one). + """ + if CONTEXT.in_pycompss(): + return __wait_on__(*args, **kwargs) + return __dummy_compss_wait_on__(*args, **kwargs) + + +def compss_get_number_of_resources() -> int: + """Request for the number of active resources. + + :return: The number of active resources. + """ + if CONTEXT.in_pycompss(): + return __get_number_of_resources__() + return __dummy_compss_get_number_of_resources__() + + +def compss_request_resources( + num_resources: int, group_name: typing.Optional[str] +) -> None: + """Request the creation of num_resources resources. + + :param num_resources: Number of resources to create. + :param group_name: Task group to notify upon resource creation. + (it can be None) + :return: None + """ + if CONTEXT.in_pycompss(): + __request_resources__(num_resources, group_name) + else: + __dummy_compss_request_resources__(num_resources, group_name) + + +def compss_free_resources( + num_resources: int, group_name: typing.Optional[str] +) -> None: + """Request the destruction of num_resources resources. + + :param num_resources: Number of resources to destroy. + :param group_name: Task group to notify upon resource creation + :return: None + """ + if CONTEXT.in_pycompss(): + __free_resources__(num_resources, group_name) + else: + __dummy_compss_free_resources__(num_resources, group_name) + + +def compss_set_wall_clock(wall_clock_limit: int) -> None: + """Set the application wall clock limit. + + :param wall_clock_limit: Wall clock limit in seconds. + :return: None + """ + if CONTEXT.in_pycompss(): + __set_wall_clock__(wall_clock_limit) + else: + __dummy_compss_set_wall_clock__(wall_clock_limit) + + +def compss_add_logger(logger_name: str) -> None: + """Add a new logger for the user. + + Enables users to redirect their output messages through the PyCOMPSs + logger, ensuring that they are correctly flushed. + Will be added to the current loggers with the current configuration. + + :param logger_name: New logger name. + :returns: None + """ + if CONTEXT.in_pycompss(): + __add_logger__(logger_name) + else: + raise NotInPyCOMPSsException( + "Add logger is only supported within PyCOMPSs" + ) + + +class TaskGroup: + """ + A CONTEXT.like class used to represent a group of tasks. + + This CONTEXT.is aimed at enabling to define groups of tasks + using the "with" statement. + + For example: + ... + with TaskGroup("my_group", False): + # call to tasks + # they will be considered within my_group group. + ... + ... + """ + + __slots__ = ["group_name", "implicit_barrier"] + + def __init__(self, group_name: str, implicit_barrier: bool = True) -> None: + """Define a new group of tasks. + + :param group_name: Group name. + :param implicit_barrier: Perform implicit barrier. + + :attr str group_name: Group name. + :attr bool implicit_barrier: Perform implicit barrier. + """ + if CONTEXT.in_pycompss(): + self.group_name = group_name + self.implicit_barrier = implicit_barrier + else: + pass + + def __enter__(self) -> None: + """Group creation.""" + if CONTEXT.in_pycompss(): + __open_task_group__(self.group_name, self.implicit_barrier) + else: + pass + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Group closing.""" + if CONTEXT.in_pycompss(): + __close_task_group__(self.group_name) + if self.implicit_barrier: + compss_barrier_group(self.group_name) + else: + pass diff --git a/examples/dds/pycompss/api/binary.py b/examples/dds/pycompss/api/binary.py new file mode 100644 index 00000000..b2800d03 --- /dev/null +++ b/examples/dds/pycompss/api/binary.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Binary decorator. + +This file contains the Binary class, needed for the binary task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.decorator import run_command + +# from pycompss.api.commons.decorator import Decorator +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.binary} +SUPPORTED_ARGUMENTS = { + LABELS.binary, + LABELS.working_dir, + LABELS.args, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = {LEGACY_LABELS.working_dir, LABELS.engine, LABELS.image} + + +class Binary: # pylint: disable=too-few-public-methods + """Binary decorator class. + + This decorator preserves the argspec, but includes the __init__ and + __call__ methods, useful on binary task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", Binary.__name__.lower())) + # super(Binary, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the binary parameters within the task core element. + + :param user_function: Function to decorate + :return: Decorated function. + """ + + @wraps(user_function) + def binary_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + # Execute the binary as with PyCOMPSs so that sequential + # execution performs as parallel. + # To disable: raise Exception(not_in_pycompss(LABELS.binary)) + # TODO: Intercept @task parameters to get stream redirection + return self.__run_binary__(args, kwargs) + + if __debug__: + logger.debug("Executing binary_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + binary_f.__doc__ = user_function.__doc__ + return binary_f + + def __run_binary__(self, args: tuple, kwargs: dict) -> int: + """Run the binary defined in the decorator when used as dummy. + + :param args: Arguments received from call. + :param kwargs: Keyword arguments received from call. + :return: Execution return code. + """ + cmd = [self.kwargs[LABELS.binary]] + return_code = run_command(cmd, args, kwargs) + return return_code + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @binary. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @binary core element.") + + # Resolve the working directory + resolve_working_dir(self.kwargs) + _working_dir = str(self.kwargs[LABELS.working_dir]) + + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + _fail_by_ev = str(self.kwargs[LABELS.fail_by_exit_value]) + + # Resolve binary + _binary = str(self.kwargs[LABELS.binary]) + + # Resolve args + _args = self.kwargs.get(LABELS.args, INTERNAL_LABELS.unassigned) + + if ( + CORE_ELEMENT_KEY in kwargs + and kwargs[CORE_ELEMENT_KEY].get_impl_type() + == IMPLEMENTATION_TYPES.container + ): + # @container decorator sits on top of @binary decorator + # Note: impl_type and impl_signature are NOT modified + # (IMPLEMENTATION_TYPES.container and "CONTAINER.function_name" + # respectively) + + impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args() + + _engine = impl_args[0] + _image = impl_args[1] + _options = impl_args[2] + + impl_args = [ + _engine, # engine + _image, # image + _options, # container options + IMPLEMENTATION_TYPES.cet_binary, # internal_type + _binary, # internal_binary + _args, + INTERNAL_LABELS.unassigned, # internal_func + _working_dir, # working_dir + _fail_by_ev, # fail_by_ev + ] + + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @container decorator does NOT sit on top of @binary decorator + + _binary = str(self.kwargs[LABELS.binary]) + + impl_type = IMPLEMENTATION_TYPES.binary + impl_signature = ".".join((impl_type, _binary)) + + impl_args = [ + _binary, # internal_binary + _working_dir, # working_dir + _args, # args + _fail_by_ev, # fail_by_ev + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level + # decorator (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### BINARY DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +binary = Binary # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/commons/__init__.py b/examples/dds/pycompss/api/commons/__init__.py new file mode 100644 index 00000000..a62b33b9 --- /dev/null +++ b/examples/dds/pycompss/api/commons/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the API common functions, constants and classes.""" diff --git a/examples/dds/pycompss/api/commons/constants.py b/examples/dds/pycompss/api/commons/constants.py new file mode 100644 index 00000000..d4090f1a --- /dev/null +++ b/examples/dds/pycompss/api/commons/constants.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - commons - constants. + +This file contains the common decorator labels. +""" + + +class _Labels: # pylint: disable=R0903,R0902 + # disable=too-few-public-methods, too-many-instance-attributes + """Currently supported labels in all decorators.""" + + __slots__ = ( + "returns", + "priority", + "on_failure", + "defaults", + "time_out", + "is_replicated", + "is_distributed", + "varargs_type", + "target_direction", + "numba", + "numba_flags", + "numba_signature", + "numba_declaration", + "tracing_hook", + "computing_nodes", + "computing_units", + "working_dir", + "args", + "parameters", + "fail_by_exit_value", + "task", + "mpi", + "mpmd_mpi", + "multinode", + "binary", + "compss", + "http", + "runner", + "processes", + "programs", + "scale_by_cu", + "app_name", + "runcompss", + "flags", + "processes_per_node", + "worker_in_master", + "engine", + "image", + "options", + "df_script", + "df_executor", + "df_lib", + "julia_executor", + "julia_args", + "julia_script", + "source_class", + "method", + "management", + "management_ignore", + "management_retry", + "management_cancel_successor", + "management_fail", + "kernel", + "chunk_size", + "is_reduce", + "cache_returns", + "service_name", + "resource", + "request", + "payload", + "payload_type", + "produces", + "updates", + "config_file", + "software_config_file", # this is internal + "properties", + "execution", + "type", + "is_workflow", + ) + + def __init__(self) -> None: # pylint: disable=too-many-statements + # Expected labels + # - Task decorator + self.target_direction = "target_direction" + self.returns = "returns" + self.cache_returns = "cache_returns" + self.priority = "priority" + self.defaults = "defaults" + self.time_out = "time_out" + self.is_replicated = "is_replicated" + self.is_distributed = "is_distributed" + self.computing_nodes = "computing_nodes" + self.computing_units = "computing_units" + self.is_reduce = "is_reduce" + self.chunk_size = "chunk_size" + self.tracing_hook = "tracing_hook" + self.numba = "numba" + self.numba_flags = "numba_flags" + self.numba_signature = "numba_signature" + self.numba_declaration = "numba_declaration" + self.varargs_type = "varargs_type" + # - Others + self.on_failure = "on_failure" + self.working_dir = "working_dir" + self.args = "args" + self.parameters = "parameters" + self.fail_by_exit_value = "fail_by_exit_value" + self.task = "task" + self.mpi = "mpi" + self.binary = "binary" + self.http = "http" + self.compss = "compss" + self.mpmd_mpi = "mpmd_mpi" + self.multinode = "multinode" + self.runner = "runner" + self.processes = "processes" + self.programs = "programs" + self.scale_by_cu = "scale_by_cu" + self.app_name = "app_name" + self.runcompss = "runcompss" + self.flags = "flags" + self.processes_per_node = "processes_per_node" + self.worker_in_master = "worker_in_master" + self.engine = "engine" + self.image = "image" + self.options = "options" + self.df_script = "df_script" + self.df_executor = "df_executor" + self.df_lib = "df_lib" + self.julia_executor = "executor" + self.julia_args = "args" + self.julia_script = "script" + self.source_class = "source_class" + self.method = "method" + self.management = "management" + self.management_ignore = "IGNORE" + self.management_retry = "RETRY" + self.management_cancel_successor = "CANCEL_SUCCESSORS" + self.management_fail = "FAIL" + self.kernel = "kernel" + + self.is_workflow = "is_workflow" + # Http tasks + self.service_name = "service_name" + self.resource = "resource" + self.request = "request" + self.payload = "payload" + self.payload_type = "payload_type" + self.produces = "produces" + self.updates = "updates" + self.config_file = "config_file" + self.software_config_file = "software_config_file" + self.properties = "properties" + self.execution = "execution" + self.type = "type" + + +class _LegacyLabels: # pylint: disable=R0903,R0902 + # disable=too-few-public-methods, too-many-instance-attributes + """Supported labels in all decorators but sensitive to be removed.""" + + __slots__ = ( + "is_replicated", + "is_distributed", + "varargs_type", + "target_direction", + "time_out", + "computing_units", + "computing_nodes", + "working_dir", + "app_name", + "worker_in_master", + "df_executor", + "df_lib", + "df_script", + "source_class", + ) + + def __init__(self) -> None: + self.is_replicated = "isReplicated" + self.is_distributed = "isDistributed" + self.varargs_type = "varargsType" + self.target_direction = "targetDirection" + self.time_out = "timeOut" + self.computing_units = "computingUnits" + self.computing_nodes = "computingNodes" + self.working_dir = "workingDir" + self.app_name = "appName" + self.worker_in_master = "workerInMaster" + self.df_executor = "dfExecutor" + self.df_lib = "dfLib" + self.df_script = "dfScript" + self.source_class = "sourceClass" + + +class _InternalLabels: # pylint: disable=too-few-public-methods + """Internal labels.""" + + __slots__ = ["unassigned"] + + def __init__(self) -> None: + self.unassigned = "[unassigned]" + + +LABELS = _Labels() +LEGACY_LABELS = _LegacyLabels() +INTERNAL_LABELS = _InternalLabels() diff --git a/examples/dds/pycompss/api/commons/data_type.py b/examples/dds/pycompss/api/commons/data_type.py new file mode 100644 index 00000000..bbb8fdf3 --- /dev/null +++ b/examples/dds/pycompss/api/commons/data_type.py @@ -0,0 +1,48 @@ +""" +Python binding to runtime data types matching. + +Autogenerated file (see generate_datatype_enums.py). + - Uses DataType.java as the source template. +""" + + +class SupportedDataTypes: # pylint: disable=too-few-public-methods + """Python binding to runtime Datatypes.""" + + BOOLEAN = 0 + CHAR = 1 + BYTE = 2 + SHORT = 3 + INT = 4 + LONG = 5 + FLOAT = 6 + DOUBLE = 7 + STRING = 8 + STRING_64 = 9 + FILE = 10 + OBJECT = 11 + PSCO = 12 + EXTERNAL_PSCO = 13 + BINDING_OBJECT = 14 + WCHAR = 15 + WSTRING = 16 + LONGLONG = 17 + VOID = 18 + ANY = 19 + ARRAY_CHAR = 20 + ARRAY_BYTE = 21 + ARRAY_SHORT = 22 + ARRAY_INT = 23 + ARRAY_LONG = 24 + ARRAY_FLOAT = 25 + ARRAY_DOUBLE = 26 + COLLECTION = 27 + DICT_COLLECTION = 28 + STREAM = 29 + EXTERNAL_STREAM = 30 + ENUM = 31 + NULL = 32 + DIRECTORY = 33 + + +DataType = SupportedDataTypes() diff --git a/examples/dds/pycompss/api/commons/decorator.py b/examples/dds/pycompss/api/commons/decorator.py new file mode 100644 index 00000000..518ba8b7 --- /dev/null +++ b/examples/dds/pycompss/api/commons/decorator.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - commons - decorator. + +This file contains very usual functions for the decorators. +""" + +import os +import subprocess +import sys +from contextlib import contextmanager + +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.runtime.task.features import TASK_FEATURES +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +# Global name to be used within kwargs for the core element. +CORE_ELEMENT_KEY = "compss_core_element" + + +def resolve_working_dir(kwargs: dict) -> None: + """Resolve the working directory considering deprecated naming. + + Updates kwargs: + - Removes workingDir if exists. + - Updates working_dir with the working directory. + + :return: None + """ + if LABELS.working_dir in kwargs: + # Accepted argument + pass + elif LEGACY_LABELS.working_dir in kwargs: + kwargs[LABELS.working_dir] = kwargs.pop(LEGACY_LABELS.working_dir) + else: + kwargs[LABELS.working_dir] = INTERNAL_LABELS.unassigned + + +def resolve_fail_by_exit_value(kwargs: dict, def_val="true") -> None: + """Resolve the fail by exit value. + + Updates kwargs: + - Updates fail_by_exit_value if necessary. + + :return: None + """ + if LABELS.fail_by_exit_value in kwargs: + fail_by_ev = kwargs[LABELS.fail_by_exit_value] + if isinstance(fail_by_ev, bool): + kwargs[LABELS.fail_by_exit_value] = str(fail_by_ev) + elif isinstance(fail_by_ev, str): + # Accepted argument + pass + elif isinstance(fail_by_ev, int): + kwargs[LABELS.fail_by_exit_value] = str(fail_by_ev) + else: + raise PyCOMPSsException( + "Incorrect format for fail_by_exit_value property. " + "It should be boolean or an environment variable" + ) + else: + kwargs[LABELS.fail_by_exit_value] = def_val + + +def process_computing_nodes(decorator_name: str, kwargs: dict) -> None: + """Process the computing_nodes from the decorator. + + We only ensure that the correct self.kwargs entry exists since its + value will be parsed and resolved by the + master.process_computing_nodes. + Used in decorators: + - mpi + - multinode + - compss + - decaf + + WARNING: Updates kwargs. + + :param decorator_name: Decorator name + :param kwargs: Key word arguments + :return: None + """ + if LABELS.computing_nodes not in kwargs: + if LEGACY_LABELS.computing_nodes not in kwargs: + # No annotation present, adding default value + kwargs[LABELS.computing_nodes] = str(1) + else: + # Legacy annotation present, switching + kwargs[LABELS.computing_nodes] = str( + kwargs.pop(LEGACY_LABELS.computing_nodes) + ) + else: + # Valid annotation found, nothing to do + pass + + if __debug__: + logger.debug( + "This %s task will have %s computing nodes.", + decorator_name, + str(kwargs[LABELS.computing_nodes]), + ) + + +################### +# COMMON CONTEXTS # +################### + + +@contextmanager +def keep_arguments( + args: tuple, kwargs: dict, prepend_strings: bool = True +) -> typing.Iterator[None]: + """Context which saves and restores the function arguments. + + It also enables or disables the PREPEND_STRINGS property from @task. + + :param args: Arguments. + :param kwargs: Key word arguments. + :param prepend_strings: Prepend strings in the task. + :return: None + """ + # Keep function arguments + saved = {} + slf = None + if len(args) > 0: + # The "self" for a method function is passed as args[0] + slf = args[0] + + # Replace and store the attributes + for key, value in kwargs.items(): + if hasattr(slf, key): + saved[key] = getattr(slf, key) + setattr(slf, key, value) + + if not prepend_strings: + TASK_FEATURES.set_prepend_strings(False) + yield + # Restore PREPEND_STRINGS to default: True + TASK_FEATURES.set_prepend_strings(True) + # Restore function arguments + if len(args) > 0: + # Put things back + for key, value in saved.items(): + setattr(slf, key, value) + + +################# +# OTHER COMMONS # +################# + + +def run_command(cmd: typing.List[str], args: tuple, kwargs: dict) -> int: + """Execute the command considering necessary the args and kwargs. + + :param cmd: Command to run. + :param args: Decorator arguments. + :param kwargs: Decorator key arguments. + :return: Execution return code. + """ + if args: + args_elements = [] + for arg in args: + if arg: + args_elements.append(arg) + cmd += args_elements + my_env = os.environ.copy() + env_path = my_env["PATH"] + if LABELS.working_dir in kwargs: + my_env["PATH"] = kwargs[LABELS.working_dir] + env_path + elif LEGACY_LABELS.working_dir in kwargs: + my_env["PATH"] = kwargs[LEGACY_LABELS.working_dir] + env_path + with subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env + ) as proc: + out, err = proc.communicate() + out_message = out.decode().strip() + err_message = err.decode().strip() + if out_message: + print(out_message) + if err_message: + sys.stderr.write(err_message + "\n") + return proc.returncode diff --git a/examples/dds/pycompss/api/commons/error_msgs.py b/examples/dds/pycompss/api/commons/error_msgs.py new file mode 100644 index 00000000..5358e8f7 --- /dev/null +++ b/examples/dds/pycompss/api/commons/error_msgs.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - commons - error messages. + +This file defines the public PyCOMPSs error messages displayed by the API. +""" + + +def not_in_pycompss(decorator_name: str) -> str: + """Retrieve the "not in PyCOMPSs scope" error message. + + :param decorator_name: Decorator name which requires the message. + :return: Not in PyCOMPSs error message. + """ + return ( + f"The {decorator_name} decorator only works within PyCOMPSs framework." + ) + + +def cast_env_to_int_error(what: str) -> str: + """Retrieve the "can not cast from env. variable to integer" error message. + + :param what: Environment variable name. + :return: Can not cast from environment variable to integer. + """ + return f"ERROR: {what} value cannot be cast from ENV variable to int" + + +def cast_string_to_int_error(what: str) -> str: + """Retrieve the "can not cast from string to integer" error message. + + :param what: Environment variable name. + :return: Can not cast from string to integer. + """ + return f"ERROR: {what} value cannot be cast from string to int" + + +def wrong_value(value_name: str, decorator_name: str) -> str: + """Retrieve the "wrong value at decorator" error message. + + :param value_name: Wrong value's name + :param decorator_name: Decorator name which requires the message. + :return: Wrong value at decorator message. + """ + return f"ERROR: Wrong {value_name} value at {decorator_name} decorator." diff --git a/examples/dds/pycompss/api/commons/implementation_types.py b/examples/dds/pycompss/api/commons/implementation_types.py new file mode 100644 index 00000000..619b4347 --- /dev/null +++ b/examples/dds/pycompss/api/commons/implementation_types.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - commons - implementation types. + +This file contains the implementation types definitions. +""" + + +class _ImplementationTypes: # pylint: disable=R0903,R0902 + # disable=too-few-public-methods, too-many-instance-attributes + """Supported implementation types.""" + + __slots__ = [ + "binary", + "cet_binary", + "compss", + "container", + "decaf", + "julia", + "method", + "mpi", + "multi_node", + "ompss", + "opencl", + "python_mpi", + "http", + ] + + def __init__(self) -> None: + self.binary = "BINARY" + self.cet_binary = "CET_BINARY" + self.compss = "COMPSs" + self.container = "CONTAINER" + self.decaf = "DECAF" + self.julia = "JULIA" + self.method = "METHOD" + self.mpi = "MPI" + self.multi_node = "MULTI_NODE" + self.ompss = "OMPSS" + self.opencl = "OPENCL" + self.python_mpi = "PYTHON_MPI" + self.http = "HTTP" + + +IMPLEMENTATION_TYPES = _ImplementationTypes() diff --git a/examples/dds/pycompss/api/commons/private_tasks.py b/examples/dds/pycompss/api/commons/private_tasks.py new file mode 100644 index 00000000..ebc63f22 --- /dev/null +++ b/examples/dds/pycompss/api/commons/private_tasks.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - COMMONS - PRIVATE TASKS. + +This file contains tasks required by any decorator. + +WARNING: This file can not be compiled with mypy since it contains + @task decorated functions. +""" + +from pycompss.api.task import task +from pycompss.api.parameter import COLLECTION_IN +from pycompss.api.parameter import FILE_OUT +from pycompss.api.parameter import FILE_IN +from pycompss.api.parameter import DIRECTORY_IN +from pycompss.api.parameter import DIRECTORY_OUT + + +@task(returns=1) +def transform(target, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to. + @param function: DT function + @param kwargs: kwargs of the DT function + :return: + """ + return function(target, **kwargs) + + +@task(returns=object, target=COLLECTION_IN) +def col_to_obj(target, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param function: DT function which accepts a collection as input and + produces an object. + :return: + """ + return function(target, **kwargs) + + +@task(destination=FILE_OUT, target=COLLECTION_IN) +def col_to_file(target, destination, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param destination: name of the file that will be produced by this task + @param function: DT function which accepts a collection as input and + produces a file. + :return: + """ + function(target, destination, **kwargs) + + +@task(destination=DIRECTORY_OUT, target=COLLECTION_IN) +def col_to_dir(target, destination, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param destination: name of the file that will be produced by this task + @param function: DT function which accepts a collection as input and + produces a file. + :return: + """ + function(target, destination, **kwargs) + + +@task(returns=object(), target=FILE_IN) +def file_to_object(target, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param function: DT function which accepts a file as input and + produces an object. + :return: + """ + return function(target, **kwargs) + + +@task(returns=object(), target=DIRECTORY_IN) +def dir_to_object(target, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param function: DT function which accepts a file as input and + produces an object. + :return: + """ + return function(target, **kwargs) + + +@task(destination=FILE_OUT) +def object_to_file(target, destination, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param destination: name of the file that will be produced by this task + @param function: DT function which accepts an object as input and + produces a file. + :return: + """ + function(target, destination, **kwargs) + + +@task(destination=DIRECTORY_OUT) +def object_to_dir(target, destination, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param destination: name of the file that will be produced by this task + @param function: DT function which accepts an object as input and + produces a file. + :return: + """ + function(target, destination, **kwargs) + + +@task(target=FILE_IN) +def file_to_col(target, function, **kwargs): + """Replace the user function with its @task equivalent. + + NOTE: Used from @data_transformation. + + @param target: the parameter that DT will be applied to + @param function: DT function which accepts a file as input and + produces a collection. + :return: + """ + return function(target, **kwargs) + + +# @task(target=FILE_IN, destination=COLLECTION_OUT) +# def _file_to_col(target, function, destination): +# function(target) diff --git a/examples/dds/pycompss/api/compss.py b/examples/dds/pycompss/api/compss.py new file mode 100644 index 00000000..995a595e --- /dev/null +++ b/examples/dds/pycompss/api/compss.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - COMPSs decorator. + +This file contains the COMPSs class, needed for the COMPSs task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.app_name} +SUPPORTED_ARGUMENTS = { + LABELS.computing_nodes, + LABELS.runcompss, + LABELS.flags, + LABELS.worker_in_master, + LABELS.app_name, + LABELS.args, + LABELS.working_dir, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.computing_nodes, + LEGACY_LABELS.worker_in_master, + LEGACY_LABELS.app_name, + LEGACY_LABELS.working_dir, +} + + +class COMPSs: # pylint: disable=too-few-public-methods + """COMPSs decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on compss task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", COMPSs.__name__.lower())) + # super(COMPSs, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + process_computing_nodes(decorator_name, self.kwargs) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the compss parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def compss_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("compss")) + + if __debug__: + logger.debug("Executing compss_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + # Set the computing_nodes variable in kwargs for its usage + # in @task decorator + kwargs[LABELS.computing_nodes] = self.kwargs[ + LABELS.computing_nodes + ] + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + compss_f.__doc__ = user_function.__doc__ + return compss_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @compss. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @compss core element.") + + # Resolve @compss specific parameters + if LABELS.runcompss in self.kwargs: + runcompss = self.kwargs[LABELS.runcompss] + else: + runcompss = ( + INTERNAL_LABELS.unassigned + ) # Empty or INTERNAL_LABELS.unassigned + + if LABELS.flags in self.kwargs: + flags = self.kwargs[LABELS.flags] + else: + flags = ( + INTERNAL_LABELS.unassigned + ) # Empty or INTERNAL_LABELS.unassigned + + if LABELS.worker_in_master in self.kwargs: + worker_in_master = self.kwargs[LABELS.worker_in_master] + elif LEGACY_LABELS.worker_in_master in self.kwargs: + worker_in_master = self.kwargs[LEGACY_LABELS.worker_in_master] + else: + worker_in_master = "true" # Empty or INTERNAL_LABELS.unassigned + + if LEGACY_LABELS.app_name in self.kwargs: + app_name = self.kwargs[LEGACY_LABELS.app_name] + else: + app_name = self.kwargs[LABELS.app_name] + # Resolve args + _args = self.kwargs.get(LABELS.args, INTERNAL_LABELS.unassigned) + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + impl_type = IMPLEMENTATION_TYPES.compss + impl_signature = ".".join((impl_type, app_name)) + impl_args = [ + runcompss, + flags, + app_name, + _args, + worker_in_master, + self.kwargs[LABELS.working_dir], + self.kwargs[LABELS.fail_by_exit_value], + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# #################### COMPSs DECORATOR ALTERNATIVE NAME #################### # +# ########################################################################### # + +compss = COMPSs # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/constraint.py b/examples/dds/pycompss/api/constraint.py new file mode 100644 index 00000000..83a1e6ce --- /dev/null +++ b/examples/dds/pycompss/api/constraint.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Constraint decorator. + +This file contains the Constraint class, needed for the task constraint +definition through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.dummy.constraint import constraint as dummy_constraint +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.typing_helper import typing +from pycompss.util.exceptions import PyCOMPSsException + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + + +class Constraint: # pylint: disable=too-few-public-methods + """Constraint decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on task constraint creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", Constraint.__name__.lower())) + # super(Constraint, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the constraints within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def constrained_f( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + if not self.scope: + d_c = dummy_constraint(self.args, self.kwargs) + return d_c.__call__(user_function)(*args, **kwargs) + + if __debug__: + logger.debug("Executing constrained_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + constrained_f.__doc__ = user_function.__doc__ + return constrained_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @constraint. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Current keyword arguments to be updated with the core + element information. + :return: None + """ + if __debug__: + logger.debug("Configuring @constraint core element.") + + is_local_key = "is_local" + is_local = False + if is_local_key in self.kwargs: + is_local = self.kwargs.pop(is_local_key) + if not isinstance(is_local, bool): + raise PyCOMPSsException( + "is_local constraint can only be defined with boolean" + ) + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @implements and @compss) + kwargs[CORE_ELEMENT_KEY].set_impl_constraints(self.kwargs) + kwargs[CORE_ELEMENT_KEY].set_impl_local(is_local) + else: + # @constraint is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_constraints(self.kwargs) + core_element.set_impl_local(is_local) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### CONSTRAINT DECORATOR ALTERNATIVE NAME ################# # +# ########################################################################### # + +constraint = Constraint # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/container.py b/examples/dds/pycompss/api/container.py new file mode 100644 index 00000000..0008b682 --- /dev/null +++ b/examples/dds/pycompss/api/container.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Container decorator. + +This file contains the Container class, needed for the container task +definition through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.engine, LABELS.image} +SUPPORTED_ARGUMENTS = {LABELS.engine, LABELS.image, LABELS.options} +DEPRECATED_ARGUMENTS = { + LABELS.fail_by_exit_value, + LABELS.working_dir, + LEGACY_LABELS.working_dir, + LABELS.binary, +} + + +class Container: # pylint: disable=too-few-public-methods + """Container decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on mpi task creation. + """ + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "@" + Container.__name__.lower() + # super(Container, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + if __debug__: + logger.debug("Init @container decorator...") + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the container parameters within the task core element. + + :param user_function: Function to decorate + :return: Decorated function. + """ + + @wraps(user_function) + def container_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("container")) + + if __debug__: + logger.debug("Executing container_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs, user_function) + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + container_f.__doc__ = user_function.__doc__ + return container_f + + def __configure_core_element__( + self, kwargs: dict, user_function: typing.Callable + ) -> None: + """Include the registering info related to @container. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :param user_function: Decorated function. + :return: None + """ + if __debug__: + logger.debug("Configuring @container core element.") + + # Resolve @container (mandatory) specific parameters + _engine = self.kwargs[LABELS.engine] + _image = self.kwargs[LABELS.image] + _func = str(user_function.__name__) + + # Type and signature + impl_type = IMPLEMENTATION_TYPES.container + impl_signature = ".".join([impl_type, _func]) + + impl_args = [ + _engine, # engine + _image, # image + self.kwargs.get(LABELS.options, INTERNAL_LABELS.unassigned), + INTERNAL_LABELS.unassigned, # internal_type + INTERNAL_LABELS.unassigned, # internal_binary + INTERNAL_LABELS.unassigned, # internal_func + INTERNAL_LABELS.unassigned, # working_dir + INTERNAL_LABELS.unassigned, + ] # fail_by_ev + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY].set_impl_container(impl_args[:3]) + else: + # @container is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + core_element.set_impl_container(impl_args[:3]) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################# CONTAINER DECORATOR ALTERNATIVE NAME #################### # +# ########################################################################### # + +container = Container # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/data_transformation.py b/examples/dds/pycompss/api/data_transformation.py new file mode 100644 index 00000000..0b3ce5a4 --- /dev/null +++ b/examples/dds/pycompss/api/data_transformation.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Data Transformation decorator. + +This file contains DT decorator class and its helper DTO class. +""" + +import inspect + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.private_tasks import transform as _transform +from pycompss.api.commons.private_tasks import col_to_obj as _col_to_obj +from pycompss.api.commons.private_tasks import col_to_file as _col_to_file +from pycompss.api.commons.private_tasks import col_to_dir as _col_to_dir +from pycompss.api.commons.private_tasks import ( + object_to_file as _object_to_file, +) +from pycompss.api.commons.private_tasks import ( + object_to_dir as _object_to_dir, +) +from pycompss.api.commons.private_tasks import file_to_col as _file_to_col +from pycompss.api.commons.private_tasks import ( + file_to_object as _file_to_object, +) +from pycompss.api.commons.private_tasks import ( + dir_to_object as _dir_to_object, +) +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +# from pycompss.runtime.task.definitions.core_element import CE + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = set() # type: typing.Set[str] +SUPPORTED_ARGUMENTS = set() # type: typing.Set[str] +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + +OBJECT_TO_FILE = 0 +FILE_TO_OBJECT = 1 +FILE_TO_COLLECTION = 2 +COLLECTION_TO_OBJECT = 3 +COLLECTION_TO_FILE = 4 +OBJECT_TO_DIRECTORY = 5 +DIRECTORY_TO_OBJECT = 6 +COLLECTION_TO_DIRECTORY = 7 + + +class DataTransformation: # pylint: disable=R0902,R0903 + # disable=too-many-instance-attributes, too-few-public-methods + """Data Transformation decorator for PyCOMPSs tasks.""" + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "user_function", + "core_element_configured", + "dt_function", + "type", + "target", + "destination", + ] + + def __init__(self, *args, **kwargs): + """Store arguments passed to the decorator. + + If the args are empty, it will mean that the decorator should get the + list of the DTO's from the call method. + + :param args: should contain only the & + :param kwargs: kwargs of the user DT function. + """ + decorator_name = "".join(("@", DataTransformation.__name__.lower())) + # super(DataTransformation, self).__init__( + # decorator_name, *args, **kwargs + # ) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + # self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + self.user_function = None + self.target = self.kwargs.pop("target", None) + self.dt_function = self.kwargs.pop("function", None) + self.type = self.kwargs.pop("type", None) + # TODO: so far we use default unique file name for all DTs, + # should it be replaced? + self.destination = self.kwargs.pop("destination", "dt_file_out") + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Call to the decorated task function. + + Call is mainly meant to generate DT (task) functions. However, if + the __init__ wasn't provided with any args, it also should extract the + DTO's from the kwargs. + + :param user_function: User function to be decorated. + :return: Decorated dummy user function. + """ + + @wraps(user_function) + def dt_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotImplementedError + + if __debug__: + logger.debug("Executing DT wrapper.") + tmp = list(args) + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + self.__call_dt__(user_function, tmp, kwargs) + with keep_arguments(tuple(tmp), kwargs, prepend_strings=True): + # no need to do anything on the worker side + ret = user_function(*tmp, **kwargs) + + return ret + + dt_f.__doc__ = user_function.__doc__ + _transform.__doc__ = user_function.__doc__ + return dt_f + + def __call_dt__( # pylint: disable=too-many-branches + self, user_function, args: list, kwargs: dict + ) -> None: + """Extract and call the DT functions. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + dts = [] + self.user_function = user_function + dt_kwargs = self.kwargs.copy() + if __debug__: + logger.debug("Configuring DT core element.") + if "dt" in kwargs: + tmp = kwargs.pop("dt") + if isinstance(tmp, DTObject): + dts.append(tmp.extract()) + elif isinstance(tmp, list): + dts = [obj.extract() for obj in tmp] + elif len(self.args) == 2: + dts.append((self.args[0], self.args[1], dt_kwargs)) + elif self.type is OBJECT_TO_FILE: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is OBJECT_TO_DIRECTORY: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is FILE_TO_OBJECT: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is DIRECTORY_TO_OBJECT: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is FILE_TO_COLLECTION: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is COLLECTION_TO_OBJECT: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is COLLECTION_TO_FILE: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + elif self.type is COLLECTION_TO_DIRECTORY: + dts.append( + (self.target, self.user_function, dt_kwargs, args, kwargs) + ) + if __debug__: + logger.debug("Applying %s DTs.", str(len(dts))) + for _dt in dts: + self._apply_dt(_dt[0], _dt[1], _dt[2], args, kwargs) + + def _apply_dt( + self, param_name, func, func_kwargs, args, kwargs + ): # pylint: disable=R0913,R0912,R0903 + # disable=too-many-arguments, too-many-branches, too-few-public-methods + """Call the data transformation function for the given parameter. + + :param param_name: parameter that DT will be applied to + :param func: DT function + :param func_kwargs: args and kwargs values of the original DT function + :param args: + :param kwargs: + :return: + """ + is_workflow = False + if LABELS.is_workflow in func_kwargs: + is_workflow = func_kwargs.pop(LABELS.is_workflow) + is_workflow = is_workflow in [True, "True", "true", 1, "1"] + + p_value = None + is_kwarg = param_name in kwargs + if is_kwarg: + p_value = kwargs.get(param_name) + else: + all_params = inspect.signature(self.user_function) # type: ignore + keyz = all_params.parameters.keys() + if param_name not in keyz: + raise PyCOMPSsException( + f"Wrong Param {param_name} in data transformation" + ) + i = list(keyz).index(param_name) + if i < len(args): + p_value = args[i] + else: + p_value = all_params.parameters.get( + param_name + ).default # type: ignore + + new_value = None + func_kwargs = _replace_func_kwargs( + func_kwargs, kwargs, args, self.user_function + ) + if is_workflow: + # no need to create a task if it's a workflow + new_value = func(p_value, **func_kwargs) + elif self.type is OBJECT_TO_FILE: + _object_to_file( + p_value, self.destination, self.dt_function, **func_kwargs + ) + new_value = self.destination + elif self.type is OBJECT_TO_DIRECTORY: + _object_to_dir( + p_value, self.destination, self.dt_function, **func_kwargs + ) + new_value = self.destination + elif self.type is FILE_TO_OBJECT: + new_value = _file_to_object( + p_value, self.dt_function, **func_kwargs + ) + elif self.type is DIRECTORY_TO_OBJECT: + new_value = _dir_to_object( + p_value, self.dt_function, **func_kwargs + ) + elif self.type is FILE_TO_COLLECTION: + size = int(func_kwargs.pop("size")) + new_value = _file_to_col( # pylint: disable=unexpected-keyword-arg + p_value, self.dt_function, returns=size, **func_kwargs + ) + elif self.type is COLLECTION_TO_OBJECT: + new_value = _col_to_obj(p_value, self.dt_function, **func_kwargs) + elif self.type is COLLECTION_TO_FILE: + _col_to_file( + p_value, self.destination, self.dt_function, **func_kwargs + ) + new_value = self.destination + elif self.type is COLLECTION_TO_DIRECTORY: + _col_to_dir( + p_value, self.destination, self.dt_function, **func_kwargs + ) + new_value = self.destination + else: + new_value = _transform(p_value, func, **func_kwargs) + + if is_kwarg or i >= len(args): + kwargs[param_name] = new_value + else: + args[i] = new_value + + +def _replace_func_kwargs(dt_f_kwargs, f_kwargs, f_args, func): + for key, value in dt_f_kwargs.items(): + if ( + isinstance(value, str) + and value.startswith("{{") + and value.endswith("}}") + ): + name = value[2:-2] + if name in f_kwargs: + dt_f_kwargs[key] = f_kwargs[name] + else: + dt_f_kwargs[key] = _get_param_from_signature( + func, name, f_args + ) + return dt_f_kwargs + + +def _get_param_from_signature(function, param_name, args): + all_params = inspect.signature(function) # type: ignore + keyz = all_params.parameters.keys() + if param_name not in keyz: + raise PyCOMPSsException( + f"Wrong Param {param_name} in data transformation" + ) + i = list(keyz).index(param_name) + if i < len(args): + p_value = args[i] + else: + p_value = all_params.parameters.get(param_name).default # type: ignore + return p_value + + +class DTObject: # pylint: disable=too-few-public-methods + """Data Transformation Object is a replacement for DT decorator definition. + + Data Transformation Object is a helper class to avoid stack of + decorators or to simplify the definition inside the user code. Arguments of + the object creation of the class is the same as Data Transformation + decorator. It always expects the parameter name as the first element, then + dt_function and the rest of the dt_function kwargs if any. + + """ + + def __init__(self, param_name, func, **func_kwargs): + """Initialize the DTO object with the given arguments. + + :param args: should contain only the & + :param kwargs: kwargs of the user DT function. + """ + self.param_name = param_name + self.func = func + self.func_kwargs = func_kwargs + + def extract(self) -> tuple: + """Extract the DTO object attributes in a tuple. + + :return: tuple of the param name, user function and its kwargs dict. + """ + return self.param_name, self.func, self.func_kwargs + + +# ########################################################################### # +# ############################# ALTERNATIVE NAMES ########################### # +# ########################################################################### # + + +dt = DataTransformation # pylint: disable=invalid-name +data_transformation = DataTransformation # pylint: disable=invalid-name +dto = DTObject # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/decaf.py b/examples/dds/pycompss/api/decaf.py new file mode 100644 index 00000000..08133236 --- /dev/null +++ b/examples/dds/pycompss/api/decaf.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Decaf decorator. + +This file contains the Decaf class, needed for the Decaf task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.df_script} +SUPPORTED_ARGUMENTS = { + LABELS.computing_nodes, + LABELS.working_dir, + LABELS.runner, + LABELS.df_executor, + LABELS.df_lib, + LABELS.df_script, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.computing_nodes, + LEGACY_LABELS.working_dir, + LEGACY_LABELS.df_executor, + LEGACY_LABELS.df_lib, + LEGACY_LABELS.df_script, +} + + +class Decaf: # pylint: disable=too-few-public-methods + """Decaf decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on decaf task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", Decaf.__name__.lower())) + # super(Decaf, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + process_computing_nodes(decorator_name, self.kwargs) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the decaf parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def decaf_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("decaf")) + + if __debug__: + logger.debug("Executing decaf_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + # Set the computing_nodes variable in kwargs for its usage + # in @task decorator + kwargs[LABELS.computing_nodes] = self.kwargs[ + LABELS.computing_nodes + ] + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + decaf_f.__doc__ = user_function.__doc__ + return decaf_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @decaf. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @decaf core element.") + + # Resolve @decaf specific parameters + runner = "mpirun" + if LABELS.runner in self.kwargs: + runner = self.kwargs[LABELS.runner] + + if LEGACY_LABELS.df_script in self.kwargs: + df_script = self.kwargs[LEGACY_LABELS.df_script] + else: + df_script = self.kwargs[LABELS.df_script] + + if LABELS.df_executor in self.kwargs: + df_executor = self.kwargs[LABELS.df_executor] + elif LEGACY_LABELS.df_executor in self.kwargs: + df_executor = self.kwargs[LEGACY_LABELS.df_executor] + else: + df_executor = ( + INTERNAL_LABELS.unassigned + ) # Empty or INTERNAL_LABELS.unassigned + + if LABELS.df_lib in self.kwargs: + df_lib = self.kwargs[LABELS.df_lib] + elif LEGACY_LABELS.df_lib in self.kwargs: + df_lib = self.kwargs[LEGACY_LABELS.df_lib] + else: + df_lib = ( + INTERNAL_LABELS.unassigned + ) # Empty or INTERNAL_LABELS.unassigned + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + impl_type = IMPLEMENTATION_TYPES.decaf + impl_signature = ".".join((impl_type, df_script)) + impl_args = [ + df_script, + df_executor, + df_lib, + self.kwargs[LABELS.working_dir], + runner, + self.kwargs[LABELS.fail_by_exit_value], + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# #################### DECAF DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +decaf = Decaf # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/dummy/__init__.py b/examples/dds/pycompss/api/dummy/__init__.py new file mode 100644 index 00000000..6d6d17ba --- /dev/null +++ b/examples/dds/pycompss/api/dummy/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the API dummy functions, constants and classes.""" diff --git a/examples/dds/pycompss/api/dummy/_decorator.py b/examples/dds/pycompss/api/dummy/_decorator.py new file mode 100644 index 00000000..68cdb6aa --- /dev/null +++ b/examples/dds/pycompss/api/dummy/_decorator.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - decorator. + +This file contains the dummy class task used as decorator. +""" + +from pycompss.util.typing_helper import typing + + +class _Dummy: + """Dummy task class (decorator style).""" + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Construct a dummy Task decorator. + + :param args: Task decorator arguments. + :param kwargs: Task decorator keyword arguments. + :returns: None + """ + self.args = args + self.kwargs = kwargs + + def __call__(self, function: typing.Any) -> typing.Any: + """Invoke the dummy decorator. + + :param function: Decorated function. + :returns: Result of executing the given function. + """ + + def wrapped_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + # returns may appear in @task decorator + if "returns" in kwargs: + kwargs.pop("returns") + return function(*args, **kwargs) + + return wrapped_f + + def __repr__(self) -> str: + attributes = f"(args: {repr(self.args)}, kwargs: {repr(self.kwargs)})" + return f"Dummy {self.__class__.__name__} decorator {attributes}" diff --git a/examples/dds/pycompss/api/dummy/api.py b/examples/dds/pycompss/api/dummy/api.py new file mode 100644 index 00000000..e8def779 --- /dev/null +++ b/examples/dds/pycompss/api/dummy/api.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - dummy - api. + +This file defines the public PyCOMPSs API functions without functionality. +""" + +import os + +from pycompss.util.typing_helper import typing + + +def compss_start( + log_level: str = "off", # pylint: disable=unused-argument + tracing: bool = False, # pylint: disable=unused-argument + interactive: bool = False, # pylint: disable=unused-argument + disable_external: bool = False, # pylint: disable=unused-argument +) -> None: # pylint: disable=unused-argument + """Start runtime dummy. + + Does nothing. + + :param log_level: Log level [ True | False ]. + :param tracing: Activate or disable tracing. + :param interactive: Boolean if interactive (ipython or jupyter). + :param disable_external: To avoid to load compss in external process. + :return: None + """ + + +def compss_stop( + code: int = 0, _hard_stop: bool = False # pylint: disable=unused-argument +) -> None: + """Stop runtime dummy. + + Does nothing. + + :param code: Stop code. + :param _hard_stop: Stop COMPSs when runtime has died. + :return: None + """ + + +def compss_file_exists( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Check if one or more files used in task exists dummy. + + Check if the file/s exists. + + :param file_name: The file/s name to check. + :return: True if exists. False otherwise. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for f_name in file_name: + if isinstance(f_name, (list, tuple)): + ret.append([compss_file_exists(name) for name in f_name]) + else: + ret.append(os.path.exists(f_name)) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_open(file_name: str, mode: str = "r") -> typing.Any: + """Open a file used in task dummy. + + Open the given file with the defined mode (see builtin open). + + :param file_name: The file name to open. + :param mode: Open mode. Options = [w, r+ or a, r or empty]. Default=r. + :return: An object of "file" type. + :raise IOError: If the file can not be opened. + """ + return open(file_name, mode) # pylint: disable=unspecified-encoding + + +def compss_delete_file( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Delete one or more files used in task dummy. + + Does nothing and always return True. + + :param file_name: File/s name. + :return: Always True. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for f_name in file_name: + if isinstance(f_name, (list, tuple)): + ret.append([compss_delete_file(name) for name in f_name]) + else: + ret.append(True) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_wait_on_file( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[list, tuple, str]: + """Wait on file used in task dummy. + + Does nothing. + + :param file_name: File/s name. + :return: The files/s name. + """ + if len(file_name) == 1: + return file_name[0] + return file_name + + +def compss_wait_on_directory( + *directory_name: typing.Union[list, tuple, str], +) -> typing.Union[list, tuple, str]: + """Wait on directory used in task dummy. + + Does nothing. + + :param directory_name: Directory/ies name. + :return: The directory/ies name. + """ + if len(directory_name) == 1: + return directory_name[0] + return directory_name + + +def compss_delete_object( + *objs: typing.Any, +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Delete one or more objects used in task dummy. + + Does nothing and always return True. + + :param objs: Object/s to delete. + :return: Always True. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for obj in objs: + if isinstance(obj, (list, tuple)): + ret.append([compss_delete_object(elem) for elem in obj]) + else: + ret.append(True) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_barrier( + no_more_tasks: bool = False, # pylint: disable=unused-argument +) -> None: + """Wait for all submitted tasks dummy. + + Does nothing. + + :param no_more_tasks: No more tasks boolean. + :return: None + """ + + +def compss_barrier_group( + group_name: str, # pylint: disable=unused-argument +) -> None: + """Wait for all submitted tasks of a group dummy. + + Does nothing. + + :param group_name: Name of the group. + :return: None + """ + + +def compss_cancel_group( + group_name: str, # pylint: disable=unused-argument +) -> None: + """Cancel all submitted tasks of a group dummy. + + Does nothing. + + :param group_name: Name of the group. + :return: None + """ + + +def compss_snapshot() -> None: + """Request a snapshot. + + Does nothing. + + :return: None + """ + + +def compss_wait_on( + *args: typing.Any, **kwargs: typing.Any # pylint: disable=unused-argument +) -> typing.Any: + """Synchronize an object used in task dummy. + + Does nothing. + + :param args: Objects to wait on. + :param kwargs: Options dictionary. + :return: The same objects defined as parameter. + """ + ret = list(map(lambda o: o, args)) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_get_number_of_resources() -> int: + """Request for the number of active resources dummy. + + Does nothing. + + :return: The number of active resources + """ + return 1 + + +def compss_request_resources( + num_resources: int, # pylint: disable=unused-argument + group_name: typing.Optional[str], # pylint: disable=unused-argument +) -> None: + """Request the creation of num_resources resources dummy. + + Does nothing. + + :param num_resources: Number of resources to create. + :param group_name: Task group to notify upon resource creation + :return: None + """ + + +def compss_free_resources( + num_resources: int, # pylint: disable=unused-argument + group_name: typing.Optional[str], # pylint: disable=unused-argument +) -> None: + """Request the destruction of num_resources resources dummy. + + Does nothing. + + :param num_resources: Number of resources to destroy. + :param group_name: Task group to notify upon resource creation + :return: None + """ + + +def compss_set_wall_clock( + wall_clock_limit: int, # pylint: disable=unused-argument +) -> None: + """Set the application wall_clock_limit dummy. + + Does nothing. + + :param wall_clock_limit: Wall clock limit in seconds. + :return: None + """ + + +class TaskGroup: + """Dummy TaskGroup context manager.""" + + def __init__( + self, group_name: str, implicit_barrier: bool = True + ) -> None: # pylint: disable=unused-argument + """Define a new group of tasks. + + :param group_name: Group name. + :param implicit_barrier: Perform implicit barrier. + """ + + def __enter__(self) -> None: + """Do nothing. + + :return: None + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Do nothing. + + :return: None + """ diff --git a/examples/dds/pycompss/api/dummy/constraint.py b/examples/dds/pycompss/api/dummy/constraint.py new file mode 100644 index 00000000..76aa47ba --- /dev/null +++ b/examples/dds/pycompss/api/dummy/constraint.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - constraint. + +This file contains the dummy class constraint used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Constraint = Dummy # pylint: disable=invalid-name +constraint = Dummy # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/dummy/container.py b/examples/dds/pycompss/api/dummy/container.py new file mode 100644 index 00000000..feebeff3 --- /dev/null +++ b/examples/dds/pycompss/api/dummy/container.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - container. + +This file contains the dummy class container used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Container = Dummy # pylint: disable=invalid-name +container = Dummy # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/dummy/on_failure.py b/examples/dds/pycompss/api/dummy/on_failure.py new file mode 100644 index 00000000..5aab87fa --- /dev/null +++ b/examples/dds/pycompss/api/dummy/on_failure.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - on failure. + +This file contains the dummy class on failure used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +OnFailure = Dummy # pylint: disable=invalid-name +on_failure = Dummy # pylint: disable=invalid-name +onFailure = Dummy # noqa: N816 # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/dummy/reduction.py b/examples/dds/pycompss/api/dummy/reduction.py new file mode 100644 index 00000000..b1c265b7 --- /dev/null +++ b/examples/dds/pycompss/api/dummy/reduction.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - reduction. + +This file contains the dummy class reduction used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Reduction = Dummy # pylint: disable=invalid-name +reduction = Dummy # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/dummy/task.py b/examples/dds/pycompss/api/dummy/task.py new file mode 100644 index 00000000..096b6921 --- /dev/null +++ b/examples/dds/pycompss/api/dummy/task.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - task. + +This file contains the dummy class task used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Task = Dummy # pylint: disable=invalid-name +task = Dummy # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/epilog.py b/examples/dds/pycompss/api/epilog.py new file mode 100644 index 00000000..7172f969 --- /dev/null +++ b/examples/dds/pycompss/api/epilog.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Epilog decorator. + +This file contains the Epilog class, needed for the task epilog definition +through the decorator. +""" +import typing +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.binary} +SUPPORTED_ARGUMENTS = { + LABELS.binary, + LABELS.args, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class Epilog: # pylint: disable=too-few-public-methods + """Epilog decorator class. + + If defined, will execute the binary after the task execution on the worker. + Should always be added on top of the 'task' definition. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given binary and args strings. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", Epilog.__name__.lower())) + # super(Epilog, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Call Epilog simply updates the CE and saves Epilog parameters. + + :param user_function: User function to be decorated. + :return: Decorated dummy user function. + """ + + @wraps(user_function) + def epilog_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotImplementedError + + if __debug__: + logger.debug("Executing epilog wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + epilog_f.__doc__ = user_function.__doc__ + return epilog_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @epilog. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @epilog core element.") + + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs, def_val="false") + + binary = self.kwargs[LABELS.binary] + epilog_args = self.kwargs.get(LABELS.args, INTERNAL_LABELS.unassigned) + fail_by = self.kwargs.get(LABELS.fail_by_exit_value) + _epilog = [binary, epilog_args, fail_by] + + core_element = kwargs.get(CORE_ELEMENT_KEY, CE()) + core_element.set_impl_epilog(_epilog) + kwargs[CORE_ELEMENT_KEY] = core_element + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### EPILOG DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +epilog = Epilog # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/exceptions.py b/examples/dds/pycompss/api/exceptions.py new file mode 100644 index 00000000..c03680d2 --- /dev/null +++ b/examples/dds/pycompss/api/exceptions.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Exception. + +This file defines the public PyCOMPSs exception. +""" + + +class COMPSsException(Exception): + """COMPSs generic exception. + + Raised by the user code. + """ + + __slots__ = ["message", "target_direction"] + + def __init__(self, message: str, target_direction: int = -1) -> None: + """Store the arguments passed to the constructor. + + :param message: Exception message. + :param target_direction: Target direction. + """ + super().__init__(message) # show traceback + self.message = message + self.target_direction = target_direction diff --git a/examples/dds/pycompss/api/http.py b/examples/dds/pycompss/api/http.py new file mode 100644 index 00000000..7bad995a --- /dev/null +++ b/examples/dds/pycompss/api/http.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Http decorator. + +This file contains the HTTP class, needed for the http task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.serialization import serializer +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.service_name, LABELS.resource, LABELS.request} +SUPPORTED_ARGUMENTS = { + LABELS.payload, + LABELS.payload_type, + LABELS.produces, + LABELS.updates, +} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class HTTP: # pylint: disable=too-few-public-methods + """HTTP decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on http task creation. + """ + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given http parameters. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", HTTP.__name__.lower())) + # super(HTTP, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + self.task_type = "http" + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the http parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def http_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + # force to serialize with JSON + serializer.FORCED_SERIALIZER = 4 + if not self.scope: + # run http + self.__run_http__(args, kwargs) + + if __debug__: + logger.debug("Executing http_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + http_f.__doc__ = user_function.__doc__ + return http_f + + @staticmethod + def __run_http__( + *args: typing.Any, # pylint: disable=unused-argument + **kwargs: typing.Any # pylint: disable=unused-argument + ) -> int: + """HTTP tasks are meant to be dummy. + + :param args: Arguments received from call. + :param kwargs: Keyword arguments received from call. + :return: Execution return code. + """ + if __debug__: + logger.debug("Running HTTP task ...") + return 200 + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @http. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @http core element.") + impl_type = IMPLEMENTATION_TYPES.http + impl_args = [ + self.kwargs["service_name"], + self.kwargs["resource"], + self.kwargs["request"], + self.kwargs.get("payload", "#"), + self.kwargs.get("payload_type", "application/json"), + self.kwargs.get("produces", "#"), + self.kwargs.get("updates", "#"), + # default value in case of failure + kwargs.get("defaults", {}).get("returns", "#"), + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ##################### HTTP DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +http = HTTP # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/implement.py b/examples/dds/pycompss/api/implement.py new file mode 100644 index 00000000..b1b333c2 --- /dev/null +++ b/examples/dds/pycompss/api/implement.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Implement (Versioning) decorator. + +This file contains the Implement class, needed for the implement definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.features import TASK_FEATURES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.source_class, LABELS.method} +SUPPORTED_ARGUMENTS = {LABELS.source_class, LABELS.method} +DEPRECATED_ARGUMENTS = {LEGACY_LABELS.source_class} + + +class Implement: # pylint: disable=too-few-public-methods + """Implement decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on implementation task creation. + """ + + __slots__ = [ + "first_register", + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given implement parameters. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + self.first_register = False + decorator_name = "".join(("@", Implement.__name__.lower())) + # super(Implement, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the implement parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def implement_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + # This is executed only when called. + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("implement")) + + if __debug__: + logger.debug("Executing implement_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + implement_f.__doc__ = user_function.__doc__ + + if CONTEXT.in_master() and not self.first_register: + self.first_register = True + TASK_FEATURES.set_register_only(True) + self.__call__(user_function)(self) + TASK_FEATURES.set_register_only(False) + + return implement_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @implement. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @implement core element.") + + # Resolve @implement specific parameters + if LEGACY_LABELS.source_class in self.kwargs: + another_class = self.kwargs[LEGACY_LABELS.source_class] + self.kwargs[LABELS.source_class] = self.kwargs.pop( + LEGACY_LABELS.source_class + ) + else: + another_class = self.kwargs[LABELS.source_class] + another_method = self.kwargs[LABELS.method] + ce_signature = ".".join((another_class, another_method)) + impl_type = IMPLEMENTATION_TYPES.method + # impl_args = [another_class, another_method] - set by @task + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_ce_signature(ce_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + # @task sets the implementation type arguments + # kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @implement is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_ce_signature(ce_signature) + core_element.set_impl_type(impl_type) + # @task sets the implementation type arguments + # core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################## IMPLEMENT DECORATOR ALTERNATIVE NAME ################### # +# ########################################################################### # + +implement = Implement # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/julia.py b/examples/dds/pycompss/api/julia.py new file mode 100644 index 00000000..4c2b6a36 --- /dev/null +++ b/examples/dds/pycompss/api/julia.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Julia decorator. + +This file contains the Julia class, needed for the Julia task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.api.multinode import set_slurm_environment +from pycompss.api.multinode import reset_slurm_environment +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.julia_script} +SUPPORTED_ARGUMENTS = { + LABELS.julia_executor, + LABELS.julia_args, + LABELS.julia_script, + LABELS.fail_by_exit_value, + LABELS.working_dir, + LABELS.computing_nodes, +} +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.working_dir, + LEGACY_LABELS.computing_nodes, +} + + +class Julia: # pylint: disable=too-few-public-methods + """Julia decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on julia task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", Julia.__name__.lower())) + # super(Julia, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + process_computing_nodes(decorator_name, self.kwargs) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the julia parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def julia_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("julia")) + + if __debug__: + logger.debug("Executing julia_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + multinode_decorator_defined = False + old_slurm_env = {} + if LABELS.computing_nodes in kwargs: + multinode_decorator_defined = True + # We need to do what @multinode does: + if CONTEXT.in_worker(): + old_slurm_env = set_slurm_environment() + + # Set the computing_nodes variable in kwargs for its usage + # in @task decorator + kwargs[LABELS.computing_nodes] = self.kwargs[ + LABELS.computing_nodes + ] + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + if multinode_decorator_defined and CONTEXT.in_worker(): + reset_slurm_environment(old_slurm_env) + + return ret + + julia_f.__doc__ = user_function.__doc__ + return julia_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @julia. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @julia core element.") + + # Resolve @julia specific parameters + julia_executor = "julia" + if LABELS.julia_executor in self.kwargs: + julia_executor = self.kwargs[LABELS.julia_executor] + + julia_args = "" + if LABELS.julia_args in self.kwargs: + julia_args = self.kwargs[LABELS.julia_args] + + julia_script = self.kwargs[LABELS.julia_script] + + # Resolve computing nodes + if LABELS.computing_nodes in kwargs: + # It has been defined in @multinode + computing_nodes = kwargs[LABELS.computing_nodes] + elif LABELS.computing_nodes in self.kwargs: + # It has been defined in @julia + computing_nodes = self.kwargs[LABELS.computing_nodes] + else: + # Not defined anywhere + computing_nodes = "1" + self.kwargs[LABELS.computing_nodes] = computing_nodes + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + impl_type = IMPLEMENTATION_TYPES.julia + impl_signature = ".".join((impl_type, julia_script)) + impl_args = [ + julia_executor, + julia_args, + julia_script, + self.kwargs[LABELS.working_dir], + self.kwargs[LABELS.fail_by_exit_value], + self.kwargs[LABELS.computing_nodes], + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# #################### DECAF DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +julia = Julia # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/local.py b/examples/dds/pycompss/api/local.py new file mode 100644 index 00000000..f7f923fb --- /dev/null +++ b/examples/dds/pycompss/api/local.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Local decorator. + +This file contains the @local decorator, which is intended to be a decorator +for non-task functions that may receive future objects as parameters +(i.e: their inputs are pycompss task outputs). It also handles INOUTs +""" + +import gc + +from pycompss.util.context import CONTEXT +from pycompss.api.api import compss_wait_on +from pycompss.runtime.management.object_tracker import OT +from pycompss.util.objects.replace import replace +from pycompss.util.typing_helper import typing + + +def local(input_function: typing.Callable) -> typing.Callable: + """Local decorator. + + :param input_function: Input function. + :return: Wrapped function. + """ + if not CONTEXT.in_pycompss(): + # Return dummy local decorator + + def wrapped_function( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + return input_function(*args, **kwargs) + + else: + + def sync_if_needed(obj: typing.Any) -> None: + if OT.is_obj_pending_to_synchronize(obj): + new_val = compss_wait_on(obj) + replace(obj, new_val) + + def wrapped_function( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + gc.collect() + _args = [] + _kwargs = {} + for arg in args: + sync_if_needed(arg) + _args.append(arg) + for key, value in kwargs.items(): + sync_if_needed(value) + _kwargs[key] = value + return input_function(*_args, **_kwargs) + + return wrapped_function + + +# ########################################################################### # +# #################### LOCAL DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +Local = local # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/mpi.py b/examples/dds/pycompss/api/mpi.py new file mode 100644 index 00000000..493843e3 --- /dev/null +++ b/examples/dds/pycompss/api/mpi.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Mpi decorator. + +This file contains the MPI class, needed for the mpi task definition through +the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.decorator import run_command +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.runner} +SUPPORTED_ARGUMENTS = { + LABELS.binary, + LABELS.processes, + LABELS.working_dir, + LABELS.runner, + LABELS.flags, + LABELS.processes_per_node, + LABELS.scale_by_cu, + LABELS.args, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.computing_units, + LEGACY_LABELS.computing_nodes, + LEGACY_LABELS.working_dir, +} + + +class Mpi: # pylint: disable=too-few-public-methods + """Mpi decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on mpi task creation. + """ + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given mpi parameters. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + self.task_type = "mpi" + self.decorator_name = "".join(("@", Mpi.__name__.lower())) + # super(MPI, self).__init__(decorator_name, *args, **kwargs) + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + if __debug__: + logger.debug("Init @mpi decorator...") + + # TODO: Maybe add here the collection layout to avoid iterate + # twice per elements + # Add _layout params to SUPPORTED_ARGUMENTS + for key in self.kwargs: + if "_layout" in key: + SUPPORTED_ARGUMENTS.add(key) + + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + self.decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the mpi parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def mpi_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + return self.__decorator_body__(user_function, args, kwargs) + + mpi_f.__doc__ = user_function.__doc__ + return mpi_f + + def __decorator_body__( + self, user_function: typing.Callable, args: tuple, kwargs: dict + ) -> typing.Any: + """Body of the mpi decorator. + + :param user_function: Decorated function. + :param args: Function arguments. + :param kwargs: Function keyword arguments. + :returns: Result of executing the user_function with the given args + and kwargs. + """ + if not self.scope: + # Execute the mpi as with PyCOMPSs so that sequential + # execution performs as parallel. + # To disable: raise Exception(not_in_pycompss("mpi")) + # TODO: Intercept @task parameters to get stream redirection + if "binary" in self.kwargs: + return self.__run_mpi__(args, kwargs) + print( + "WARN: Python MPI as dummy is not fully supported. " + "Executing decorated function." + ) + return user_function(*args, **kwargs) + + if __debug__: + logger.debug("Executing mpi_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + # The processes' parameter will have to go down until the execution + # is invoked. To this end, set the computing_nodes variable in kwargs + # for its usage in @task decorator + # WARNING: processes can be an int, a env string, a str with + # dynamic variable name. + if "processes" in self.kwargs: + kwargs["computing_nodes"] = self.kwargs["processes"] + else: + # If processes not defined, check computing_units or set default + process_computing_nodes(self.decorator_name, self.kwargs) + kwargs["computing_nodes"] = self.kwargs["computing_nodes"] + if "processes_per_node" in self.kwargs: + kwargs["processes_per_node"] = self.kwargs["processes_per_node"] + else: + kwargs["processes_per_node"] = 1 + if __debug__: + logger.debug( + "This MPI task will have %s processes " + "and %s processes per node.", + str(kwargs["computing_nodes"]), + str(kwargs["processes_per_node"]), + ) + + prepend_strings = self.task_type == IMPLEMENTATION_TYPES.python_mpi + + with keep_arguments(args, kwargs, prepend_strings=prepend_strings): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + def __run_mpi__(self, args: tuple, kwargs: dict) -> int: + """Run the mpi binary defined in the decorator when used as dummy. + + :param args: Arguments received from call. + :param kwargs: Keyword arguments received from call. + :return: Execution return code. + """ + cmd = [self.kwargs[LABELS.runner]] + if LABELS.processes in self.kwargs: + cmd += ["-np", self.kwargs[LABELS.processes]] + elif LABELS.computing_nodes in self.kwargs: + cmd += ["-np", self.kwargs[LABELS.computing_nodes]] + elif LEGACY_LABELS.computing_nodes in self.kwargs: + cmd += ["-np", self.kwargs[LEGACY_LABELS.computing_nodes]] + + if LABELS.flags in self.kwargs: + cmd += self.kwargs[LABELS.flags].split() + cmd += [self.kwargs[LABELS.binary]] + + return run_command(cmd, args, kwargs) + + def __resolve_collection_layout_params__(self) -> list: + """Resolve the collection layout, such as blocks, strides, etc. + + :return: list(param_name, block_count, block_length, stride). + :raises PyCOMPSsException: If the collection layout does not contain + block_count. + """ + num_layouts = 0 + layout_params = [] + for key, value in self.kwargs.items(): + if "_layout" in key: + num_layouts += 1 + param_name = key.split("_layout")[0] + collection_layout = value + + block_count = self.__get_block_count__(collection_layout) + block_length = self.__get_block_length__(collection_layout) + stride = self.__get_stride__(collection_layout) + + if (block_length != -1 and block_count == -1) or ( + stride != -1 and block_count == -1 + ): + msg = "Error: collection_layout must contain block_count!" + raise PyCOMPSsException(msg) + layout_params.extend( + [ + param_name, + str(block_count), + str(block_length), + str(stride), + ] + ) + layout_params.insert(0, str(num_layouts)) + return layout_params + + @staticmethod + def __get_block_count__(collection_layout: dict) -> int: + """Get the block count from the given collection layout. + + :param collection_layout: Collection layout. + :return: Block count value. + """ + if "block_count" in collection_layout: + return collection_layout["block_count"] + return -1 + + @staticmethod + def __get_block_length__(collection_layout: dict) -> int: + """Get the block length from the given collection layout. + + :param collection_layout: Collection layout. + :return: Block length value. + """ + if "block_length" in collection_layout: + return collection_layout["block_length"] + return -1 + + @staticmethod + def __get_stride__(collection_layout: dict) -> int: + """Get the stride from the given collection layout. + + :param collection_layout: Collection layout. + :return: Stride value. + """ + if "stride" in collection_layout: + return collection_layout["stride"] + return -1 + + def __configure_core_element__( # pylint: disable=too-many-branches + self, kwargs: dict + ) -> None: + """Include the registering info related to @mpi. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @mpi core element.") + + # Resolve @mpi specific parameters + if LABELS.binary in self.kwargs: + binary = self.kwargs[LABELS.binary] + impl_type = IMPLEMENTATION_TYPES.mpi + else: + binary = INTERNAL_LABELS.unassigned + impl_type = IMPLEMENTATION_TYPES.python_mpi + self.task_type = impl_type + + runner = self.kwargs[LABELS.runner] + + if LABELS.flags in self.kwargs: + flags = self.kwargs[LABELS.flags] + else: + flags = ( + INTERNAL_LABELS.unassigned + ) # Empty or INTERNAL_LABELS.unassigned + + # Check if scale by cu is defined + scale_by_cu_str = self.__resolve_scale_by_cu__() + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + # Resolve parameter collection layout + collection_layout_params = self.__resolve_collection_layout_params__() + + if LABELS.processes in self.kwargs: + proc = self.kwargs[LABELS.processes] + elif LABELS.computing_nodes in self.kwargs: + proc = self.kwargs[LABELS.computing_nodes] + elif LEGACY_LABELS.computing_nodes in self.kwargs: + proc = self.kwargs[LEGACY_LABELS.computing_nodes] + else: + proc = "1" + + if LABELS.processes_per_node in self.kwargs: + ppn = str(self.kwargs[LABELS.processes_per_node]) + else: + ppn = "1" + + if binary == INTERNAL_LABELS.unassigned: + impl_signature = impl_type + "." + else: + impl_signature = ".".join((impl_type, str(proc), binary)) + + impl_args = [ + binary, + self.kwargs[LABELS.working_dir], + runner, + ppn, + flags, + scale_by_cu_str, + self.kwargs.get(LABELS.args, INTERNAL_LABELS.unassigned), + self.kwargs[LABELS.fail_by_exit_value], + ] + + if impl_type == IMPLEMENTATION_TYPES.python_mpi: + impl_args = impl_args + collection_layout_params + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + def __resolve_scale_by_cu__(self) -> str: + """Check if scale_by_cu is defined and process it. + + :return: Scale by cu value as string. + :raises PyCOMPSsException: If scale_by_cu is not bool or string. + """ + if LABELS.scale_by_cu in self.kwargs: + scale_by_cu = self.kwargs[LABELS.scale_by_cu] + if isinstance(scale_by_cu, bool): + if scale_by_cu: + scale_by_cu_str = "true" + else: + scale_by_cu_str = "false" + elif str(scale_by_cu).lower() in ["true", "false"]: + scale_by_cu_str = str(scale_by_cu).lower() + else: + raise PyCOMPSsException( + "Incorrect format for scale_by_cu property. " + "It should be boolean or 'true' or 'false'" + ) + else: + scale_by_cu_str = "false" + return scale_by_cu_str + + +# ########################################################################### # +# ##################### MPI DECORATOR ALTERNATIVE NAME ###################### # +# ########################################################################### # + +mpi = Mpi # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/mpmd_mpi.py b/examples/dds/pycompss/api/mpmd_mpi.py new file mode 100644 index 00000000..097986d8 --- /dev/null +++ b/examples/dds/pycompss/api/mpmd_mpi.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - MPMD MPI decorator. + +This file contains the mpmd mpi class, needed for the multiple program mpi +task definition through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.runner} +SUPPORTED_ARGUMENTS = { + LABELS.runner, + LABELS.programs, + LABELS.working_dir, + LABELS.processes_per_node, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class MPMDMPI: # pylint: disable=R0903,R0902 + # disable=too-few-public-methods, too-many-instance-attributes + """MPMDMPI decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on mpmd_mpi task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + "task_type", + "processes", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given mpi parameters. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", MPMDMPI.__name__.lower())) + # super(MPMDMPI, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + # MPMD_MPI specific: + self.task_type = "mpmd_mpi" + self.processes = 0 + if self.scope: + if __debug__: + logger.debug("Init @mpmd_mpi decorator...") + + # Add _layout params to SUPPORTED_ARGUMENTS + for key in self.kwargs: + if "_layout" in key: + SUPPORTED_ARGUMENTS.add(key) + + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + self.decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the mpmd mpi parameters within the task core element. + + :param user_function: User function to be decorated. + :return: Decorated dummy user function, which will invoke MPMD MPI + task. + """ + + @wraps(user_function) + def mpmd_mpi_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + return self.__decorator_body__(user_function, args, kwargs) + + mpmd_mpi_f.__doc__ = user_function.__doc__ + return mpmd_mpi_f + + def __decorator_body__( + self, user_function: typing.Callable, args: tuple, kwargs: dict + ) -> typing.Any: + """Body of the mpmd_mpi decorator. + + :param user_function: Decorated function. + :param args: Function arguments. + :param kwargs: Function keyword arguments. + :returns: Result of executing the user_function with the given args + and kwargs. + """ + if not self.scope: + raise NotImplementedError + + if __debug__: + logger.debug("Executing mpmd_mpi_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + kwargs[LABELS.processes_per_node] = self.kwargs.get( + LABELS.processes_per_node, 1 + ) + kwargs[LABELS.computing_nodes] = self.processes + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + def __get_programs_params__(self) -> list: + """Resolve the collection layout, such as blocks, strides, etc. + + :return: list(programs_length, binary, args, processes) + :raises PyCOMPSsException: If programs are not dict objects. + """ + programs = self.kwargs[LABELS.programs] + programs_args = [str(len(programs))] + + for program in programs: + if not isinstance(program, dict): + raise PyCOMPSsException( + "Incorrect 'program' param in MPMD MPI" + ) + + binary = program.get(LABELS.binary, None) + if not binary: + raise PyCOMPSsException("No binary file provided for MPMD MPI") + + params = program.get(LABELS.args, INTERNAL_LABELS.unassigned) + procs = str(program.get(LABELS.processes, 1)) + programs_args.extend([binary, params, procs]) + + # increase total # of processes for this mpmd task + self.processes += program.get(LABELS.processes, 1) + + return programs_args + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @mpmd_mpi. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @mpmd_mpi core element.") + + # Resolve @mpmd_mpi specific parameters + impl_type = "MPMDMPI" + runner = self.kwargs[LABELS.runner] + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + ppn = str(self.kwargs.get(LABELS.processes_per_node, 1)) + impl_signature = ".".join((impl_type, str(ppn))) + + prog_params = self.__get_programs_params__() + + impl_args = [ + runner, + self.kwargs[LABELS.working_dir], + ppn, + self.kwargs[LABELS.fail_by_exit_value], + ] + impl_args.extend(prog_params) + + if CORE_ELEMENT_KEY in kwargs: + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ##################### MPI DECORATOR ALTERNATIVE NAME ###################### # +# ########################################################################### # + +mpmd_mpi = MPMDMPI # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/multinode.py b/examples/dds/pycompss/api/multinode.py new file mode 100644 index 00000000..d3285d18 --- /dev/null +++ b/examples/dds/pycompss/api/multinode.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Multinode decorator. + +This file contains the Multinode class, needed for the MultiNode task +definition through the decorator. +""" + +import os +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = set() # type: typing.Set[str] +SUPPORTED_ARGUMENTS = {LABELS.computing_nodes, LABELS.processes_per_node} +DEPRECATED_ARGUMENTS = {LEGACY_LABELS.computing_nodes} +SLURM_SKIP_VARS = [ + "SLURM_JOBID", + "SLURM_JOB_ID", + "SLURM_USER", + "SLURM_QOS", + "SLURM_PARTITION", + "SLURM_OVERLAP", +] + + +class MultiNode: # pylint: disable=too-few-public-methods + """MultiNode decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on MultiNode task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", MultiNode.__name__.lower())) + # super(MultiNode, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + process_computing_nodes(decorator_name, self.kwargs) + if LABELS.processes_per_node not in kwargs: + kwargs[LABELS.processes_per_node] = 1 + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the multinode parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def multinode_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("MultiNode")) + + if __debug__: + logger.debug("Executing multinode_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + if CONTEXT.in_worker(): + old_slurm_env = set_slurm_environment() + + # Set the computing_nodes variable in kwargs for its usage + # in @task decorator + kwargs[LABELS.computing_nodes] = self.kwargs[ + LABELS.computing_nodes + ] + if LABELS.processes_per_node in self.kwargs: + kwargs[LABELS.processes_per_node] = self.kwargs[ + LABELS.processes_per_node + ] + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + if CONTEXT.in_worker(): + reset_slurm_environment(old_slurm_env) + + return ret + + multinode_f.__doc__ = user_function.__doc__ + return multinode_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @multinode. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @multinode core element.") + + # Resolve @multinode specific parameters + impl_type = IMPLEMENTATION_TYPES.multi_node + if LABELS.processes_per_node in self.kwargs: + ppn = str(self.kwargs[LABELS.processes_per_node]) + else: + ppn = "1" + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY] = core_element + if __debug__: + logger.debug("Adding ppn in core element impl type args.") + kwargs[CORE_ELEMENT_KEY].set_impl_type_args([ppn]) + + # Set as configured + self.core_element_configured = True + + +def set_slurm_environment() -> dict: + """Set SLURM environment. + + :return: old Slurm environment + """ + num_nodes = int(os.environ["COMPSS_NUM_NODES"]) + num_threads = int(os.environ["COMPSS_NUM_THREADS"]) + total_processes = num_nodes * num_threads + hostnames = os.environ["COMPSS_HOSTNAMES"] + nodes = set(hostnames.split(",")) + old_slurm_env = remove_slurm_environment() + # set slurm environment with COMPSs variables + os.environ["SLURM_NTASKS"] = str(total_processes) + os.environ["SLURM_NNODES"] = str(num_nodes) + os.environ["SLURM_JOB_NUM_NODES"] = str(num_nodes) + os.environ["SLURM_NODELIST"] = ",".join(nodes) + os.environ["SLURM_JOB_NODELIST"] = ",".join(nodes) + os.environ["SLURM_TASKS_PER_NODE"] = "".join( + (str(num_threads), "(x", str(num_nodes), ")") + ) + os.environ["SLURM_CPUS_PER_NODE"] = "".join( + (str(num_threads), "(x", str(num_nodes), ")") + ) + return old_slurm_env + + +def remove_slurm_environment() -> dict: + """Remove the Slurm variables from the environment. + + :return: removed Slurm variables. + """ + old_slurm_env = {} + for key, value in os.environ.items(): + if key.startswith("SLURM"): + if key not in SLURM_SKIP_VARS: + old_slurm_env[key] = value + os.environ.pop(key) + # TODO: ISSUE DECTECTED - WAS NOT RETURNING old_slurm_env: ASK JORGE + return old_slurm_env + + +def reset_slurm_environment( + old_slurm_env: typing.Optional[dict] = None, +) -> None: + """Reestablish SLURM environment. + + :return: None + """ + if old_slurm_env: + for key, value in old_slurm_env.items(): + os.environ[key] = value + + +# ########################################################################### # +# ################## MultiNode DECORATOR ALTERNATIVE NAME ################### # +# ########################################################################### # + +multinode = MultiNode # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/ompss.py b/examples/dds/pycompss/api/ompss.py new file mode 100644 index 00000000..0f275ba2 --- /dev/null +++ b/examples/dds/pycompss/api/ompss.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - OmpSs decorator. + +This file contains the OmpSs class, needed for the OmpSs task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import process_computing_nodes +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.binary} +SUPPORTED_ARGUMENTS = { + LABELS.computing_nodes, + LABELS.working_dir, + LABELS.binary, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.computing_nodes, + LEGACY_LABELS.working_dir, +} + + +class OmpSs: # pylint: disable=too-few-public-methods + """OmpSs decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on OmpSs task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", OmpSs.__name__.lower())) + # super(OmpSs, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + process_computing_nodes(decorator_name, self.kwargs) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the ompss parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def ompss_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("ompss")) + + if __debug__: + logger.debug("Executing ompss_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + # Set the computing_nodes variable in kwargs for its usage + # in @task decorator + kwargs[LABELS.computing_nodes] = self.kwargs[ + LABELS.computing_nodes + ] + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + ompss_f.__doc__ = user_function.__doc__ + return ompss_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @ompss. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @ompss core element.") + + # Resolve @ompss specific parameters + binary = self.kwargs[LABELS.binary] + + # Resolve the working directory + resolve_working_dir(self.kwargs) + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + impl_type = IMPLEMENTATION_TYPES.ompss + impl_signature = "".join((IMPLEMENTATION_TYPES.ompss, ".", binary)) + impl_args = [ + binary, + self.kwargs[LABELS.working_dir], + self.kwargs[LABELS.fail_by_exit_value], + ] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# #################### OMPSs DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +ompss = OmpSs # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/on_failure.py b/examples/dds/pycompss/api/on_failure.py new file mode 100644 index 00000000..43f02111 --- /dev/null +++ b/examples/dds/pycompss/api/on_failure.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - on_failure decorator. + +This file contains the on_failure class, needed for the on failure management +definition through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.dummy.on_failure import on_failure as dummy_on_failure +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_mandatory_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + + +MANDATORY_ARGUMENTS = {LABELS.management} +SUPPORTED_ARGUMENTS = { + LABELS.management_ignore, + LABELS.management_retry, + LABELS.management_cancel_successor, + LABELS.management_fail, +} + + +class OnFailure: # pylint: disable=R0903,R0902 + # disable=too-few-public-methods, too-many-instance-attributes + """OnFailure decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on task on_failure task management definition. + """ + + __slots__ = [ + "on_failure_action", + "defaults", + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given on_failure. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", OnFailure.__name__.lower())) + # super(OnFailure, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_mandatory_arguments( + MANDATORY_ARGUMENTS, list(kwargs.keys()), decorator_name + ) + + # Save the parameters into self so that they can be accessed when + # the task fails and the action needs to be taken + self.on_failure_action = kwargs.pop(LABELS.management) + # Check supported management values + if self.on_failure_action not in SUPPORTED_ARGUMENTS: + raise PyCOMPSsException( + f"ERROR: Unsupported on failure action: " + f"{self.on_failure_action}" + ) + # Keep all defaults in a dictionary + self.defaults = kwargs + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the on_failure within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def constrained_f( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + if not self.scope: + d_c = dummy_on_failure(self.args, self.kwargs) + return d_c.__call__(user_function)(*args, **kwargs) + + if __debug__: + logger.debug("Executing on_failure_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + # Set the on failure management action and default variables in + # kwargs for its usage in @task decorator + kwargs["on_failure"] = self.on_failure_action + kwargs["defaults"] = self.defaults + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + constrained_f.__doc__ = user_function.__doc__ + return constrained_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @on_failure. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Current keyword arguments to be updated with the core + element information. + :return: None + """ + if __debug__: + logger.debug("Configuring @on_failure core element.") + + if CORE_ELEMENT_KEY not in kwargs: + # @on_failure is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + kwargs[CORE_ELEMENT_KEY] = CE() + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### ON FAILURE DECORATOR ALTERNATIVE NAME ################# # +# ########################################################################### # + +on_failure = OnFailure # pylint: disable=invalid-name +onFailure = OnFailure # noqa: N816 # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/opencl.py b/examples/dds/pycompss/api/opencl.py new file mode 100644 index 00000000..02d4ef1d --- /dev/null +++ b/examples/dds/pycompss/api/opencl.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - OpenCL decorator. + +This file contains the OpenCL class, needed for the opencl task definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import resolve_working_dir +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.kernel} +SUPPORTED_ARGUMENTS = {LABELS.kernel, LABELS.working_dir} +DEPRECATED_ARGUMENTS = {LEGACY_LABELS.working_dir} + + +class OpenCL: # pylint: disable=too-few-public-methods + """OpenCL decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on opencl task creation. + """ + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given constraints. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + decorator_name = "".join(("@", OpenCL.__name__.lower())) + # super(OpenCL, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the opencl parameters within the task core element. + + :param user_function: Function to decorate. + :return: Decorated function. + """ + + @wraps(user_function) + def opencl_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotInPyCOMPSsException(not_in_pycompss("opencl")) + + if __debug__: + logger.debug("Executing opencl_f wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + # master code - or worker with nesting enabled + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + opencl_f.__doc__ = user_function.__doc__ + return opencl_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @opencl. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @opencl core element.") + + # Resolve @opencl specific parameters + kernel = self.kwargs[LABELS.kernel] + + # Resolve the working directory + resolve_working_dir(self.kwargs) + + impl_type = IMPLEMENTATION_TYPES.opencl + impl_signature = ".".join((impl_type, kernel)) + impl_args = [kernel, self.kwargs[LABELS.working_dir]] + + if CORE_ELEMENT_KEY in kwargs: + # Core element has already been created in a higher level decorator + # (e.g. @constraint) + kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) + kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + else: + # @binary is in the top of the decorators stack. + # Instantiate a new core element object, update it and include + # it into kwarg + core_element = CE() + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### OPENCL DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +opencl = OpenCL # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/parameter.py b/examples/dds/pycompss/api/parameter.py new file mode 100644 index 00000000..bca31da3 --- /dev/null +++ b/examples/dds/pycompss/api/parameter.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Parameter definitions. + +This file contains the classes needed for the parameter definition. +1. DIRECTION. + - IN + - OUT + - INOUT + - CONCURRENT + - COMMUTATIVE + - IN_DELETE +2. TYPE. + - FILE + - BOOLEAN + - STRING + - INT + - LONG + - FLOAT + - OBJECT + - COLLECTION + - DICTIONARY + - EXTERNAL_PSCO + - EXTERNAL_STREAM + - ... (check _DataType generated during installation) +3. IOSTREAM. + - STDIN + - STDOUT + - STDERR + - UNSPECIFIED +4. PREFIX. +5. ALIASES. +""" + +from pycompss.api.commons.data_type import DataType as _DataType +from pycompss.runtime.mpi.keys import MPI_LAYOUT_KEYS as _MPI_LAYOUT_KEYS +from pycompss.runtime.task.keys import PARAM_ALIAS_KEYS as _PARAM_ALIAS_KEYS +from pycompss.runtime.task.keys import PARAM_DICT_KEYS as _PARAM_DICT_KEYS + +# Type definitions -> Numbers match both C and Java enums and are generated +# during the installation. +TYPE = _DataType + + +# Numbers match both C and Java enums +class SupportedDirections: # pylint: disable=too-few-public-methods + """Used as enum for direction types.""" + + IN = 0 + OUT = 1 + INOUT = 2 + CONCURRENT = 3 + COMMUTATIVE = 4 + IN_DELETE = 5 + + +# Numbers match both C and Java enums +class SupportedIoStreams: # pylint: disable=too-few-public-methods + """Used as enum for stream types.""" + + STDIN = 0 + STDOUT = 1 + STDERR = 2 + UNSPECIFIED = 3 + + +# String that identifies the prefix +class SupportedPrefixes: # pylint: disable=too-few-public-methods + """Used as enum for prefix.""" + + PREFIX = "null" # NOSONAR + + +DIRECTION = SupportedDirections() +IOSTREAM = SupportedIoStreams() +PREFIX = SupportedPrefixes() + + +class _Param: # pylint: disable=too-few-public-methods + """Private class which hides the parameter key to be used.""" + + __slots__ = ["key"] + + def __init__(self, key: str) -> None: + self.key = key + + +# Aliases for objects (just direction) +IN = _Param(_PARAM_ALIAS_KEYS.IN) +OUT = _Param(_PARAM_ALIAS_KEYS.OUT) +INOUT = _Param(_PARAM_ALIAS_KEYS.INOUT) +CONCURRENT = _Param(_PARAM_ALIAS_KEYS.CONCURRENT) +COMMUTATIVE = _Param(_PARAM_ALIAS_KEYS.COMMUTATIVE) +IN_DELETE = _Param(_PARAM_ALIAS_KEYS.IN_DELETE) + +# Aliases for files with direction +FILE = _Param(_PARAM_ALIAS_KEYS.FILE) +FILE_IN = _Param(_PARAM_ALIAS_KEYS.FILE_IN) +FILE_OUT = _Param(_PARAM_ALIAS_KEYS.FILE_OUT) +FILE_INOUT = _Param(_PARAM_ALIAS_KEYS.FILE_INOUT) +FILE_CONCURRENT = _Param(_PARAM_ALIAS_KEYS.FILE_CONCURRENT) +FILE_COMMUTATIVE = _Param(_PARAM_ALIAS_KEYS.FILE_COMMUTATIVE) + +# Aliases for files with stream +FILE_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_STDIN) +FILE_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_STDERR) +FILE_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_STDOUT) + +# Aliases for files with direction and stream +FILE_IN_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_IN_STDIN) +FILE_IN_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_IN_STDERR) +FILE_IN_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_IN_STDOUT) +FILE_OUT_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_OUT_STDIN) +FILE_OUT_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_OUT_STDERR) +FILE_OUT_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_OUT_STDOUT) +FILE_INOUT_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_INOUT_STDIN) +FILE_INOUT_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_INOUT_STDERR) +FILE_INOUT_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_INOUT_STDOUT) +FILE_CONCURRENT_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDIN) +FILE_CONCURRENT_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDERR) +FILE_CONCURRENT_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDOUT) +FILE_COMMUTATIVE_STDIN = _Param(_PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDIN) +FILE_COMMUTATIVE_STDERR = _Param(_PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDERR) +FILE_COMMUTATIVE_STDOUT = _Param(_PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDOUT) + +# Aliases for dirs +DIRECTORY = _Param(_PARAM_ALIAS_KEYS.DIRECTORY) +DIRECTORY_IN = _Param(_PARAM_ALIAS_KEYS.DIRECTORY_IN) +DIRECTORY_OUT = _Param(_PARAM_ALIAS_KEYS.DIRECTORY_OUT) +DIRECTORY_INOUT = _Param(_PARAM_ALIAS_KEYS.DIRECTORY_INOUT) + +# Aliases for collections +COLLECTION = _Param(_PARAM_ALIAS_KEYS.COLLECTION) +COLLECTION_IN = _Param(_PARAM_ALIAS_KEYS.COLLECTION_IN) +COLLECTION_INOUT = _Param(_PARAM_ALIAS_KEYS.COLLECTION_INOUT) +COLLECTION_OUT = _Param(_PARAM_ALIAS_KEYS.COLLECTION_OUT) +COLLECTION_IN_DELETE = _Param(_PARAM_ALIAS_KEYS.COLLECTION_IN_DELETE) +COLLECTION_FILE = _Param(_PARAM_ALIAS_KEYS.COLLECTION_FILE) +COLLECTION_FILE_IN = _Param(_PARAM_ALIAS_KEYS.COLLECTION_FILE_IN) +COLLECTION_FILE_INOUT = _Param(_PARAM_ALIAS_KEYS.COLLECTION_FILE_INOUT) +COLLECTION_FILE_OUT = _Param(_PARAM_ALIAS_KEYS.COLLECTION_FILE_OUT) + +# Aliases for dictionary collections +DICTIONARY = _Param(_PARAM_ALIAS_KEYS.DICT_COLLECTION) +DICTIONARY_IN = _Param(_PARAM_ALIAS_KEYS.DICT_COLLECTION_IN) +DICTIONARY_INOUT = _Param(_PARAM_ALIAS_KEYS.DICT_COLLECTION_INOUT) +DICTIONARY_OUT = _Param(_PARAM_ALIAS_KEYS.DICT_COLLECTION_OUT) +DICTIONARY_IN_DELETE = _Param(_PARAM_ALIAS_KEYS.DICT_COLLECTION_IN_DELETE) + +# Aliases for streams +STREAM_IN = _Param(_PARAM_ALIAS_KEYS.STREAM_IN) +STREAM_OUT = _Param(_PARAM_ALIAS_KEYS.STREAM_OUT) + +# Aliases for std IO streams (just stream direction) +STDIN = IOSTREAM.STDIN +STDOUT = IOSTREAM.STDOUT +STDERR = IOSTREAM.STDERR + +# Aliases for parameter definition as dictionary +# - Parameter type +Type = _PARAM_DICT_KEYS.TYPE # pylint: disable=invalid-name +# - Parameter direction +Direction = _PARAM_DICT_KEYS.DIRECTION # pylint: disable=invalid-name +# - Parameter stream +StdIOStream = _PARAM_DICT_KEYS.STDIOSTREAM # pylint: disable=invalid-name +# - Parameter prefix +Prefix = _PARAM_DICT_KEYS.PREFIX # pylint: disable=invalid-name +# - Collection recursive depth +Depth = _PARAM_DICT_KEYS.DEPTH # pylint: disable=invalid-name +# - Parameter weight +Weight = _PARAM_DICT_KEYS.WEIGHT # pylint: disable=invalid-name +# - Parameter keep rename property +Keep_rename = _PARAM_DICT_KEYS.KEEP_RENAME # pylint: disable=invalid-name +# - Enable/disable store in cache +Cache = _PARAM_DICT_KEYS.CACHE # pylint: disable=invalid-name + +# Aliases for collection layout for native mpi tasks +block_count = _MPI_LAYOUT_KEYS.block_count # pylint: disable=invalid-name +block_length = _MPI_LAYOUT_KEYS.block_length # pylint: disable=invalid-name +stride = _MPI_LAYOUT_KEYS.stride # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/prolog.py b/examples/dds/pycompss/api/prolog.py new file mode 100644 index 00000000..12d2de4f --- /dev/null +++ b/examples/dds/pycompss/api/prolog.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Prolog decorator. + +This file contains the Prolog class, needed for the task prolog definition +through the decorator. +""" + +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.binary} +SUPPORTED_ARGUMENTS = { + LABELS.binary, + LABELS.args, + LABELS.fail_by_exit_value, +} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class Prolog: # pylint: disable=too-few-public-methods + """Prolog decorator class. + + If defined, will execute the binary before the task execution on the + worker. + Should always be added on top of the 'task' definition. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given prolog arguments. + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", Prolog.__name__.lower())) + # super(Prolog, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Call Prolog simply updates the CE and saves Prolog parameters. + + :param user_function: User function to be decorated. + :return: Decorated dummy user function. + """ + + @wraps(user_function) + def prolog_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + raise NotImplementedError + + if __debug__: + logger.debug("Executing prolog wrapper.") + + if ( + CONTEXT.in_master() or CONTEXT.is_nesting_enabled() + ) and not self.core_element_configured: + self.__configure_core_element__(kwargs) + + with keep_arguments(args, kwargs, prepend_strings=True): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + prolog_f.__doc__ = user_function.__doc__ + return prolog_f + + def __configure_core_element__(self, kwargs: dict) -> None: + """Include the registering info related to @prolog. + + IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. + + :param kwargs: Keyword arguments received from call. + :return: None + """ + if __debug__: + logger.debug("Configuring @prolog core element.") + + # Resolve the fail by exit value + resolve_fail_by_exit_value(self.kwargs) + + binary = self.kwargs[LABELS.binary] + prolog_args = self.kwargs.get(LABELS.args, INTERNAL_LABELS.unassigned) + fail_by = self.kwargs.get(LABELS.fail_by_exit_value) + _prolog = [binary, prolog_args, fail_by] + + core_element = kwargs.get(CORE_ELEMENT_KEY, CE()) + core_element.set_impl_prolog(_prolog) + kwargs[CORE_ELEMENT_KEY] = core_element + # Set as configured + self.core_element_configured = True + + +# ########################################################################### # +# ################### PROLOG DECORATOR ALTERNATIVE NAME ##################### # +# ########################################################################### # + +prolog = Prolog # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/reduction.py b/examples/dds/pycompss/api/reduction.py new file mode 100644 index 00000000..29b2028a --- /dev/null +++ b/examples/dds/pycompss/api/reduction.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Reduction decorator. + +This file contains the Reduction class, needed for the reduction of data +elements task definition. +""" + +import os +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import keep_arguments +from pycompss.api.commons.error_msgs import cast_env_to_int_error +from pycompss.api.commons.error_msgs import cast_string_to_int_error +from pycompss.api.dummy.reduction import reduction as dummy_reduction +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +# Used only for typing - disabled formatting due to multiline. +# fmt: off +from pycompss.runtime.\ + task.definitions.core_element import CE # pylint: disable=unused-import +# fmt: on + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = set() # type: typing.Set[str] +SUPPORTED_ARGUMENTS = {LABELS.chunk_size, LABELS.is_reduce} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + + +class Reduction: # pylint: disable=too-few-public-methods + """Reduction decorator class. + + This decorator also preserves the argspec, but includes the __init__ and + __call__ methods, useful on Reduction task creation. + """ + + __slots__ = [ + "decorator_name", + "args", + "kwargs", + "scope", + "core_element", + "core_element_configured", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + self = itself. + args = not used. + kwargs = dictionary with the given Reduce parameters + + :param args: Arguments + :param kwargs: Keyword arguments + """ + decorator_name = "".join(("@", Reduction.__name__.lower())) + # super(Reduction, self).__init__(decorator_name, *args, **kwargs) + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Optional[CE] + self.core_element_configured = False + if self.scope: + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + # Get the computing nodes + self.__process_reduction_params__() + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Parse and set the reduce parameters within the task core element. + + :param user_function: Function to decorate + :return: Decorated function. + """ + + @wraps(user_function) + def reduce_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + if not self.scope: + d_r = dummy_reduction(self.args, self.kwargs) + return d_r.__call__(user_function)(*args, **kwargs) + + if __debug__: + logger.debug("Executing reduce_f wrapper.") + + # Set the chunk size and is_reduce variables in kwargs for their + # usage in @task decorator + kwargs[LABELS.chunk_size] = self.kwargs[LABELS.chunk_size] + kwargs[LABELS.is_reduce] = self.kwargs[LABELS.is_reduce] + + with keep_arguments(args, kwargs, prepend_strings=False): + # Call the method + ret = user_function(*args, **kwargs) + + return ret + + reduce_f.__doc__ = user_function.__doc__ + return reduce_f + + def __process_reduction_params__(self) -> None: + """Process the chunk size and is_reduce from the decorator. + + :return: None + """ + # Resolve @reduce specific parameters + if LABELS.chunk_size not in self.kwargs: + chunk_size = 0 + else: + chunk_size_kw = self.kwargs[LABELS.chunk_size] + if isinstance(chunk_size_kw, int): + chunk_size = chunk_size_kw + elif isinstance(chunk_size_kw, str): + # Convert string to int + chunk_size = self.__parse_chunk_size__(chunk_size_kw) + else: + raise PyCOMPSsException( + "ERROR: Wrong chunk_size value at @reduction decorator." + ) + + if LABELS.is_reduce not in self.kwargs: + is_reduce = True + else: + is_reduce = self.kwargs[LABELS.is_reduce] + + if __debug__: + logger.debug( + "The task is_reduce flag is set to: %s", str(is_reduce) + ) + logger.debug( + "This Reduction task will have %s sized chunks", + str(chunk_size), + ) + + # Set the chunk_size variable in kwargs for its usage in @task + self.kwargs[LABELS.chunk_size] = chunk_size + self.kwargs[LABELS.is_reduce] = is_reduce + + @staticmethod + def __parse_chunk_size__(chunk_size: str) -> int: + """Parse chunk size as string and returns its value as integer. + + :param chunk_size: Chunk size as string. + :return: Chunk size as integer. + :raises PyCOMPSsException: Can not cast string to int error. + """ + # Check if it is an environment variable to be loaded + if chunk_size.strip().startswith("$"): + # Chunk size is an ENV variable, load it + env_var = chunk_size.strip()[1:] # Remove $ + if env_var.startswith("{"): + env_var = env_var[1:-1] # remove brackets + try: + parsed_chunk_size = int(os.environ[env_var]) + except ValueError as chunk_size_env_var_error: + raise PyCOMPSsException( + cast_env_to_int_error(LABELS.chunk_size) + ) from chunk_size_env_var_error + else: + # ChunkSize is in string form, cast it + try: + parsed_chunk_size = int(chunk_size) + except ValueError as chunk_size_error: + raise PyCOMPSsException( + cast_string_to_int_error(LABELS.chunk_size) + ) from chunk_size_error + return parsed_chunk_size + + +# ########################################################################### # +# ################# REDUCTION DECORATOR ALTERNATIVE NAME #################### # +# ########################################################################### # + +reduction = Reduction # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/software.py b/examples/dds/pycompss/api/software.py new file mode 100644 index 00000000..c10ff090 --- /dev/null +++ b/examples/dds/pycompss/api/software.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Software decorator. + +This file contains the Software class, needed for the software task definition +through the decorator. Software decorator can be used to move the definition of +the multiple decorators to a JSON file. + +WARNING: CAN NOT BE COMPILED WITH MYPY SINCE THE SOFTWARE DECORATOR + INHERITS FROM TASK DECORATOR. +""" +import builtins +import json +import sys +from functools import wraps +from pycompss.runtime.task.master import TaskMaster + +from pycompss.util.context import CONTEXT +from pycompss.api import binary +from pycompss.api import mpi +from pycompss.api import mpmd_mpi +from pycompss.api import multinode +from pycompss.api import http +from pycompss.api import compss +from pycompss.api import task +from pycompss.api import parameter + +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.decorator import resolve_fail_by_exit_value +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing +from pycompss.runtime.task.worker import TaskWorker + + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + +MANDATORY_ARGUMENTS = {LABELS.config_file} +SUPPORTED_ARGUMENTS = {LABELS.config_file} +DEPRECATED_ARGUMENTS = set() # type: typing.Set[str] + +SUPPORTED_DECORATORS = { + LABELS.mpi: (mpi, mpi.mpi), + LABELS.task: (task, task.task), + LABELS.binary: (binary, binary.binary), + LABELS.mpmd_mpi: (mpmd_mpi, mpmd_mpi.mpmd_mpi), + LABELS.http: (http, http.http), + LABELS.compss: (compss, compss.compss), + LABELS.multinode: (multinode, multinode.multinode), +} + + +class Software( + task.task +): # pylint: disable=too-few-public-methods, too-many-instance-attributes + """Software decorator class. + + When provided with a config file, it can replicate any existing python + decorator by wrapping the user function with the decorator defined in + the config file. Arguments of the decorator should be defined in the + config file which is in JSON format. + """ + + __slots__ = [ + "args", + "kwargs", + "config_args", + "decor", + "constraints", + "container", + "prolog", + "epilog", + "parameters", + "file_path", + "is_workflow", + ] + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Initialize the software decorator. + + Only when in the Master, parses the config file and generates + the decorators. Otherwise, just an "__init__". + + :param args: not used (maybe should be?). + :param kwargs: so far contains only the JSON configuration file path. + + """ + super().__init__(*args, **kwargs) + decorator_name = "".join(("@", Software.__name__.lower())) + self.task_type = None # type: typing.Any + self.config_args = None # type: typing.Any + self.decor = None # type: typing.Any + self.constraints = None # type: typing.Any + self.container = None # type: typing.Any + self.prolog = None # type: typing.Any + self.epilog = None # type: typing.Any + self.parameters = {} # type: typing.Dict + self.is_workflow = False # type: bool + self.file_path = None + + self.decorator_name = decorator_name + self.args = args + self.kwargs = kwargs + self.scope = CONTEXT.in_pycompss() + self.core_element = None # type: typing.Any + self.core_element_configured = False + + self.registered_signatures = {} + self.constraint_args = {} + + # no need to parse the config file in the worker. all the @task params + # are passed inside "kwargs" + if self.scope and CONTEXT.in_master(): + if __debug__: + logger.debug("Init @software decorator..") + # Check the arguments + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, + list(kwargs.keys()), + decorator_name, + ) + + def __call__( # pylint: disable=R0915 + self, user_function: typing.Callable + ) -> typing.Callable: + # disable=too-many-statements + """Parse and set the software parameters within the task core element. + + When called, @software decorator basically wraps the user function + into the "real" decorator and passes the args and kwargs. + + :param user_function: User function to be decorated. + :return: User function decorated with the decor type defined by the + user. + """ + + @wraps(user_function) + def software_f( # pylint: disable=R0914,R0911,R0912,R0915 + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + # disable=too-many-locals, too-many-return-statements + # disable=too-many-branches, too-many-statements + if not self.scope: + # Execute the software as with PyCOMPSs so that sequential + # execution performs as parallel. + # To disable: raise Exception(not_in_pycompss(LABELS.binary)) + return user_function(*args, **kwargs) + + self.decorated_function.function = user_function + + updated_args = self.pop_file_path(args) + self.parse_config_file() + + if CONTEXT.in_worker(): + self.decorator_arguments.update_arguments(self.parameters) + worker = TaskWorker( + self.decorator_arguments, self.decorated_function + ) + m_result = worker.call(*updated_args, **kwargs) + # Force flush stdout and stderr + sys.stdout.flush() + sys.stderr.flush() + # Remove worker + del worker + return m_result + + if self.is_workflow: + # no need to do anything, just run the user code + return user_function(*args, **kwargs) + + if __debug__: + logger.debug("Executing software_f wrapper.") + + if self.constraints is not None: + core_element = CE() + core_element.set_impl_constraints(self.constraints) + kwargs[CORE_ELEMENT_KEY] = core_element + + if self.prolog is not None: + resolve_fail_by_exit_value(self.prolog, "True") + prolog_binary = self.prolog[LABELS.binary] + prolog_params = self.prolog.get( + LABELS.args, INTERNAL_LABELS.unassigned + ) + prolog_fail_by = self.prolog.get(LABELS.fail_by_exit_value) + _prolog = [prolog_binary, prolog_params, prolog_fail_by] + + prolog_core_element = kwargs.get(CORE_ELEMENT_KEY, CE()) + prolog_core_element.set_impl_prolog(_prolog) + kwargs[CORE_ELEMENT_KEY] = prolog_core_element + + if self.epilog is not None: + resolve_fail_by_exit_value(self.epilog, "False") + epilog_binary = self.epilog[LABELS.binary] + epilog_params = self.epilog.get( + LABELS.args, INTERNAL_LABELS.unassigned + ) + epilog_fail_by = self.epilog.get(LABELS.fail_by_exit_value) + _epilog = [epilog_binary, epilog_params, epilog_fail_by] + + epilog_core_element = kwargs.get(CORE_ELEMENT_KEY, CE()) + epilog_core_element.set_impl_epilog(_epilog) + kwargs[CORE_ELEMENT_KEY] = epilog_core_element + + if self.container is not None: + _func = str(user_function.__name__) + impl_type = IMPLEMENTATION_TYPES.container + impl_signature = ".".join((impl_type, _func)) + + core_element = kwargs.get(CORE_ELEMENT_KEY, CE()) + impl_args = [ + self.container[LABELS.engine], # engine + self.container[LABELS.image], # image + self.container.get( + LABELS.options, INTERNAL_LABELS.unassigned + ), + INTERNAL_LABELS.unassigned, # internal_type + INTERNAL_LABELS.unassigned, # internal_binary + INTERNAL_LABELS.unassigned, # internal_func + INTERNAL_LABELS.unassigned, # working_dir + INTERNAL_LABELS.unassigned, + ] # fail_by_ev + core_element.set_impl_type(impl_type) + core_element.set_impl_signature(impl_signature) + core_element.set_impl_type_args(impl_args) + kwargs[CORE_ELEMENT_KEY] = core_element + + if CORE_ELEMENT_KEY in kwargs: + self.core_element_configured = True + + if not self.decor: + # It's a PyCOMPSs task with only @task and @software + # decorators, so everything from the config file is already + # in the CE. + return user_function(*args, **kwargs) + + decorator = self.decor + + # if the function is meant to be called on the worker, we must send + # the config file as a FILE_IN param + if decorator in [task.task, multinode.multinode]: + self.decorator_arguments.update_arguments(self.parameters) + kwargs[LABELS.software_config_file] = self.kwargs.get( + LABELS.config_file + ) + + if not self.parameters: + # @task definition is not in the config file, call the user + # function which includes @task + def decor_f(): + def function(): + ret = decorator(**self.config_args) + return ret(user_function)(*args, **kwargs) + + return function() + + decor_f.__doc__ = user_function.__doc__ + return decor_f() + + if self.decor is not task.task: + # it is not a regular @task, build the decorator with + # "execution" key-values and the @task decorator with + # "parameters" key-values + def decor_f_nonregular(): + def function(*_, **__): + t_t = task.task(**self.parameters) + return t_t(user_function)(*_, **__) + + dec = decorator(**self.config_args) + return dec(function)(*args, **kwargs) + + return decor_f_nonregular() + + # regular task definition inside a config file + self.__check_core_element__(kwargs, user_function) + master = TaskMaster( + user_function, + self.core_element, + self.decorator_arguments, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) + ( + future_object, + self.core_element, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) = master.call(args, kwargs) + + del master + return future_object + + software_f.__doc__ = user_function.__doc__ + return software_f + + def pop_file_path(self, *args) -> typing.Tuple: + """Pop JSON configuration file path from the args. + + :param args: args of the task function + :return: args without JSON config file path + """ + if CONTEXT.in_master(): + if not self.file_path: + self.file_path = self.kwargs.get(LABELS.config_file) + return args + if CONTEXT.in_worker(): + tmp = list(*args) # type: typing.List[typing.Any] + for i, value in enumerate(tmp): + if value.name == "#kwarg_software_config_file": + self.file_path = value.file_name.source_path + break + else: + return args + tmp.pop(i) + return tuple(tmp) + raise PyCOMPSsException("Unexpected context!") + + def parse_config_file(self) -> None: + """ + Parse the config file and set self's task_type, decor, and config args. + + :return: None + """ + if not self.file_path: + raise PyCOMPSsException(" ERROR: Incorrect file_path.") + with open( # pylint: disable=unspecified-encoding + self.file_path, "r" + ) as file_path_descriptor: + config = json.load(file_path_descriptor) + + execution = config.get(LABELS.execution, {}) + self.parameters = config.get(LABELS.parameters, {}) + exec_type = execution.pop(LABELS.type, None) + if exec_type is None: + print("WARN: Execution type not provided for @software task") + elif exec_type == LABELS.task: + self.task_type, self.decor = SUPPORTED_DECORATORS[exec_type] + if __debug__: + logger.debug("Executing Software as task function ...") + elif exec_type == "workflow": + if __debug__: + logger.debug("Executing Software as Workflow ...") + self.is_workflow = True + return + elif exec_type.lower() not in SUPPORTED_DECORATORS: + msg = ( + f"Error: Executor Type {exec_type} is not supported " + f"for software task." + ) + raise PyCOMPSsException(msg) + else: + exec_type = exec_type.lower() + self.task_type, self.decor = SUPPORTED_DECORATORS[exec_type] + mand_args = self.task_type.MANDATORY_ARGUMENTS + if not all(arg in execution for arg in mand_args): + msg = f"Error: Missing arguments for '{self.task_type}'." + raise PyCOMPSsException(msg) + + self.replace_param_types() + + # send the config file to the worker as well + if CONTEXT.in_master() and self.decor in [ + task.task, + multinode.multinode, + ]: + self.parameters[LABELS.software_config_file] = parameter.FILE_IN + + self.config_args = execution + self.constraints = config.get("constraints", None) + self.container = config.get("container", None) + self.prolog = config.get("prolog", None) + self.epilog = config.get("epilog", None) + + def replace_param_types(self): + """Replace the strings with Parameters form the API. + + :return: + """ + # replace python param types if any + for key, value in self.parameters.items(): + if isinstance(value, str) and hasattr(parameter, value): + self.parameters[key] = getattr(parameter, value) + + # convert the "returns" value + rets = self.parameters.get("returns", None) + if rets and isinstance(rets, str) and hasattr(builtins, rets): + self.parameters["returns"] = getattr(builtins, rets) + + +# ########################################################################### # +# ##################### Software DECORATOR ALTERNATIVE NAME ################# # +# ########################################################################### # + + +software = Software # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/api/task.py b/examples/dds/pycompss/api/task.py new file mode 100644 index 00000000..19775486 --- /dev/null +++ b/examples/dds/pycompss/api/task.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - Task decorator. + +This file contains the Task class, needed for the task definition. +""" + +from __future__ import print_function + +import inspect +import os +import sys +from functools import wraps + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.api.dummy.task import task as dummy_task +from pycompss.runtime.start.initialization import LAUNCH_STATUS +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.runtime.task.definitions.arguments import TaskArguments +from pycompss.runtime.task.definitions.function import FunctionDefinition +from pycompss.runtime.task.definitions.constraints import ConstraintDescription +from pycompss.runtime.task.master import TaskMaster +from pycompss.runtime.task.worker import TaskWorker +from pycompss.util.logger.helpers import init_logging_worker +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.logger.level import LOG_LEVEL +from pycompss.util.objects.properties import get_module_name +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.helpers import EventMaster +from pycompss.util.tracing.helpers import TRACING +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing + + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + + +class Task: # pylint: disable=R0902, R0903 + # disable=too-many-instance-attributes, too-few-public-methods, + """This is the Task decorator implementation. + + It is implemented as a class and consequently this implementation can be + divided into two natural steps: decoration process and function call. + + Decoration process is what happens when the Python parser reads a decorated + function. The actual function is not called, but the @task() triggers + the process that stores and processes the parameters of the decorator. + This first step corresponds to the class constructor. + + Function call is what happens when the user calls their function somewhere + in the code. A decorator simply adds pre and post steps in this function + call, allowing us to change and process the arguments. This second steps + happens in the __call__ implementation. + + Also, the call itself does different things in the master than in the + worker. We must also handle the case when the user just runs the app with + python and no PyCOMPSs. + The specific implementations can be found in TaskMaster.call(), + TaskWorker.call() and self._sequential_call() + """ + + __slots__ = [ + "task_type", + "decorator_name", + "scope", + "core_element", + "core_element_configured", + "decorator_arguments", + "decorated_function", + "constraint_args", + "registered_signatures", + "constraint_args", + ] + + def __init__( + self, *args: typing.Any, **kwargs: typing.Any # pylint: disable=W0613 + ) -> None: + # disable = unused - argument + """Task constructor. + + This part is called in the decoration process, not as an + explicit function call. + + We do two things here: + a) Assign default values to unspecified fields. + b) Transform the parameters from user friendly types + (i.e Parameter.IN, etc) to a more convenient internal + representation. + + :param args: Decorator positional parameters (ignored). + :param kwargs: Decorator parameters. A task decorator has no positional + arguments. + """ + self.task_type = IMPLEMENTATION_TYPES.method + self.decorator_name = "".join(("@", Task.__name__.lower())) + self.scope = CONTEXT.in_pycompss() + # Instantiate Core Element + self.core_element = CE() + self.core_element_configured = False + # Instantiate TaskArguments object from kwargs + task_decorator_arguments = TaskArguments() + task_decorator_arguments.update_arguments(kwargs) + self.decorator_arguments = task_decorator_arguments + # Instantiate FunctionDefinition object + self.decorated_function = FunctionDefinition() + self.registered_signatures = ( + {} + ) # type: typing.Dict[str, typing.Dict[str, typing.List[str]]] + self.constraint_args = ( + {} + ) # type: typing.Dict[str, ConstraintDescription] + + def __call__(self, user_function: typing.Callable) -> typing.Callable: + """Perform the task processing. + + This function is called in all explicit function calls. + Note that in PyCOMPSs a single function call will be transformed into + two calls, as both master and worker need to call the function. + + The work to do in the master part is totally different + from the job to do in the worker part. This is why there are + some other functions like master_call, worker_call, and + _sequential_call. + + There is also a third case that happens when the user runs a PyCOMPSs + code without PyCOMPSs. This case is straightforward: just call the + user function with the user parameters and return whatever the user + code returned. Therefore, we can just return the user function. + + :param user_function: Function to decorate. + :return: The function to be executed. + """ + self.decorated_function.function = user_function + + @wraps(user_function) + def task_decorator( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + return self.__decorator_body__(user_function, args, kwargs) + + return task_decorator + + def __decorator_body__( + self, user_function: typing.Callable, args: tuple, kwargs: dict + ) -> typing.Any: + """Body of the task decorator. + + :param user_function: Decorated function. + :param args: Function arguments. + :param kwargs: Function keyword arguments. + :returns: Result of executing the user_function with the given args + and kwargs. + """ + # Determine the context and decide what to do + if CONTEXT.in_master(): + # @task being executed in the master + # Each task will have a TaskMaster, so its content will + # not be shared. + self.__check_core_element__(kwargs, user_function) + with EventMaster(TRACING_MASTER.task_instantiation): + master = TaskMaster( + user_function, + self.core_element, + self.decorator_arguments, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) + master_result = master.call(args, kwargs) + ( + future_object, + self.core_element, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) = master_result + del master + return future_object + if CONTEXT.in_worker(): + if "compss_key" in kwargs.keys(): + # Is nesting enabled is set in piper common utils + is_nesting_enabled = CONTEXT.is_nesting_enabled() + if is_nesting_enabled: + if __debug__: + # Update the whole logger since it will be in the + # job out/err files + init_logging_worker( + LOG_REMITTENT.WORKER, + LOG_LEVEL.DEBUG, + TRACING.is_tracing(), + kwargs["compss_log_files"][0], + kwargs["compss_log_files"][1], + ) + # @task being executed in the worker + with EventInsideWorker( + TRACING_WORKER.worker_task_instantiation + ): + worker = TaskWorker( + self.decorator_arguments, + self.decorated_function, + ) + worker_result = worker.call(*args, **kwargs) + # Force flush stdout and stderr + sys.stdout.flush() + sys.stderr.flush() + # Remove worker + del worker + if is_nesting_enabled: + if __debug__: + # Reestablish logger handlers + init_logging_worker( + LOG_REMITTENT.WORKER, + LOG_LEVEL.DEBUG, + TRACING.is_tracing(), + ) + return worker_result + + # There is no compss_key in kwargs.keys() => task invocation + # within task: + # - submit the task to the runtime if nesting is enabled. + # - execute sequentially if nested is not enabled. + if CONTEXT.is_nesting_enabled(): + # Each task will have a TaskMaster, so its content will + # not be shared. + with EventMaster(TRACING_MASTER.task_instantiation): + master = TaskMaster( + user_function, + self.core_element, + self.decorator_arguments, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) + master_result = master.call(args, kwargs) + ( + future_object, + self.core_element, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) = master_result + del master + return future_object + # Called from another task within the worker + # Ignore the @task decorator and run it sequentially + message = ( + f"WARNING: Calling task: {str(user_function.__name__)} from " + f"this task.\n" + f" It will be executed sequentially within the caller " + f"task." + ) + print(message, file=sys.stderr) + return self._sequential_call(*args, **kwargs) + # We are neither in master nor in the worker, or the user has + # stopped the interactive session. + # Therefore, the user code is being executed with no + # launch_compss/enqueue_compss/runcompss/interactive session + return self._sequential_call(*args, **kwargs) + + def _sequential_call( + self, *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + """Sequential task execution. + + The easiest case: just call the user function and return whatever it + returns. + + :return: The user function return. + """ + # Inspect the user function, get information about the arguments and + # their names + # This defines self.param_args, self.param_varargs, + # self.param_kwargs and self.param_defaults + # And gives non-None default values to them if necessary + d_t = dummy_task(args, kwargs) + return d_t(self.decorated_function.function)(*args, **kwargs) + + def __check_core_element__( + self, kwargs: dict, user_function: typing.Callable + ) -> None: + """Check Core Element for containers. + + :param kwargs: Keyword arguments. + :param user_function: User function. + :return: None (updates the Core Element of the given kwargs). + """ + if ( + CORE_ELEMENT_KEY in kwargs + and kwargs[CORE_ELEMENT_KEY].get_impl_type() + == IMPLEMENTATION_TYPES.container + ): + # The task is using a container + impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args() + _type = impl_args[3] + if _type == INTERNAL_LABELS.unassigned: + # The task is not invoking a binary + _engine = impl_args[0] + _image = impl_args[1] + _options = impl_args[2] + _type = "CET_PYTHON" + _module_name = str(self.__get_module_name__(user_function)) + _function_name = str(user_function.__name__) + _func_complete = f"{_module_name}&{_function_name}" + impl_args = [ + _engine, # engine + _image, # image + _options, # container options + _type, # internal_type + INTERNAL_LABELS.unassigned, # internal_binary + INTERNAL_LABELS.unassigned, # internal_parameters + _func_complete, # internal_func + INTERNAL_LABELS.unassigned, # working_dir + INTERNAL_LABELS.unassigned, + ] # fail_by_ev + kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) + + @staticmethod + def __get_module_name__(user_function: typing.Callable) -> str: + """Get the module name from the user function. + + :param user_function: User function. + :returns: The module name where the user function is defined. + """ + mod = inspect.getmodule(user_function) # type: typing.Any + module_name = mod.__name__ + if module_name == "__main__": + # The module where the function is defined was run as __main__, + # We need to find out the real module name + # Get the real module name from our launch.py APP_PATH global + # variable + # It is guaranteed that this variable will always exist because + # this code is only executed when we know we are in the master + path = LAUNCH_STATUS.get_app_path() + # Get the file name + file_name = os.path.splitext(os.path.basename(path))[0] + # Get the module + module_name = get_module_name(path, file_name) + return module_name + + +# task can be also typed as Task +task = Task # pylint: disable=invalid-name diff --git a/examples/dds/pycompss/dds/README.md b/examples/dds/pycompss/dds/README.md new file mode 100755 index 00000000..3c866469 --- /dev/null +++ b/examples/dds/pycompss/dds/README.md @@ -0,0 +1,38 @@ +# PyCOMPSs Distributed Data Set (DDS) + +DDS is a lightweight library for [PyCOMPSs](https://pypi.org/project/pycompss/) +developers which contains some basic and widely used data processing methods +such as map, filter, reduce, etc. The main purpose of this library is to avoid +implementations of simple 'task' functions by developers. DDS is trustful and it +processes the data in the most adequate way in terms of parallelism. + + +### How To Use +Just clone this repo to your local, import it, and enjoy! + + +### The Most Useful Methods +``` +map +filter +reduce +count +max +min +sum +foreach +map_values +combine_by_key +reduce_by_key +collect +collect_as_dict +``` + +Please feel free to suggest more methods! + + +### Examples +Please see 'examples' and 'doctests' from 'dds.py'. + + +Copyright 2018 [Barcelona Supercomputing Center](www.bsc.es) diff --git a/examples/dds/pycompss/dds/__init__.py b/examples/dds/pycompss/dds/__init__.py new file mode 100644 index 00000000..a14b2292 --- /dev/null +++ b/examples/dds/pycompss/dds/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the DDS functions, constants and classes.""" + +from pycompss.dds.dds import DDS # noqa: F401 diff --git a/examples/dds/pycompss/dds/core/__init__.py b/examples/dds/pycompss/dds/core/__init__.py new file mode 100644 index 00000000..7b740420 --- /dev/null +++ b/examples/dds/pycompss/dds/core/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the DDS core.""" diff --git a/examples/dds/pycompss/dds/core/partition_generators.py b/examples/dds/pycompss/dds/core/partition_generators.py new file mode 100644 index 00000000..0c13716f --- /dev/null +++ b/examples/dds/pycompss/dds/core/partition_generators.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Partition generators. + +Partitions should be loaded with data in the last step only. Thus, generator +objects are necessary. Inside 'task' functions we will call their 'generate' +method in order to retrieve the partition. These partitions can previously be +loaded on master and sent to workers, or read from files on worker nodes. +""" + +import pickle +import sys +from pycompss.util.exceptions import DDSException +from dataclay import DataClayObject, activemethod +from collections.abc import Iterable +import logging +logger = logging.getLogger(__name__) + + +class IPartitionGenerator(DataClayObject): # pylint: disable=too-few-public-methods + """Everyone implements this.""" + + @activemethod + def retrieve_data(self): + """Retrieve data. + + :raises NotImplementedError: Not implemented function. + """ + raise NotImplementedError + + +class BasicDataLoader(IPartitionGenerator): # pylint: disable=R0903 + """Basic data loader.""" + + @activemethod + def __init__(self, data): + """Create a new BasicDataLoader object. + + :param data: Data. + :returns: None. + """ + super().__init__() + self.data = data + + @activemethod + def retrieve_data(self): + """Retrieve data. + + :returns: Data. + """ + ret = [] + if isinstance(self.data, list): + ret.extend(self.data) + else: + ret.append(self.data) + return ret + + +class IteratorLoader(IPartitionGenerator): # pylint: disable=R0903 + """Iterator Loader.""" + iterable: Iterable + start: int + end: int + + @activemethod + def __init__(self, iterable, start, end): + """Create new IteratorLoader object. + + :param iterable: Iterable object. + :param start: Start position. + :param end: End Position. + :returns: None. + """ + logger.info("ITERATOR LOADER YUHUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU") + print("MAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHA") + super().__init__() + self.iterable = iterable + print("ITERAAAAALBE????, %s" , self.get_iterable()) + self.start = start + self.end = end + + @activemethod + def get_iterable(self): + print("PERO AQUI ARRIBA????????????") + return self.iterable + + @activemethod + def retrieve_data(self): + """Divide and retrieve the next partition. + + :returns: Data. + """ + ret = [] + # If it's a dict + if isinstance(self.iterable, dict): + sorted_keys = sorted(self.iterable.keys()) + for key in sorted_keys[self.start : self.end]: # noqa: E203 + ret.append((key, self.iterable[key])) + elif isinstance(self.iterable, list): + for item in iter( + self.iterable[self.start : self.end] # noqa: E203 + ): + ret.append(item) + else: + index = 0 + for item in iter(self.iterable): + index += 1 + if index > self.end: + break + if index > self.start: + ret.append(item) + return ret + + +class WorkerFileLoader(IPartitionGenerator): # pylint: disable=R0903 + """Worker file loader.""" + + @activemethod + def __init__( + self, file_paths, single_file=False, start=0, chunk_size=None + ): + """Create new WorkerFileLoader object. + + :param file_paths: List of file paths. + :param single_file: Is a single file? + :param start: Start position. + :param chunk_size: Chunk size. + :returns: None. + """ + super().__init__() + self.file_paths = file_paths + self.single_file = single_file + self.start = start + self.chunk_size = chunk_size + + if self.single_file and not chunk_size: + raise DDSException("Missing chunk_size argument...") + + @activemethod + def retrieve_data(self): + """Retrieve data. + + :returns: Data. + """ + if self.single_file: + file_path = self.file_paths[0] + with open(file_path) as file_paths_fd: # pylint: disable=W1514 + file_paths_fd.seek(self.start) + temp = file_paths_fd.read(self.chunk_size) + return [temp] + + ret = [] + for file_path in self.file_paths: + with open(file_path) as file_path_fd: # pylint: disable=W1514 + content = file_path_fd.read() + ret.append((file_path, content)) + + return ret + + +class PickleLoader(IPartitionGenerator): # pylint: disable=R0903 + """Pickle loader.""" + + @activemethod + def __init__(self, pickle_path): + """Create new WorkerFileLoader object. + + :param pickle_path: Pickled file path. + :returns: None. + """ + super().__init__() + self.pickle_path = pickle_path + + @activemethod + def retrieve_data(self): + """Retrieve data. + + :returns: Data. + """ + with open(self.pickle_path, "rb") as pickle_path_fd: + ret = pickle.load(pickle_path_fd) + return ret + + +def read_in_chunks(file_name, chunk_size=1024, strip=True): + """Lazy function (generator) to read a file piece by piece. + + :param file_name: File name to read. + :param chunk_size: Chunk size (Default: 1k). + :param strip: If it requires stripping. + :returns: Next partition. + """ + partition = [] + with open(file_name) as file_name_fd: # pylint: disable=W1514 + collected = 0 + for line in file_name_fd: + _line = line.rstrip("\n") if strip else line + partition.append(_line) + collected += sys.getsizeof(_line) + if collected > chunk_size: + yield partition + partition = [] + collected = 0 + + if partition: + yield partition + + +def read_lines(file_name, num_of_lines=1024, strip=True): + """Lazy function (generator) to read a file line by line. + + :param file_name: File to read. + :param num_of_lines: Total number of lines in each partition. + :param strip: If line separators should be stripped from lines. + :returns: Next partition. + """ + partition = [] + with open(file_name) as file_name_fd: # pylint: disable=W1514 + collected = 0 + for line in file_name_fd: + _line = line.rstrip("\n") if strip else line + partition.append(_line) + collected += 1 + if collected > num_of_lines: + yield partition + partition = [] + collected = 0 + + if partition: + yield partition diff --git a/examples/dds/pycompss/dds/core/tasks.py b/examples/dds/pycompss/dds/core/tasks.py new file mode 100644 index 00000000..b2d6da27 --- /dev/null +++ b/examples/dds/pycompss/dds/core/tasks.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs DDS - Tasks.""" + +import os +import pickle + +from pycompss.api.parameter import INOUT +from pycompss.api.parameter import IN +from pycompss.api.parameter import COLLECTION_OUT +from pycompss.api.parameter import COLLECTION_IN +from pycompss.api.parameter import Type +from pycompss.api.parameter import Depth +from pycompss.api.task import task +from pycompss.dds.core.partition_generators import IPartitionGenerator + +from dataclay import activemethod + +MARKER = "COMPSS_DEFAULT_VALUE_TO_BE_USED_AS_A_MARKER" +FILE_NAME_LENGTH = 5 + + + + +@task(returns=1, collection={Type: COLLECTION_IN, Depth: 1}) +@activemethod +def map_partition(func, partition, collection=None): + """Map the given function to the partition. + + :param func: A function that return only one argument which is iterable. + :param partition: The partition itself or a partition generator object. + :param collection: Partition when partition is a collection. + :return: The transformed partition. + """ + partition = partition or collection or [] + print("////////////////// ", partition.get_iterable()) + if isinstance(partition, IPartitionGenerator): + partition = partition.retrieve_data() + + res = func(partition) + del partition + return res + + +@task( + col={Type: COLLECTION_OUT, Depth: 1}, + collection={Type: COLLECTION_IN, Depth: 1}, +) +def distribute_partition( + col, func, partitioner_func, partition, collection=None +): + """Distribute (k, v) structured elements of the partition on 'buckets'. + + :param col: Empty 'buckets', must be repleced with COLLECTION_OUT. + :param func: Function from DDS object to be applied to the partition before + the distribution. + :param partitioner_func: A function to find element's corresponding bucket. + :param partition: The partition itself or a partition generator object. + :param collection: If the partition is a collection of future objects. + :return: Fill the empty 'buckets' with the elements of the partition. + """ + partition = partition or collection or [] + + if isinstance(partition, IPartitionGenerator): + partition = partition.retrieve_data() + + partition = func(partition) if func else partition + + nop = len(col) + for key, value in partition: + col[partitioner_func(key) % nop].append((key, value)) + + +@task(first=INOUT, rest={Type: COLLECTION_IN, Depth: 1}) +def reduce_dicts(first, rest): + """Reduce dictionaries. + + CAUTION! Modifies first dictionary. + + :param first: First dictionary. + :param rest: Second dictionary. + :return: None. + """ + dicts = iter(rest) + + for _dict in dicts: + for k in _dict: + first[k] += _dict[k] + + +@task(returns=list, iterator=IN) +def task_dict_to_list(iterator, total_parts, partition_num): + """Convert dictionary to (key, value) pairs. + + :param iterator: Iterator object. + :param total_parts: Total parts. + :param partition_num: Number of partitions. + :return: List of (key, value) pairs + """ + ret = [] + sorted_keys = sorted(iterator.keys()) + total = len(sorted_keys) + chunk_size = max(1, total / total_parts) + start = chunk_size * partition_num + is_last = total_parts == partition_num + 1 + + if is_last: + for i in sorted_keys[start:]: + ret.append((i, iterator[i])) + else: + for i in sorted_keys[start : start + chunk_size]: # noqa: E203 + ret.append((i, iterator[i])) + + return ret + + +@task(returns=1, parts={Type: COLLECTION_IN, Depth: 1}) +def reduce_multiple(function, parts): + """Reduce multiple. + + :param function: Reducing function. + :param parts: List of elements. + :returns: Reduction result. + """ + partitions = iter(parts) + try: + res = next(partitions)[0] + except StopIteration: + return None + + for part in partitions: + if part: + res = function(res, part[0]) + + return [res] + + +@task(returns=list) +def task_collect_samples(partition, num_of_samples, key_func): + """Collect samples. + + :param partition: List of elements. + :param num_of_samples: Number of samples. + :param key_func: Key function. + :return: Collected samples. + """ + ret = [] + total = len(partition) + step = max(total // num_of_samples, 1) + for _i in range(0, total, step): + ret.append(key_func(partition[_i][0])) + + return ret + + +@task(collection={Type: COLLECTION_IN, Depth: 1}) +def map_and_save_text_file(func, index, path, partition, collection=None): + """Map and save text file. + + Same as 'map_partition' function with the only difference that this one + saves the result as a text file. + + :param func: Function to apply. + :param index: Important to keep the order of the partitions. + :param path: Directory to save the partition. + :param partition: Partition. + :param collection: Is collection? + :return: No return value skips the serialization phase. + """ + partition = partition or collection or [] + + if isinstance(partition, IPartitionGenerator): + partition = partition.retrieve_data() + + partition = func(partition) if func else partition + + file_name = os.path.join(path, str(index).zfill(FILE_NAME_LENGTH)) + with open(file_name, "w") as _: # pylint: disable=W1514 + for item in partition: + _.write("\n".join([str(item)])) + + +@task(collection={Type: COLLECTION_IN, Depth: 1}) +def map_and_save_pickle(func, index, path, partition, collection=None): + """Map and save pickled file. + + Same as 'map_partition' function with the only difference that this one + saves the result as a pickle file. + + :param func: Function to apply. + :param index: Important to keep the order of the partitions. + :param path: Directory to save the partition. + :param partition: Partition. + :param collection: Is collection? + :return: None. + """ + partition = partition or collection or [] + + if isinstance(partition, IPartitionGenerator): + partition = partition.retrieve_data() + + partition = func(partition) if func else partition + + file_name = os.path.join(path, str(index).zfill(FILE_NAME_LENGTH)) + with open(file_name, "wb") as file_name_fd: + pickle.dump(list(partition), file_name_fd) diff --git a/examples/dds/pycompss/dds/core/utils.py b/examples/dds/pycompss/dds/core/utils.py new file mode 100644 index 00000000..3e419223 --- /dev/null +++ b/examples/dds/pycompss/dds/core/utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Utils. + +This file contains the DDS interface utilities. +""" + + +def default_hash(obj): + """Get the hash of the given object. + + :param obj: Object to calculate the hash. + :return: Hash value. + """ + return hash(obj) diff --git a/examples/dds/pycompss/dds/dds.py b/examples/dds/pycompss/dds/dds.py new file mode 100644 index 00000000..ee7bd055 --- /dev/null +++ b/examples/dds/pycompss/dds/dds.py @@ -0,0 +1,1037 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - API. + +This file contains the DDS interface. +""" + +import heapq +import bisect +import itertools +import functools +import os +from collections import defaultdict +from collections import deque + +from pycompss.api.api import compss_wait_on +from pycompss.api.api import compss_delete_object +from pycompss.api.api import compss_barrier +from pycompss.dds.core.partition_generators import IPartitionGenerator +from pycompss.dds.core.partition_generators import BasicDataLoader +from pycompss.dds.core.partition_generators import IteratorLoader +from pycompss.dds.core.partition_generators import WorkerFileLoader +from pycompss.dds.core.partition_generators import PickleLoader +from pycompss.dds.core.partition_generators import read_in_chunks +from pycompss.dds.core.partition_generators import read_lines +from pycompss.dds.core.tasks import map_partition +from pycompss.dds.core.tasks import distribute_partition +from pycompss.dds.core.tasks import reduce_dicts +from pycompss.dds.core.tasks import task_dict_to_list +from pycompss.dds.core.tasks import reduce_multiple +from pycompss.dds.core.tasks import task_collect_samples +from pycompss.dds.core.tasks import map_and_save_text_file +from pycompss.dds.core.tasks import map_and_save_pickle +from pycompss.dds.core.tasks import MARKER +from pycompss.dds.core.utils import default_hash +from pycompss.util.tracing.helpers import EventMaster + + +class DDS: # pylint: disable=too-many-public-methods + """Distributed Data Set object.""" + + def __init__(self): + """Create a new DDS object.""" + super().__init__() + self.partitions = [] + self.func = None + + # Partition As A Collection + # True if partitions are not Future Objects but list of Future Objects + self.paac = False + + def load(self, iterator, num_of_parts=10, paac=False, persistent=False): + """Load and distribute the iterator on partitions. + + :param iterator: Partitions iterator. + :param num_of_parts: Number of parts. + :param paac: Partition as a collection. + :returns: Self. + """ + self.paac = paac + if num_of_parts == -1: + self.partitions = iterator + return self + + total = len(iterator) + if not total: + return self + + chunk_sizes = [(total // num_of_parts)] * num_of_parts + extras = total % num_of_parts + for i in range(extras): + chunk_sizes[i] += 1 + + start = 0 + for size in chunk_sizes: + end = start + size + print("~~~~~~~~~~~~MAKE EPRSISTWENT~~~~~~~~~~~~~~~~~~~~") + _partition_loader = IteratorLoader(iterator, start, end) + if(persistent):_partition_loader.make_persistent() + print("¬¬¬¬¬¬¬¬¬¬¬¬¬%s¬¬¬¬¬¬¬¬¬¬¬¬", _partition_loader.is_persistent) + print("¬¬¬¬¬¬¬¬¬¬¬¬¬%s¬¬¬¬¬¬¬¬¬¬¬¬", _partition_loader.get_iterable()) + self.partitions.append(_partition_loader) + start = end + + return self + + def load_file(self, file_path, chunk_size=1024, worker_read=False, persistent=False): + """Read file in chunks and save it onto partitions. + + Usage sample: + >>> with open("test.file", "w") as testFile: + ... _ = testFile.write("Hello world!") + >>> DDS().load_file("test.file", 6).collect() + ['Hello ', 'world!'] + + :param file_path: A path to a file to be loaded. + :param chunk_size: Size of chunks in bytes. + :param worker_read: If reading the file in the worker + (skips first bytes). + :return: Self. + """ + if worker_read: + with open(file_path) as file_path_fd: # pylint: disable=W1514 + file_path_fd.seek(0, 2) + total = file_path_fd.tell() + parsed = 0 + while parsed < total: + _partition_loader = WorkerFileLoader( + [file_path], + single_file=True, + start=parsed, + chunk_size=chunk_size, + ) + if(persistent):_partition_loader.make_persistent() + self.partitions.append(_partition_loader) + parsed += chunk_size + else: + with open(file_path, "r") as file_path_fd: # pylint: disable=W1514 + chunk = file_path_fd.read(chunk_size) + while chunk: + _partition_loader = BasicDataLoader(chunk) + _partition_loader.make_persistent() + self.partitions.append(_partition_loader) + chunk = file_path_fd.read(chunk_size) + + return self + + def load_text_file( + self, file_name, chunk_size=1024, in_bytes=True, strip=True, persistent=False + ): + r"""Load a text file into partitions with 'chunk_size' lines on each. + + Usage sample: + >>> with open("test.txt", "w") as testFile: + ... _ = testFile.write("First Line! \n") + ... _ = testFile.write("Second Line! \n") + >>> DDS().load_text_file("test.txt").collect() + ['First Line! ', 'Second Line! '] + + :param file_name: A path to a file to be loaded. + :param chunk_size: Size of chunks in bytes. + :param in_bytes: If chunk size is in bytes or in number of lines. + :param strip: If line separators should be stripped from lines. + :return: Self. + """ + func = read_in_chunks if in_bytes else read_lines + + for _p in func(file_name, chunk_size, strip=strip): + partition_loader = BasicDataLoader(_p) + if(persistent):partition_loader.make_persistent() + self.partitions.append(partition_loader) + + return self + + def load_files_from_dir(self, dir_path, num_of_parts=-1, persistent=False): + """Read multiple files from a given directory. + + Each file and its content is saved in a tuple in ('file_path', + 'file_content') format. + + :param dir_path: A directory that all files will be loaded from. + :param num_of_parts: Can be set to -1 to create one partition per file. + :return: Self. + """ + files = sorted(os.listdir(dir_path)) + total = len(files) + num_of_parts = total if num_of_parts < 0 else num_of_parts + partition_sizes = [(total // num_of_parts)] * num_of_parts + extras = total % num_of_parts + for i in range(extras): + partition_sizes[i] += 1 + + start = 0 + for size in partition_sizes: + end = start + size + partition_files = [] + for file_name in files[start:end]: + file_path = os.path.join(dir_path, file_name,persistent=persistent) + partition_files.append(file_path) + + _partition_loader = WorkerFileLoader(partition_files) + if(persistent):_partition_loader.make_persistent() + self.partitions.append(_partition_loader) + start = end + + return self + + def load_pickle_files(self, dir_path, persistent=False): + """Load serialized partitions from pickle files. + + :param dir_path: Path to serialized partitions. + :return: Self. + """ + files = sorted(os.listdir(dir_path)) + for _f in files: + file_name = os.path.join(dir_path, _f,persistent=persistent) + _partition_loader = PickleLoader(file_name) + if(persistent):_partition_loader.make_persistent() + self.partitions.append(_partition_loader) + + return self + + def union(self,persistent=False, *args,): + """Combine this data set with some other DDS data. + + Usage sample: + >>> first = DDS().load([0, 1, 2, 3, 4], 2) + >>> second = DDS().load([5, 6, 7, 8, 9], 3) + >>> first.union(second).count() + 10 + + :param args: Arbitrary amount of DDS objects. + :return: New DDS object combining two DDS objects. + """ + current = list(self.collect(future_objects=True)) + for dds in args: + temp = list(dds.collect(future_objects=True)) + current.extend(temp) + + return DDS().load(current, num_of_parts=-1, persistent=persistent) + + def num_of_partitions(self): + """Get the total amount of partitions. + + Usage sample: + >>> DDS().load(range(10), 5).num_of_partitions() + 5 + + :return: Number of partitions. + """ + return len(self.partitions) + + def map(self, func, *args, **kwargs): + """Apply the given function to each element of the dataset. + + Usage sample: + >>> dds = DDS().load(range(10), 5).map(lambda x: x * 2) + >>> sorted(dds.collect()) + [0, 2, 4, 6, 8, 10, 12, 14, 16, 18] + + :param func: Function to apply. + :param args: Arguments. + :param kwargs: Keyword arguments. + :returns: New child DDS object. + """ + + def mapper(partition): + results = [] + for element in partition: + results.append(func(element, *args, **kwargs)) + return results + + return _ChildDDS(self, mapper) + + def map_partitions(self, func): + """Apply a function to each partition of this data set. + + Usage sample: + >>> DDS().load(range(10), 5).map_partitions( + ... lambda x: [sum(x)] + ... ).collect(True) + [[1], [5], [9], [13], [17]] + + :param func: Function to apply. + :returns: New child DDS object. + """ + return _ChildDDS(self, func) + + def flat_map(self, func, *args, **kwargs): + """Apply a function to each element of the dataset. + + NOTE: Extends the derived element(s) if possible. + + Usage sample: + >>> dds = DDS().load([2, 3, 4]) + >>> sorted(dds.flat_map(lambda x: range(1, x)).collect()) + [1, 1, 1, 2, 2, 3] + + :param func: A function that should return a list, tuple or + another kind of iterable. + :param args: Arguments. + :param kwargs: Keyword arguments. + :returns: New child DDS object. + """ + + def mapper(iterator): + res = [] + for item in iterator: + res.extend(func(item, *args, **kwargs)) + return res + + return self.map_partitions(mapper) + + def filter(self, func): + """Filter elements of this data set by applying a given function. + + Usage sample: + >>> DDS().load(range(10), 5).filter(lambda x: x % 2).count() + 5 + + :param func: Filtering function. + :returns: New child DDS object filtered. + """ + + def _filter(iterator): + return filter(func, iterator) + + return self.map_partitions(_filter) + + def reduce(self, func, initial=MARKER, arity=-1): + """Reduce the whole data set. + + Usage sample: + >>> DDS().load(range(10), 5).reduce((lambda b, a: b + a) , 100) + 145 + + :param func: A reduce function which should take two parameters as + inputs and return a single result which will be sent to + itself again. + :param initial: Initial value for reducer which will be used to reduce + the first element with. + :param arity: Tree depth. + :return: Reduced result (inside a DDS if necessary). + """ + + def local_reducer(partition): + """Reduce a partition and retrieve it as a one element partition. + + :param partition: Partition. + :return: One element partition. + """ + iterator = iter(partition) + try: + init = next(iterator) + except StopIteration: + return [] + + return [functools.reduce(func, iterator, init)] + + local_results = self.map_partitions(local_reducer).collect( + future_objects=True + ) + + local_results = deque(local_results) + + # If initial value is set, add it to the list as well + if initial != MARKER: + local_results.append([initial]) + + arity = arity if arity > 0 else len(self.partitions) + branch = [] + + while local_results: + while local_results and len(branch) < arity: + temp = local_results.popleft() + branch.append(temp) + + if len(branch) == 1: + branch = compss_wait_on(branch[0]) + break + + temp = reduce_multiple(func, branch) + local_results.append(temp) + branch = [] + + return branch[0] + + def distinct(self,persistent=False): + """Get the distinct elements of this data set. + + Usage sample: + >>> test = list(range(10)) + >>> test.extend(list(range(5))) + >>> len(test) + 15 + >>> DDS().load(test, 5).distinct().count() + 10 + + :returns: New child DDS object with distinct elements. + """ + return ( + self.map(lambda x: (x, None)) + .reduce_by_key(lambda x, _: x, persistent=persistent) + .map(lambda x: x[0]) + ) + + def count_by_value(self, arity=2, as_dict=True, as_fo=False, persistent=False): + """Amount of each element on this data set. + + Usage sample: + >>> first = DDS().load([0, 1, 2], 2) + >>> second = DDS().load([2, 3, 4], 3) + >>> dict(sorted( + ... first.union(second).count_by_value(as_dict=True).items() + ... )) + {0: 1, 1: 1, 2: 2, 3: 1, 4: 1} + + :param arity: Tree depth. + :param as_dict: As dictionary. + :param as_fo: As future object. + :return: List of tuples (element, number). + """ + + def count_partition(iterator): + counts = defaultdict(int) + for obj in iterator: + counts[obj] += 1 + return counts + + # Count locally and create dictionary partitions + local_results = self.map_partitions(count_partition).collect( + future_objects=True + ) + # Create a deque from partitions and start reduce + future_objects = deque(local_results) + + branch = [] + while future_objects: + branch = [] + while future_objects and len(branch) < arity: + temp = future_objects.popleft() + branch.append(temp) + + if len(branch) == 1: + break + + first, branch = branch[0], branch[1:] + reduce_dicts(first, branch) + future_objects.append(first) + + if as_dict: + if as_fo: + return branch[0] + branch[0] = compss_wait_on(branch[0]) + return dict(branch[0]) + + length = self.num_of_partitions() + new_partitions = [] + for i in range(length): + new_partitions.append(task_dict_to_list(branch[0], length, i)) + + return DDS().load(new_partitions, -1, persistent=persistent) + + def key_by(self, func): + """Create a (key,value) pair for each element where 'key' is f(value). + + Usage sample: + >>> dds = DDS().load(range(3), 2) + >>> dds.key_by(lambda x: str(x)).collect() + [('0', 0), ('1', 1), ('2', 2)] + + :param func: A Key Creator function which takes the element as a + parameter and returns the key. + :return: List of (key, value) pairs. + """ + return self.map(lambda x: (func(x), x)) + + def sum(self): + """Sum everything up. + + Usage sample: + >>> DDS().load(range(3), 2).sum() + 3 + + :returns: The sum of everything. + """ + return sum(self.map_partitions(lambda x: [sum(x)]).collect()) + + def count(self): + """Count everything up. + + Usage sample: + >>> DDS().load(range(3), 2).count() + 3 + + :return: Total number of elements. + """ + return self.map_partitions(lambda i: [sum(1 for _ in i)]).sum() + + def foreach(self, func): + """Apply a function to each element of this data set. + + CAUTION: Does not return anything. + + :param func: A void function. + :returns: None + """ + self.map(func) + # Wait for all the tasks to finish + compss_barrier() + + def collect( + self, # pylint: disable=R0912 + keep_partitions=False, + future_objects=False, + ): + """Return all elements from all partitions. + + Elements can be grouped by partitions by setting keep_partitions value + as True. + + Usage sample: + >>> dds = DDS().load(range(10), 2) + >>> dds.collect(True) + [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] + >>> DDS().load(range(10), 2).collect() + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + :param keep_partitions: Keep Partitions? + :param future_objects: Future objects? + :return: All elements from all partitions. + """ + processed = [] + if self.func: + print("FUNC????????????!!!!!!!!!!!!!!!!!!!!!!!!!!") + if self.paac: + for col in self.partitions: + processed.append(map_partition(self.func, None, col)) + else: + for _p in self.partitions: + print("½½½½½½½½½ ", _p.get_iterable()) + processed.append(map_partition(self.func, _p)) + # Reset the function! + self.func = None + else: + for _p in self.partitions: + if isinstance(_p, IPartitionGenerator): + print("IS AN INSTANCE!!!!!!!!!!!!!!!!!!!!!!!!!!") + processed.append(_p.retrieve_data()) + else: + print("NOT IS AN INSTANCE!!!!!!!!!!!!!!!!!!!!!!!!!!") + processed.append(_p) + + # Future objects cannot be extended for now... + if future_objects: + return processed + + processed = compss_wait_on(processed) + + ret = [] + if not keep_partitions: + for _pp in processed: + ret.extend(_pp) + else: + for _pp in processed: + ret.append(list(_pp)) + return ret + + def save_as_text_file(self, path): + """Save string representations of DDS elements as text files. + + This saving creates one file per partition. + + :param path: Destination file path. + :return: None. + """ + if self.paac: + for i, _p in enumerate(self.partitions): + map_and_save_text_file(self.func, i, path, None, _p) + compss_delete_object(_p) + else: + for i, _p in enumerate(self.partitions): + map_and_save_text_file(self.func, i, path, _p) + + def save_as_pickle(self, path): + """Save partitions of this DDS as pickle files. + + Each partition is saved as a separate file for the sake of parallelism. + + :param path:Destination file path. + :return: None. + """ + if self.paac: + for i, _p in enumerate(self.partitions): + map_and_save_pickle(self.func, i, path, None, _p) + else: + for i, _p in enumerate(self.partitions): + map_and_save_pickle(self.func, i, path, _p) + + # ################################################################ # + # ############## Functions for (Key, Value) pairs. ############### # + # ################################################################ # + + def collect_as_dict(self): + """Get (key,value) as { key: value }. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).collect_as_dict() + {'a': 1, 'b': 1} + + :return: Dict. + """ + return dict(self.collect()) + + def keys(self): + """Get keys. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).keys().collect() + ['a', 'b'] + + :return: List of keys. + """ + return self.map(lambda x: x[0]) + + def values(self): + """Get values. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 2)]).values().collect() + [1, 2] + + :return: List of values. + """ + return self.map(lambda x: x[1]) + + def partition_by( + self, partitioner_func=default_hash, num_of_partitions=-1, persistent=False + ): + """Create partitions by a Partition Func. + + Usage sample: + >>> dds = DDS().load(range(6)).map(lambda x: (x, x)) + >>> dds.partition_by(num_of_partitions=3).collect(True) + [[(0, 0), (3, 3)], [(1, 1), (4, 4)], [(2, 2), (5, 5)]] + + :param partitioner_func: A Function distribute data on partitions based + on for example, hash function. + :param num_of_partitions: Number of partitions to be created. + :return: Partitions. + """ + + def combine_lists(_partition): + # Elements of the partition are grouped by their + # previous partitions + ret = [] + for _li in _partition: + ret.extend(_li) + return ret + + nop = ( + len(self.partitions) + if num_of_partitions == -1 + else num_of_partitions + ) + + grouped = defaultdict(list) + + if self.paac: + for collection in self.partitions: + col = [[] for _ in range(nop)] + with EventMaster(3002): + distribute_partition( + col, self.func, partitioner_func, None, collection + ) + compss_delete_object(collection) + for _i in range(nop): + grouped[_i].append(col[_i]) + else: + for _part in self.partitions: + col = [[] for _ in range(nop)] + with EventMaster(3002): + distribute_partition( + col, self.func, partitioner_func, _part + ) + for _i in range(nop): + grouped[_i].append(col[_i]) + + future_partitions = [] + for key in sorted(grouped.keys()): + future_partitions.append(grouped[key]) + + return ( + DDS() + .load(future_partitions, -1, True, persistent=persistent) + .map_partitions(combine_lists) + ) + + def map_values(self, func): + """Apply a function to each value of (k, v) element of this data set. + + Usage sample: + >>> DDS().load([("a", 1), ("b", 1)]).map_values( + ... lambda x: x+1 + ... ).collect() + [('a', 2), ('b', 2)] + + :param func: A function which takes 'value's as parameter. + :return: New DDS. + """ + + def dummy(pair): + return pair[0], func(pair[1]) + + return self.map(dummy) + + def flatten_by_key(self, func): + """Reverse of combine by key.Flat (k, v) as (k, v1), (k, v2) etc. + + In detail: (key, values) as (key, value1), (key, value2) ... + + Usage sample: + >>> DDS().load([('a',[1, 2]), ('b',[1])]).flatten_by_key( + ... lambda x: x + ... ).collect() + [('a', 1), ('a', 2), ('b', 1)] + + :param func: A function to parse values. + :return: Flattened by key. + """ + + def dummy(key_value): + return ((key_value[0], x) for x in func(key_value[1])) + + return self.flat_map(dummy) + + def join(self, other, num_of_partitions=-1,persistent=False): + """Join DDS objects. + + Usage sample: + >>> x = DDS().load([("a", 1), ("b", 3)]) + >>> y = DDS().load([("a", 2), ("b", 4)]) + >>> sorted(x.join(y).collect()) + [('a', (1, 2)), ('b', (3, 4))] + + :param other: Another DDS object. + :param num_of_partitions: Number of partitions. + :return: Joined DDS objects. + """ + + def dispatch(seq): + buf_1, buf_2 = [], [] + for num, value in seq: + if num == 1: + buf_1.append(value) + elif num == 2: + buf_2.append(value) + return [(v, w) for v in buf_1 for w in buf_2] + + nop = ( + len(self.partitions) + if num_of_partitions == -1 + else num_of_partitions + ) + + buf_a = self.map_values(lambda v: (1, v)) + buf_b = other.map_values(lambda y: (2, y)) + + return ( + buf_a.union(buf_b,persistent=persistent) + .group_by_key(num_of_parts=nop,persistent=persistent) + .flatten_by_key(lambda x: dispatch(iter(x))) + ) + + def combine_by_key( + self, creator_func, combiner_func, merger_function, total_parts=-1,persistent=False + ): + """Combine elements of each key. + + :param creator_func: To apply to the first element of the key. Takes + only one argument which is the value from (k, v) + pair. (e.g: v = list(v)). + :param combiner_func: To apply when a new element with the same 'key' + is found. It is used to combine partitions + locally. Takes 2 arguments; first one is the + result of 'creator_func' where the second one + is a 'value' of the same 'key' from the same + partition. (e.g: v1.append(v2)). + :param merger_function: To merge local results. Basically takes two + arguments -both are results of 'combiner_func'. + (e.g: list_1.extend(list_2)). + :param total_parts: Number of partitions after combinations. + :return: Combined by key DDS object. + """ + + def combine_partition(partition): + """Combine partitions. + + :param partition: Dictionary of partitions. + :returns: List of combined partitions. + """ + res = {} + for key, val in partition: + res[key] = ( + combiner_func(res[key], val) + if key in res + else creator_func(val) + ) + return list(res.items()) + + def merge_partition(partition): + """Merge partitions. + + :param partition: Dictionary of partitions. + :returns: List of merged partitions. + """ + res = {} + for key, val in partition: + res[key] = ( + merger_function(res[key], val) if key in res else val + ) + return list(res.items()) + + ret = ( + self.map_partitions(combine_partition) + .partition_by(num_of_partitions=total_parts,persistent=persistent) + .map_partitions(merge_partition) + ) + + return ret + + def reduce_by_key(self, func,persistent=False): + """Reduce values for each key. + + Usage sample: + >>> DDS().load([("a",1), ("a",2)]).reduce_by_key( + ... (lambda a, b: a+b) + ... ).collect() + [('a', 3)] + + :param func: a reducer function which takes two parameters and + returns one. + :returns: Reduced values. + """ + return self.combine_by_key((lambda x: x), func, func,persistent=persistent) + + def count_by_key(self, as_dict=False,persistent=False): + """Count by key. + + Usage sample: + >>> DDS().load([("a", 100), ("a", 200)]).count_by_key(True) + {'a': 2} + + :param as_dict: See 'as_dict' argument of 'combine_by_key'. + :return: A new DDS with data set of list of tuples + (element, occurrence). + """ + return self.map(lambda x: x[0]).count_by_value(as_dict=as_dict,persistent=persistent) + + def sort_by_key( + self, ascending=True, num_of_parts=None, key_func=lambda x: x, persistent=False + ): + """Sort by key. + + :param ascending: Ascending. + :param num_of_parts: Number of parts. + :param key_func: Key function. + :return: Sorted by key DDS object. + """ + if num_of_parts is None: + num_of_parts = len(self.partitions) + + # Collect everything to take samples + col_parts = self.collect(future_objects=True) + samples = [] + for _part in col_parts: + samples.append(task_collect_samples(_part, 20, key_func)) + + samples = sorted( + list(itertools.chain.from_iterable(compss_wait_on(samples))) + ) + + bounds = [ + samples[int(len(samples) * (i + 1) / num_of_parts)] + for i in range(0, num_of_parts - 1) + ] + + def range_partitioner(key): + """Partition a range. + + :param key: Partition key. + :return: Partitioned range. + """ + part = bisect.bisect_left(bounds, key_func(key)) + if ascending: + return part + return num_of_parts - 1 - part + + def sort_partition(iterator): + """Sort a partition locally. + + :param iterator: List iterator. + :return: Sorted partition. + """ + chunk_size = 500 + iterator = iter(iterator) + chunks = [] + while True: + chunk = list(itertools.islice(iterator, chunk_size)) + chunk.sort( + key=lambda kv: key_func(kv[0]), reverse=not ascending + ) + if len(chunk) > 0: + chunks.append(chunk) + if len(chunk) < chunk_size: + break + else: + chunks.append( + chunk.sort( + key=lambda kv: key_func(kv[0]), reverse=not ascending + ) + ) + + if len(chunks) == 1: + return chunks[0] + else: + return heapq.merge( + *chunks, + key=lambda kv: key_func(kv[0]), + reverse=not ascending + ) + + partitioned = DDS().load(col_parts, -1, persistent=persistent).partition_by(range_partitioner,persistent=persistent) + return partitioned.map_partitions(sort_partition) + + def group_by_key(self, num_of_parts=-1,persistent=False): + """Group values of each key in a single list. + + A special and most used case of 'combine_by_key'. + + Usage sample: + >>> x = DDS().load([("a", 1), ("b", 2), ("a", 2), ("b", 4)]) + >>> sorted(x.group_by_key().collect()) + [('a', [1, 2]), ('b', [2, 4])] + + :param num_of_parts: Number of parts. + :returns: Grouped by key DDS object. + """ + + def _create(value): + return [value] + + def _merge(container, value): + container.append(value) + return container + + def _combine(container_a, container_b): + container_a.extend(container_b) + return container_a + + return self.combine_by_key( + _create, _merge, _combine, total_parts=num_of_parts, persistent=persistent + ) + + def take(self, num): + """Take the first num elements of DDS. + + :param num: Number of elements to be retrieved. + :return: First elements of DDS. + """ + items = [] + partitions = self.collect(future_objects=True) + taken = 0 + + for part in partitions: + _p = iter(compss_wait_on(part)) + while taken < num: + try: + items.append(next(_p)) + taken += 1 + except StopIteration: + break + if taken >= num: + break + + return items[:num] + + +class _ChildDDS(DDS): + """_ChildDDS class. + + Similar as DDS objects, with the only difference that _ChildDDS objects + inherit the partitions from their parents, and have functions to be mapped + to their partitions. + """ + + def __init__(self, parent, func): + """Create a new _ChildDDS object. + + :param parent: Parent DDS object. + :param func: Function. + """ + super().__init__() + self.paac = parent.paac + + if not isinstance(parent, _ChildDDS): + self.func = func + if isinstance(parent, DDS): + self.partitions = parent.partitions + else: + self.partitions = parent.partitions + par_func = parent.func + + def wrap_parent_func(partition): + return func(par_func(partition)) + + self.func = wrap_parent_func + + +def _run_tests(): + """Run tests. + + :returns: None. + """ + import doctest # pylint: disable=C0415 + + doctest.testmod() + + # Clean after testing + to_be_removed = ["test.file", "test.txt"] + for file_name in to_be_removed: + try: + os.remove(file_name) + except OSError: + pass + + +if __name__ == "__main__": + _run_tests() \ No newline at end of file diff --git a/examples/dds/pycompss/dds/examples/__init__.py b/examples/dds/pycompss/dds/examples/__init__.py new file mode 100644 index 00000000..800c4109 --- /dev/null +++ b/examples/dds/pycompss/dds/examples/__init__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the DDS examples.""" + +from pycompss.dds.examples.examples import inverted_indexing # noqa: F401 +from pycompss.dds.examples.examples import pi_estimation # noqa: F401 +from pycompss.dds.examples.examples import terasort # noqa: F401 +from pycompss.dds.examples.examples import transitive_closure # noqa: F401 +from pycompss.dds.examples.examples import word_count # noqa: F401 +from pycompss.dds.examples.examples import wordcount_k_means # noqa: F401 diff --git a/examples/dds/pycompss/dds/examples/examples.py b/examples/dds/pycompss/dds/examples/examples.py new file mode 100644 index 00000000..6286f9ca --- /dev/null +++ b/examples/dds/pycompss/dds/examples/examples.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples. + +This file contains the DDS examples. +""" + +import time + +from pycompss.dds.examples.pi_estimation import pi_estimation +from pycompss.dds.examples.word_count import word_count +from pycompss.dds.examples.terasort import terasort +from pycompss.dds.examples.inverted_indexing import inverted_indexing +from pycompss.dds.examples.transitive_closure import transitive_closure +from pycompss.dds.examples.wordcount_k_means import wordcount_k_means + + +def run_examples(): + """Run available examples. + + Each example is in a single file that creates a sample dataset + to be tested. + + :return: None. + """ + print("________RUNNING EXAMPLES_________") + start_time = time.time() + + pi_estimation_check = pi_estimation() + word_count_check = word_count() + terasort_check = terasort() + inverted_indexing_check = inverted_indexing() + transitive_closure_check = transitive_closure() + wordcount_k_means_check = wordcount_k_means() + print("____FINISHED RUNNING EXAMPLES____") + print("STATUS:") + print(f"- Pi estimation: {pi_estimation_check}") + print(f"- Wordcount: {word_count_check}") + print(f"- Terasort: {terasort_check}") + print(f"- Inverted indexing: {inverted_indexing_check}") + print(f"- Transitive closure: {transitive_closure_check}") + print(f"- Wordcount k-means: {wordcount_k_means_check}") + print("---------------------------------") + print(f"- TOTAL ELAPSED TIME: {time.time() - start_time} (s)") + print("---------------------------------") + + if ( + pi_estimation_check + and word_count_check + and terasort_check + and inverted_indexing_check + and transitive_closure_check + and wordcount_k_means + ): + print("ALL EXAMPLES FINISHED SUCCESSFULLY") + else: + print("ERRORS WHERE FOUND IN ONE OR MORE EXAMPLES") + + print("---------------------------------") + + +if __name__ == "__main__": + run_examples() diff --git a/examples/dds/pycompss/dds/examples/inverted_indexing.py b/examples/dds/pycompss/dds/examples/inverted_indexing.py new file mode 100644 index 00000000..80d7236f --- /dev/null +++ b/examples/dds/pycompss/dds/examples/inverted_indexing.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - inverted indexing. + +This file contains the DDS inverted indexing example. +""" + +import os +import random +import shutil +import time +from collections import defaultdict + +from pycompss.dds import DDS + + +def create_dataset() -> (str, str): + """Create a dummy dataset. + + :return: Path to a folder containing a set of dummy files and the expected. + """ + random.seed(1) + vocabulary = [ + "PyCOMPSs", + "Hello", + "World", + "Lorem", + "Ipsum", + "Barcelona", + "Supercomputing", + "Center", + ] + current_directory = os.getcwd() + dataset_path = os.path.join(current_directory, "inverted_indexing_dataset") + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + files = [] + for i in range(4): + files.append(os.path.join(dataset_path, f"file_{i}.txt")) + pairs = defaultdict(list) + + for word in vocabulary: + _files = random.sample(files, 2) + for _file in _files: + with open(_file, "a") as tmp_f: + tmp_f.write(word + " ") + pairs[word].append(_file) + + return dataset_path, pairs + + +def clean_dataset(dataset_path): + """Remove the given dataset. + + :param dataset_path: Folder to be removed. + :return: None. + """ + shutil.rmtree(dataset_path) + + +def _invert_files(pair): + """Invert files. + + :param pair: Pair. + :results: List of items. + """ + res = {} + for word in pair[1].split(): + res[word] = [pair[0]] + return list(res.items()) + + +def inverted_indexing(): + """Inverted indexing. + + :results: Inverted indexing result. + """ + print("--- INVERTED INDEXING ---") + + # By default, create a dummy dataset and perform wordcount over it. + # It could be changed to: + # path_file = sys.argv[1] + # if you desire to perform the word count over a given dataset + # (remember to comment the check_results call in this case). + dataset_path, pairs = create_dataset() + start_time = time.time() + + results = ( + DDS() + .load_files_from_dir(dataset_path) + .flat_map(_invert_files) + .reduce_by_key(lambda a, b: a + b) + .collect() + ) + + print(f"- Results: {results[-1:]}") + print(f"- Elapsed Time: {time.time() - start_time} (s)") + print("-------------------------") + + clean_dataset(dataset_path) + + check = [] + for word, files in results: + check.append(set(pairs[word]).issubset(set(files))) + return all(test for test in check) diff --git a/examples/dds/pycompss/dds/examples/pi_estimation.py b/examples/dds/pycompss/dds/examples/pi_estimation.py new file mode 100644 index 00000000..693d094e --- /dev/null +++ b/examples/dds/pycompss/dds/examples/pi_estimation.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - pi estimation. + +This file contains the DDS pi estimation example. +""" + +import random +import time + +from pycompss.dds import DDS + + +def inside(_): + """Check if inside. + + :return: If inside. + """ + rand_x = random.random() + rand_y = random.random() + return (rand_x * rand_x) + (rand_y * rand_y) < 1 + + +def pi_estimation(): + """Pi estimation. + + Example is taken from: https://spark.apache.org/examples.html + + :return: If the pi value calculated is between 3.1 and 3.2. + """ + start = time.time() + + print("--- PI ESTIMATION ---") + + print("- Estimating Pi by 'throwing darts' algorithm.") + tries = 100000 + print(f"- Number of tries: {tries}") + + count = DDS().load(range(0, tries), 10).filter(inside).count() + rough_pi = 4.0 * count / tries + + print(f"- Pi is roughly {rough_pi}") + print("- Elapsed Time: ", time.time() - start) + print("---------------------") + + return 3.1 < rough_pi < 3.2 diff --git a/examples/dds/pycompss/dds/examples/terasort.py b/examples/dds/pycompss/dds/examples/terasort.py new file mode 100644 index 00000000..efd4e6f3 --- /dev/null +++ b/examples/dds/pycompss/dds/examples/terasort.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - terasort. + +This file contains the DDS terasort example. +""" + +import os +import shutil +import random +import time + +from pycompss.dds import DDS + + +def create_dataset() -> (str, str): + """Create a dummy dataset. + + :return: Path to a folder containing a set of dummy files and result path. + """ + random.seed(1) + current_directory = os.getcwd() + dataset_path = os.path.join(current_directory, "terasort_dataset") + dataset_dest_path = os.path.join( + current_directory, "terasort_dataset_result" + ) + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + for i in range(4): + file_path = os.path.join(dataset_path, f"file_{i}.txt") + with open(file_path, "w", encoding="utf-8") as file_path_fd: + for j in range(10): + file_path_fd.write(f"{random.randint(0, 1000)},{i*j}\n") + # if not os.path.exists(dataset_dest_path): + # os.makedirs(dataset_dest_path) + return dataset_path, dataset_dest_path + + +def check_results(results) -> bool: + """Check if the given results match the expected result. + + CAUTION: Only works for the dummy dataset. + + :param results: Dictionary containing the words and their appearance. + :return: If the result is the expected or not. + """ + expected = [ + ("104", "15"), + ("120", "0"), + ("137", "0"), + ("2", "12"), + ("214", "5"), + ("22", "27"), + ("234", "6"), + ("261", "0"), + ("272", "18"), + ("29", "8"), + ("31", "24"), + ("325", "21"), + ("388", "3"), + ("399", "2"), + ("443", "4"), + ("456", "16"), + ("460", "0"), + ("483", "1"), + ("499", "7"), + ("507", "0"), + ("582", "0"), + ("605", "9"), + ("622", "6"), + ("64", "0"), + ("667", "2"), + ("712", "14"), + ("738", "0"), + ("779", "0"), + ("780", "8"), + ("782", "0"), + ("785", "10"), + ("807", "4"), + ("821", "0"), + ("821", "3"), + ("855", "0"), + ("867", "0"), + ("914", "9"), + ("923", "18"), + ("96", "6"), + ("967", "12"), + ] + + return results == expected + + +def clean_dataset(dataset_path): + """Remove the given dataset. + + :param dataset_path: Folder to be removed. + :return: None. + """ + shutil.rmtree(dataset_path) + + +def files_to_pairs(element): + """Pair files. + + :param element: String of elements. + :return: List of pairs. + """ + tuples = [] + lines = element[1].split("\n") + for _l in lines: + if not _l: + continue + k_v = _l.split(",") + tuples.append(tuple(k_v)) + + return tuples + + +def terasort(): + """Apply terasort over a dummy dataset. + + :return: Sorting result. + """ + print("--- TERASORT ---") + + # By default, create a dummy dataset and perform wordcount over it. + # It could be changed to: + # dir_path = sys.argv[1] + # dest_path = sys.argv[2] + # if you desire to perform the word count over a given dataset + # (remember to comment the check_results call in this case). + dir_path, _ = create_dataset() + + start_time = time.time() + + results = ( + DDS() + .load_files_from_dir(dir_path) + .flat_map(files_to_pairs) + .sort_by_key() + # .save_as_text_file(dest_path) + .collect() + ) + + print(f"- Results: {results}") + print(f"- Elapsed Time: {time.time() - start_time} (s)") + print("----------------") + + clean_dataset(dir_path) + + return check_results(results) diff --git a/examples/dds/pycompss/dds/examples/transitive_closure.py b/examples/dds/pycompss/dds/examples/transitive_closure.py new file mode 100644 index 00000000..7425daa0 --- /dev/null +++ b/examples/dds/pycompss/dds/examples/transitive_closure.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- co_ding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - transitive closure. + +This file contains the DDS transitive closure example. +""" + +import time +import random + +from pycompss.dds import DDS + + +def _generate_graph(): + """Generate graph. + + :return: Set of edges. + """ + random.seed(1) + num_edges = 10 + num_vertices = 5 + rand = random.Random(42) + + edges = set() + while len(edges) < num_edges: + src = rand.randrange(0, num_vertices) + dst = rand.randrange(0, num_vertices) + if src != dst: + edges.add((src, dst)) + return edges + + +def transitive_closure(partitions=2): + """Transitive closure. + + :param partitions: Number of partitions. + :results: Transitive closure result. + """ + print("--- TRANSITIVE CLOSURE ---") + + edges = _generate_graph() + start_time = time.time() + + o_d = DDS().load(edges, partitions).collect(future_objects=True) + + # Because join() joins on keys, the edges are stored in reversed order. + edges = DDS().load(o_d, -1).map(lambda x_y: (x_y[1], x_y[0])) + + next_count = DDS().load(o_d, -1).count() + + while True: + old_count = next_count + # Perform the join, obtaining an RDD of (y, (z, x)) pairs, + # then project the result to obtain the new (x, z) paths. + new_edges = ( + DDS() + .load(o_d, -1) + .join(edges) + .map(lambda __a_b: (__a_b[1][1], __a_b[1][0])) + ) + o_d = ( + DDS() + .load(o_d, -1) + .union(new_edges) + .distinct() + .collect(future_objects=True) + ) + + next_count = DDS().load(o_d, -1).count() + + if next_count == old_count: + break + + print(f"- TC has {next_count} edges") + print(f"- Elapsed Time: {time.time() - start_time} (s)") + print("--------------------------") + + return next_count == 20 diff --git a/examples/dds/pycompss/dds/examples/word_count.py b/examples/dds/pycompss/dds/examples/word_count.py new file mode 100644 index 00000000..1598f171 --- /dev/null +++ b/examples/dds/pycompss/dds/examples/word_count.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - word count. + +This file contains the DDS word count example. +""" + +import os +import shutil +import time + +from pycompss.dds import DDS + + +def create_dataset() -> str: + """Create a dummy dataset. + + :return: Path to a folder containing a set of dummy files. + """ + current_directory = os.getcwd() + dataset_path = os.path.join(current_directory, "wordcount_dataset") + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + for i in range(4): + file_path = os.path.join(dataset_path, f"file_{i}.txt") + with open(file_path, "w", encoding="utf-8") as file_path_fd: + for j in range(10): + file_path_fd.write( + f"Dummy dataset content number {i} in line {j}\n" + ) + return dataset_path + + +def check_results(results) -> bool: + """Check if the given results match the expected result. + + CAUTION: Only works for the dummy dataset. + + :param results: Dictionary containing the words and their appearance. + :return: If the result is the expected or not. + """ + expected = { + "Dummy": 40, + "dataset": 40, + "content": 40, + "number": 40, + "0": 14, + "in": 40, + "line": 40, + "1": 14, + "2": 14, + "3": 14, + "4": 4, + "5": 4, + "6": 4, + "7": 4, + "8": 4, + "9": 4, + } + return results == expected + + +def clean_dataset(dataset_path): + """Remove the given dataset. + + :param dataset_path: Folder to be removed. + :return: None. + """ + shutil.rmtree(dataset_path) + + +def word_count(): + """Word count. + + Perform word counting from a dummy dataset. + + :results: If the result matches the expected. + """ + print("--- WORD COUNT ---") + + # By default, create a dummy dataset and perform wordcount over it. + # It could be changed to: + # path_file = sys.argv[1] + # if you desire to perform the word count over a given dataset + # (remember to comment the check_results call in this case). + dataset_path = create_dataset() + start_time = time.time() + + results = ( + DDS() + .load_files_from_dir(dataset_path) + .flat_map(lambda x: x[1].split()) + .map(lambda x: "".join(e for e in x if e.isalnum())) + .count_by_value(as_dict=True) + ) + + print(f"- Results: {results}") + print(f"- Elapsed Time: {time.time() - start_time} (s)") + print("------------------") + + clean_dataset(dataset_path) + + return check_results(results) diff --git a/examples/dds/pycompss/dds/examples/wordcount_k_means.py b/examples/dds/pycompss/dds/examples/wordcount_k_means.py new file mode 100644 index 00000000..2bcc04f4 --- /dev/null +++ b/examples/dds/pycompss/dds/examples/wordcount_k_means.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - Examples - wordcount k-means. + +This file contains the DDS wordcount k-means example. +""" + +import os +import time +import numpy as np +from collections import deque +from collections import defaultdict + +from pycompss.api.api import compss_wait_on +from pycompss.dds import DDS +from pycompss.dds.examples.word_count import create_dataset +from pycompss.dds.examples.word_count import clean_dataset +from pycompss.dds.examples.wordcount_k_means_tasks import ( + cluster_points_partial, +) +from pycompss.dds.examples.wordcount_k_means_tasks import partial_sum +from pycompss.dds.examples.wordcount_k_means_tasks import task_count_locally +from pycompss.dds.examples.wordcount_k_means_tasks import reduce_centers +from pycompss.dds.examples.wordcount_k_means_tasks import get_similar_files + + +def has_converged(mu, old_mu, epsilon): + """Check if Kmeans has converged. + + Checks the distance difference between centers and old centers and + compares with epsilon. + + :param mu: Centers. + :param old_mu: Old centers. + :param epsilon: Epsilon. + :return: If has converged. + """ + if not old_mu: + return False + + aux = [np.linalg.norm(old_mu[i] - mu[i]) for i in range(len(mu))] + distance = sum(aux) + print("Distance_T: " + str(distance)) + return distance < (epsilon**2) + + +def merge_reduce(function, data): + """Merge reducing as binary tree. + + :param function: Merging function. + :param data: List of elements to reduce. + :results: Reduced result. + """ + queue = deque(list(range(len(data)))) + while len(queue): + elem_x = queue.popleft() + if len(queue): + elem_y = queue.popleft() + data[elem_x] = function(data[elem_x], data[elem_y]) + queue.append(elem_x) + else: + return data[elem_x] + + +# # Used only in step 2 of wordcount k-means +# def __count_locally__(element): +# from collections import Counter +# file_name, text = element +# +# filtered_words = [word for word in text.split() if word.isalnum()] +# cnt = Counter(filtered_words) +# +# for _word in vocab.keys(): +# if _word not in cnt: +# cnt[_word] = 0 +# +# return file_name, sorted(cnt.items()) +# +# +# # Used only in step 2 of wordcount k-means +# def __gen_array__(element): +# import numpy as np +# values = [int(v) for k, v in element[1]] +# return np.array(values) + + +def check_results(results) -> bool: + """Check if the given results match the expected result. + + CAUTION: Only works for the dummy dataset. + + :param results: Dictionary containing the words and their appearance. + :return: If the result is the expected or not. + """ + expected = [ + [ + "file_0.txt", + [ + ("file_1.txt", 0.9764065991188053), + ("file_2.txt", 0.973527551367599), + ("file_3.txt", 0.9737561884661387), + ], + ], + [ + "file_1.txt", + [ + ("file_0.txt", 0.9764065991188053), + ("file_2.txt", 0.9955979819385568), + ("file_3.txt", 0.9946889892919744), + ], + ], + [ + "file_2.txt", + [ + ("file_0.txt", 0.973527551367599), + ("file_1.txt", 0.9955979819385568), + ("file_3.txt", 0.9947750570357584), + ], + ], + [ + "file_3.txt", + [ + ("file_0.txt", 0.9737561884661387), + ("file_1.txt", 0.9946889892919744), + ("file_2.txt", 0.9947750570357584), + ], + ], + ] + return results == expected + + +def wordcount_k_means(dim=16): + """Wordcount K-means. + + The number of dimensions corresponds to: dim = len(vocabulary) + + :param dim: Dimensions. + :return: The wordcount k-means evaluation check. + """ + print("--- WORDCOUNT K-MEANS ---") + np.random.seed(1) + + # By default, create a dummy dataset and perform wordcount over it. + # It could be changed to: + # path_file = sys.argv[1] + # if you desire to perform the word count over a given dataset + # (remember to comment the check_results call in this case). + path_file = create_dataset() + start_time = time.time() + + vocab = ( + DDS() + .load_files_from_dir(path_file, num_of_parts=4) + .flat_map(lambda x: x[1].split()) + .map(lambda x: "".join(e for e in x if e.isalnum())) + .count_by_value(arity=2, as_dict=True, as_fo=True) + ) + + total = len(os.listdir(path_file)) + max_iter = 2 + frags = 4 + epsilon = 1e-10 + size = total / frags + k = 4 + + # to access file names by index returned from the clusters. + # load_files_from_list will also sort them alphabetically + indexes = [ + os.path.join(path_file, f) for f in sorted(os.listdir(path_file)) + ] + + # step 2 + # wc_per_file = DDS().load_files_from_dir(files_path, num_of_parts=frags)\ + # .map(__count_locally__, vocabulary)\ + # .map(__gen_array__)\ + + wc_per_file = [] + + for file_name in sorted(os.listdir(path_file)): + wc_per_file.append( + task_count_locally(os.path.join(path_file, file_name), vocab) + ) + + mu = [np.random.randint(1, 3, dim) for _ in range(frags)] + + old_mu = [] + clusters = [] + iteration = 0 + + while iteration < max_iter and not has_converged(mu, old_mu, epsilon): + old_mu = mu + clusters = [ + cluster_points_partial([wc_per_file[f]], mu, int(f * size)) + for f in range(frags) + ] + partial_result = [ + partial_sum([wc_per_file[f]], clusters[f], int(f * size)) + for f in range(frags) + ] + mu = merge_reduce(reduce_centers, partial_result) + mu = compss_wait_on(mu) + mu = [mu[c][1] / mu[c][0] for c in mu] + while len(mu) < k: + # Add a new random center if one of the centers has no points. + mu.append(np.random.randint(1, 3, dim)) + iteration += 1 + + clusters_with_frag = compss_wait_on(clusters) + cluster_sets = defaultdict(list) + + for _d in clusters_with_frag: + for _k in _d: + cluster_sets[_k] += [indexes[i] for i in _d[_k]] + + # step 4 and 5 combined + sims_per_file = {} + + for k in cluster_sets: + clus = cluster_sets[k] + for fayl in clus: + sims_per_file[fayl] = get_similar_files(fayl, clus) + + sims_per_file = compss_wait_on(sims_per_file) + + results = [] + for file_path in list(sims_per_file.keys())[:10]: + file_name = os.path.basename(file_path) + sims = [] + for sim in sims_per_file[file_path][:5]: + sims.append((os.path.basename(sim[0]), sim[1])) + results.append([file_name, sims]) + print(f"- {file_name} --- sims --> {sims}") + + print(f"- Iterations: {iteration}") + print(f"- Elapsed Time: {time.time() - start_time} (s)") + print("-------------------------") + + clean_dataset(path_file) + + return check_results(results) diff --git a/examples/dds/pycompss/dds/examples/wordcount_k_means_tasks.py b/examples/dds/pycompss/dds/examples/wordcount_k_means_tasks.py new file mode 100644 index 00000000..a786a32a --- /dev/null +++ b/examples/dds/pycompss/dds/examples/wordcount_k_means_tasks.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs DDS - examples - wordcount k-means tasks. + +This file contains the DDS tasks example. +""" + +from collections import Counter +import numpy as np +from pycompss.api.parameter import COLLECTION_IN + +from pycompss.api.task import task + + +@task(returns=dict, xp=COLLECTION_IN) +def cluster_points_partial(xp, mu, ind): + """Measure the distance from the centers to the given points. + + :param xp: Points. + :param mu: Centers. + :param ind: Offset. + :returns: Dictionary with the distances. + """ + dic = {} + for x in enumerate(xp): + bestmukey = min( + [(i[0], np.linalg.norm(x[1] - mu[i[0]])) for i in enumerate(mu)], + key=lambda t: t[1], + )[0] + + if bestmukey not in dic: + dic[bestmukey] = [x[0] + ind] + else: + dic[bestmukey].append(x[0] + ind) + + return dic + + +@task(returns=dict, xp=COLLECTION_IN) +def partial_sum(xp, clusters, ind): + """Accumulates the distances. + + :param xp: Points. + :param clusters: Clusters (points associated to each cluster). + :param ind: Offset. + :returns: Dictionary with the accumulated distance. + """ + p = [(i, [(xp[j - ind]) for j in clusters[i]]) for i in clusters] + dic = {} + for i, l in p: + dic[i] = (len(l), np.sum(l, axis=0)) + return dic + + +@task() +def task_count_locally(file_path, vocab): + """Task count task. + + :param file_path: Input file. + :param vocab: Words filter. + :returns: np array with the appearances. + """ + # read the file + with open(file_path) as file_path_fd: # pylint: disable=W1514 + text = file_path_fd.read() + + filtered_words = [word for word in text.split() if word.isalnum()] + cnt = Counter(filtered_words) + + for _word in vocab.keys(): + if _word not in cnt: + cnt[_word] = 0 + + values = [int(v) for k, v in sorted(cnt.items())] + return np.array(values) + + +# dict inout?? +@task(returns=dict, priority=True) +def reduce_centers(a, b): + """Reduce centers. + + :param a: First dictionary. + :param b: Second dictionary. + :results: Updated a + """ + for key in b: + if key not in a: + a[key] = b[key] + else: + a[key] = (a[key][0] + b[key][0], a[key][1] + b[key][1]) + return a + + +@task(returns=list) +def get_similar_files(fayl, cluster, threshold=0.90): + """Calculate average similarity of a file against a list of files. + + :param threshold: Threshold level. + :param fayl: File to be compared with its cluster. + :param cluster: File names to be compared with the file. + :return: Average similarity. + """ + import spacy # pylint: disable=import-outside-toplevel + + nlp = spacy.load("en_core_web_sm") + + with open(fayl) as fayl_fd: # pylint: disable=W1514 + d1 = nlp(fayl_fd.read()) + ret = [] + + for other in cluster: + if other == fayl: + continue + with open(other) as other_fd: # pylint: disable=W1514 + d2 = nlp(other_fd.read()) + s = d1.similarity(d2) + if s >= threshold: + ret.append((other, s)) + return ret diff --git a/examples/dds/pycompss/ext/compssmodule.cc b/examples/dds/pycompss/ext/compssmodule.cc new file mode 100644 index 00000000..b62165d6 --- /dev/null +++ b/examples/dds/pycompss/ext/compssmodule.cc @@ -0,0 +1,1049 @@ +/* + * Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include +/* ****************************************************************** */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +bool DEBUG_MODE = false; +char* HEADER = "[PY-C EXTENSION] - "; + +struct module_state { + PyObject* error; +}; + +/* + This is simply a data container + See process_task +*/ +struct parameter { + PyObject* value; + int type; + int direction; + int stream; + std::string prefix; + int size; + std::string name; + std::string c_type; + std::string weight; + int keep_rename; + + parameter(PyObject *v, int t, int d, int s, std::string p, int sz, std::string n, std::string ct, std::string w, int kr) { + value = v; + type = t; + direction = d; + stream = s; + prefix = p; + size = sz; + name = n; + c_type = ct; + weight = w; + keep_rename = kr; + } + + parameter() { } + ~parameter() { + // Nothing to do, as the PyObject pointed by value is owned and managed by + // the Python interpreter (and therefore incorrect to free it) + } + +}; + +/* + Python3 compatibility only. + See https://docs.python.org/3/howto/cporting.html +*/ +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#define PyInt_FromLong PyLong_FromLong +#define PyInt_AsLong PyLong_AsLong + +////////////////////////////////////////////////////////////// +// Debugging functions +////////////////////////////////////////////////////////////// + +static void debug(const char * format, ...){ + if (DEBUG_MODE == true) { + va_list arg; + va_start(arg, format); + std::string message = HEADER; + message += format; + vprintf(message.c_str(), arg); + va_end(arg); + fflush(stdout); + } +} + +/* + Set debug mode. + Separate function since it can be deactivated and/or activated when + necessary. +*/ +static PyObject* set_debug(PyObject* self, PyObject* args) { + bool debug_mode = PyObject_IsTrue(PyTuple_GetItem(args, 0)); + DEBUG_MODE = debug_mode; + Py_RETURN_NONE; +} + +////////////////////////////////////////////////////////////// +// Internal functions +////////////////////////////////////////////////////////////// + +/* + Auxiliary function to print errors. +*/ +static PyObject* error_out(PyObject *m) { + struct module_state* st = GETSTATE(m); + PyErr_SetString(st->error, "Compss extension module: Something bad happened"); + return NULL; +} + +/* + Auxiliary function that allows us to translate a PyStringObject + (which is also a PyObject) to a char* + + W A R N I N G + + Do not EVER free a pointer obtained from here. It will eventually happen + when the PyObject container gets refcd to 0. +*/ +static char* _pystring_to_char(PyObject* c) { + return + PyBytes_AsString(PyUnicode_AsEncodedString(c, "utf-8", "Error ~")); +} + +/* + Auxiliary functions to convert pystring to string. +*/ +static std::string _pystring_to_string(PyObject* c) { + return std::string( + PyBytes_AsString(PyUnicode_AsEncodedString(c, "utf-8", "Error ~")) + ); +} + +/* + Given an integer that can be translated to a type according to the + datatype enum (see param_metadata.h), return its size. + If compiled with debug, the name of the type will also appear. +*/ +static int _get_type_size(int type) { + switch ((enum datatype) type) { + case file_dt: + debug("- Type: file_dt\n"); + return sizeof(char*); + case external_stream_dt: + debug("- Type: external_stream_dt\n"); + return sizeof(char*); + case external_psco_dt: + debug("- Type: external_psco_dt\n"); + return sizeof(char*); + case string_dt: + debug("- Type: string_dt\n"); + return sizeof(char*); + case string_64_dt: + debug("- Type: string_64_dt\n"); + return sizeof(char*); + case int_dt: + debug("- Type: int_dt\n"); + return sizeof(int); + case long_dt: + debug("- Type: long_dt\n"); + return sizeof(long); + case double_dt: + debug("- Type: double_dt\n"); + return sizeof(double); + case boolean_dt: + debug("- Type: boolean_dt\n"); + return sizeof(int); + default: + debug("- Type: default\n"); + return 0; + } + // Here for anti-warning purposes, but this statement will never be reached + return 0; +} + +/* + Writes the bit representation of the contents of a PyObject in the + memory address indicated by the void*. +*/ +static void* _get_void_pointer_to_content(PyObject* val, int type, int size) { + // void is not a sizeable type, so we need something that allows us + // to allocate the exact byte amount we want + void* ret = new std::uint8_t[size]; + switch ((enum datatype) type) { + case file_dt: + case directory_dt: + case external_stream_dt: + case external_psco_dt: + case string_dt: + case string_64_dt: + case collection_dt: + case dict_collection_dt: + *(char**)ret = _pystring_to_char(val); + break; + case int_dt: + *(int*)ret = int(PyInt_AsLong(val)); + break; + case long_dt: + *(long*)ret = PyLong_AsLong(val); + break; + case double_dt: + *(double*)ret = PyFloat_AsDouble(val); + break; + case boolean_dt: + *(int*)ret = int(PyInt_AsLong(val)); + break; + default: + break; + } + return ret; +} + + +////////////////////////////////////////////////////////////// +// API functions +////////////////////////////////////////////////////////////// + +/* + Start a COMPSs-Runtime instance. +*/ +static PyObject* start_runtime(PyObject* self, PyObject* args) { + debug("Start runtime\n"); + GS_On(); + Py_RETURN_NONE; +} + +/* + Stop the current COMPSs-Runtime instance. +*/ +static PyObject* stop_runtime(PyObject* self, PyObject* args) { + int code = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + debug("Stop runtime with code: %i\n", (code)); + GS_Off(code); + Py_RETURN_NONE; +} + +/* + Cancel all application tasks +*/ +static PyObject* cancel_application_tasks(PyObject* self, PyObject* args){ + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + debug("COMPSs cancel application tasks for AppId: %ld\n", (app_id)); + GS_Cancel_Application_Tasks(app_id); + Py_RETURN_NONE; +} + +/* + A function that initiatiates the pipe mechanism setting two pipes. + First argument will be the command_pipe, where the binding_commons library + writes the commands, and the second argument will be the result_pipe, where + the binding_commons library expects the command results +*/ +static PyObject* set_pipes(PyObject* self, PyObject* args){ + char* command_pipe = _pystring_to_char(PyTuple_GetItem(args, 0)); + char* result_pipe = _pystring_to_char(PyTuple_GetItem(args, 1)); + GS_set_pipes(command_pipe,result_pipe); + Py_RETURN_NONE; +} + +/* + A function that reads a command from the pipe mechanism set with set_pipes + method. No arguments are used in the function. The result is a string containing + the command read from the pipe. +*/ +static PyObject* read_pipes(PyObject* self, PyObject* args){ + char* command; + GS_read_pipes(&command); + PyObject *ret = Py_BuildValue("s", command); + return ret; +} + +/* + A function that, given a task with its decorator parameters, translates these + fields to a C-friendly format and sends them to the bindings_common part, + which is responsible to send them to the COMPSs runtime via JNI. + + This function can be decomposed into three major parts: argument parsing, + argument packing and conversion to bindings_common argument format. +*/ +static PyObject* process_task(PyObject* self, PyObject* args) { + /* + Parse python object arguments and get pointers to them. + lsiiiiiiiOOOOOO must be read as + "long, string, integer, integer, integer, integer, integer, integer, + Object, Object, Object, Object, Object, Object" + */ + debug("Process task:\n"); + long app_id; + char* signature; + char* on_failure; + int priority, num_nodes, reduce, chunk_size, replicated, distributed, has_target, num_returns, time_out; + PyObject *values; + PyObject *names; + PyObject *compss_types; + PyObject *compss_directions; + PyObject *compss_streams; + PyObject *compss_prefixes; + PyObject *content_types; + PyObject *weights; + PyObject *keep_renames; + // See comment from above for the meaning of this "magic" string + if(!PyArg_ParseTuple(args, "lssiiiiiiiiiOOOOOOOOO", &app_id, &signature, &on_failure, &time_out, &priority, + &num_nodes, &reduce, &chunk_size, &replicated, &distributed, &has_target, &num_returns, &values, &names, &compss_types, + &compss_directions, &compss_streams, &compss_prefixes, &content_types, &weights, &keep_renames)) { + // Return NULL after ParseTuple automatically translates to "wrong + // arguments were passed, we expected an integer instead"-like errors + return NULL; + } + debug("- App id: %ld\n", app_id); + debug("- Signature: %s\n", signature); + debug("- On Failure: %s\n", on_failure); + debug("- Time Out: %d\n", time_out); + debug("- Priority: %d\n", priority); + debug("- Reduce: %d\n", reduce); + debug("- Chunk size: %d\n", chunk_size); + debug("- MPI Num nodes: %d\n", num_nodes); + debug("- Replicated: %d\n", replicated); + debug("- Distributed: %d\n", distributed); + debug("- Has target: %d\n", has_target); + /* + Obtain and set all parameter data, and pack it in a struct vector. + + See parameter and its field at the top of this source code file + */ + Py_ssize_t num_pars = PyList_Size(values); + debug("Num pars: %d\n", int(num_pars)); + std::vector< parameter > params(num_pars); + std::vector< char* > prefix_charp(num_pars); + std::vector< char* > name_charp(num_pars); + std::vector< char* > c_type_charp(num_pars); + std::vector< char* > weight_charp(num_pars); + int num_fields = 9; + std::vector< void* > unrolled_parameters(num_fields * num_pars, NULL); + + for(int i = 0; i < num_pars; ++i) { + debug("Processing parameter %d ...\n", i); + PyObject *value = PyList_GetItem(values, i); + PyObject *type = PyList_GetItem(compss_types, i); + PyObject *direction = PyList_GetItem(compss_directions, i); + PyObject *stream = PyList_GetItem(compss_streams, i); + PyObject *prefix = PyList_GetItem(compss_prefixes, i); + PyObject *name = PyList_GetItem(names, i); + PyObject *c_type = PyList_GetItem(content_types, i); + PyObject *weight = PyList_GetItem(weights, i); + PyObject *k_rename = PyList_GetItem(keep_renames, i); + std::string received_prefix = _pystring_to_string(prefix); + std::string received_name = _pystring_to_string(name); + std::string received_c_type = _pystring_to_string(c_type); + std::string received_weight = _pystring_to_string(weight); + + params[i] = parameter( + value, + int(PyInt_AsLong(type)), + int(PyInt_AsLong(direction)), + int(PyInt_AsLong(stream)), + received_prefix, + _get_type_size(int(PyInt_AsLong(type))), + received_name, + received_c_type, + received_weight, + int(PyInt_AsLong(k_rename)) + ); + + debug("Adapting C++ data to BC-JNI format...\n"); + /* + Adapt the parsed data to a bindings-common friendly format. + These pointers do NOT need to be freed, as the pointed contents + are out of our control (will be scope-cleaned, or point to PyObject + contents) + */ + prefix_charp[i] = (char*) params[i].prefix.c_str(); + name_charp[i] = (char*) params[i].name.c_str(); + c_type_charp[i] = (char*) params[i].c_type.c_str(); + weight_charp[i] = (char*) params[i].weight.c_str(); + + debug("Processing parameter %d\n", i); + /* + Adapt the parsed data to a bindings-common friendly format. + These pointers do NOT need to be freed, as the pointed contents + are out of our control (will be scope-cleaned, or point to PyObject + contents) + */ + unrolled_parameters[num_fields * i + 0] = _get_void_pointer_to_content(params[i].value, params[i].type, params[i].size); + unrolled_parameters[num_fields * i + 1] = (void*) ¶ms[i].type; + unrolled_parameters[num_fields * i + 2] = (void*) ¶ms[i].direction; + unrolled_parameters[num_fields * i + 3] = (void*) ¶ms[i].stream; + unrolled_parameters[num_fields * i + 4] = (void*) &prefix_charp[i]; + unrolled_parameters[num_fields * i + 5] = (void*) &name_charp[i]; + unrolled_parameters[num_fields * i + 6] = (void*) &c_type_charp[i]; + unrolled_parameters[num_fields * i + 7] = (void*) &weight_charp[i]; + unrolled_parameters[num_fields * i + 8] = (void*) ¶ms[i].keep_rename; + + debug("----> Value is at %p\n", ¶ms[i].value); + debug("----> Type: %d\n", params[i].type); + debug("----> Direction: %d\n", params[i].direction); + debug("----> Stream: %d\n", params[i].stream); + debug("----> Prefix: %s\n", prefix_charp[i]); + debug("----> Size: %d\n", params[i].size); + debug("----> Name: %s\n", name_charp[i]); + debug("----> Content: %s\n", c_type_charp[i]); + debug("----> Weight: %s\n", weight_charp[i]); + debug("----> Keep rename: %d\n", params[i].keep_rename); + + } + + debug("Calling GS_ExecuteTaskNew...\n"); + /* + Finally, call bindings common with all the computed parameters + */ + GS_ExecuteTaskNew( + app_id, + signature, + on_failure, + time_out, + priority, + num_nodes, + reduce, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + num_pars, + &unrolled_parameters[0] // hide the fact that params is a std::vector + ); + debug("Returning from process_task...\n"); + Py_RETURN_NONE; +} + +/* + process http task +*/ +static PyObject* process_http_task(PyObject* self, PyObject* args) { + /* + Parse python object arguments and get pointers to them. + lsiiiiiiiOOOOOO must be read as + "long, string, integer, integer, integer, integer, integer, integer, + Object, Object, Object, Object, Object, Object" + */ + debug("Process http task:\n"); + long app_id; + char* signature; + char* on_failure; + int priority, num_nodes, reduce, chunk_size, replicated, distributed, has_target, num_returns, time_out; + PyObject *values; + PyObject *names; + PyObject *compss_types; + PyObject *compss_directions; + PyObject *compss_streams; + PyObject *compss_prefixes; + PyObject *content_types; + PyObject *weights; + PyObject *keep_renames; + // See comment from above for the meaning of this "magic" string + if(!PyArg_ParseTuple(args, "lssiiiiiiiiiOOOOOOOOO", &app_id, &signature, &on_failure, &time_out, &priority, &num_nodes, &reduce, + &chunk_size, &replicated, &distributed, &has_target, &num_returns, &values, &names, &compss_types, + &compss_directions, &compss_streams, &compss_prefixes, &content_types, &weights, &keep_renames)) { + // Return NULL after ParseTuple automatically translates to "wrong + // arguments were passed, we expected an integer instead"-like errors + return NULL; + } + debug("- App id: %ld\n", app_id); + debug("- Signature: %s\n", signature); + debug("- On Failure: %s\n", on_failure); + debug("- Time Out: %d\n", time_out); + debug("- Priority: %d\n", priority); + debug("- Reduce: %d\n", reduce); + debug("- Chunk size: %d\n", chunk_size); + debug("- MPI Num nodes: %d\n", num_nodes); + debug("- Replicated: %d\n", replicated); + debug("- Distributed: %d\n", distributed); + debug("- Has target: %d\n", has_target); + /* + Obtain and set all parameter data, and pack it in a struct vector. + + See parameter and its field at the top of this source code file + */ + Py_ssize_t num_pars = PyList_Size(values); + debug("Num pars: %d\n", int(num_pars)); + std::vector< parameter > params(num_pars); + std::vector< char* > prefix_charp(num_pars); + std::vector< char* > name_charp(num_pars); + std::vector< char* > c_type_charp(num_pars); + std::vector< char* > weight_charp(num_pars); + int num_fields = 9; + std::vector< void* > unrolled_parameters(num_fields * num_pars, NULL); + + for(int i = 0; i < num_pars; ++i) { + debug("Processing parameter %d ...\n", i); + PyObject *value = PyList_GetItem(values, i); + PyObject *type = PyList_GetItem(compss_types, i); + PyObject *direction = PyList_GetItem(compss_directions, i); + PyObject *stream = PyList_GetItem(compss_streams, i); + PyObject *prefix = PyList_GetItem(compss_prefixes, i); + PyObject *name = PyList_GetItem(names, i); + PyObject *c_type = PyList_GetItem(content_types, i); + PyObject *weight = PyList_GetItem(weights, i); + PyObject *k_rename = PyList_GetItem(keep_renames, i); + std::string received_prefix = _pystring_to_string(prefix); + std::string received_name = _pystring_to_string(name); + std::string received_c_type = _pystring_to_string(c_type); + std::string received_weight = _pystring_to_string(weight); + + params[i] = parameter( + value, + int(PyInt_AsLong(type)), + int(PyInt_AsLong(direction)), + int(PyInt_AsLong(stream)), + received_prefix, + _get_type_size(int(PyInt_AsLong(type))), + received_name, + received_c_type, + received_weight, + int(PyInt_AsLong(k_rename)) + ); + + debug("Adapting C++ data to BC-JNI format...\n"); + /* + Adapt the parsed data to a bindings-common friendly format. + These pointers do NOT need to be freed, as the pointed contents + are out of our control (will be scope-cleaned, or point to PyObject + contents) + */ + prefix_charp[i] = (char*) params[i].prefix.c_str(); + name_charp[i] = (char*) params[i].name.c_str(); + c_type_charp[i] = (char*) params[i].c_type.c_str(); + weight_charp[i] = (char*) params[i].weight.c_str(); + + debug("Processing parameter %d\n", i); + /* + Adapt the parsed data to a bindings-common friendly format. + These pointers do NOT need to be freed, as the pointed contents + are out of our control (will be scope-cleaned, or point to PyObject + contents) + */ + unrolled_parameters[num_fields * i + 0] = _get_void_pointer_to_content(params[i].value, params[i].type, params[i].size); + unrolled_parameters[num_fields * i + 1] = (void*) ¶ms[i].type; + unrolled_parameters[num_fields * i + 2] = (void*) ¶ms[i].direction; + unrolled_parameters[num_fields * i + 3] = (void*) ¶ms[i].stream; + unrolled_parameters[num_fields * i + 4] = (void*) &prefix_charp[i]; + unrolled_parameters[num_fields * i + 5] = (void*) &name_charp[i]; + unrolled_parameters[num_fields * i + 6] = (void*) &c_type_charp[i]; + unrolled_parameters[num_fields * i + 7] = (void*) &weight_charp[i]; + unrolled_parameters[num_fields * i + 8] = (void*) ¶ms[i].keep_rename; + + debug("----> Value is at %p\n", ¶ms[i].value); + debug("----> Type: %d\n", params[i].type); + debug("----> Direction: %d\n", params[i].direction); + debug("----> Stream: %d\n", params[i].stream); + debug("----> Prefix: %s\n", prefix_charp[i]); + debug("----> Size: %d\n", params[i].size); + debug("----> Name: %s\n", name_charp[i]); + debug("----> Content: %s\n", c_type_charp[i]); + debug("----> Weight: %s\n", weight_charp[i]); + debug("----> Keep rename: %d\n", params[i].keep_rename); + + } + + debug("Calling GS_ExecuteHttpTask...\n"); + /* + Finally, call bindings common with all the computed parameters + */ + GS_ExecuteHttpTask( + app_id, + signature, + on_failure, + time_out, + priority, + num_nodes, + reduce, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + num_pars, + &unrolled_parameters[0] // hide the fact that params is a std::vector + ); + debug("Returning from process_http_task...\n"); + Py_RETURN_NONE; +} + +/* + Given a PyCOMPSs-id file, check if it has been accessed before +*/ +static PyObject* accessed_file(PyObject* self, PyObject* args) { + debug("####C#### ACCESSED FILE\n"); + long app_id = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + char* file_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Accessed file? %s\n", (file_name)); + if (GS_Accessed_File(app_id, file_name) == 0) { + debug("- False\n"); + Py_RETURN_FALSE; + } else { + debug("- True\n"); + Py_RETURN_TRUE; + } +} + +/* + Given a PyCOMPSs-id file, get its corresponding COMPSs-id file. +*/ +static PyObject* open_file(PyObject* self, PyObject* args) { + debug("####C#### GET FILE\n"); + long app_id = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + char* file_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Open file %s\n", (file_name)); + int mode = int(PyInt_AsLong(PyTuple_GetItem(args, 2))); + debug("- mode: %i\n", (mode)); + char* compss_name; + GS_Open_File(app_id, file_name, mode, &compss_name); + debug("COMPSs file to open: %s\n", (compss_name)); + PyObject *ret = Py_BuildValue("s", compss_name); + // File_name must NOT be freed, as it points to a PyObject + // The same applies to compss_name + return ret; +} + +/* + Maybe the name is not the most appropriate one. + This function notifies the runtime that the current file will have one less + reader from now on. + Deletion is only performed when this file has no readers and no writers + AND someone asked to delete this file, so it may not happen + immediately after calling this function. +*/ +static PyObject* delete_file(PyObject* self, PyObject* args) { + debug("####C#### DELETE FILE\n"); + long app_id = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + char* file_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Delete file: %s\n", (file_name)); + bool wait = PyObject_IsTrue(PyTuple_GetItem(args, 2)); + debug("- Wait: %s\n", (wait ? "true" : "false")); + bool applicationDelete = PyObject_IsTrue(PyTuple_GetItem(args, 3)); + debug("- applicationDelete: %s\n", (applicationDelete ? "true" : "false")); + GS_Delete_File(app_id, file_name, wait, applicationDelete); + debug("COMPSs file deleted: %s\n", (file_name)); + Py_RETURN_NONE; +} + +/* + Closes the file, but this will never imply that this file will ever get deleted + (not at least with this call). + The difference in the meanings is the following: close_file only notifies + that you want to close the file, but you have no idea if this file will be + open again in the future, while delete_file implies knowledge that this + file is obsolete and it can be safely deleted. +*/ +static PyObject* close_file(PyObject* self, PyObject* args) { + long app_id = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + char* file_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Close file: %s\n", (file_name)); + int mode = int(PyInt_AsLong(PyTuple_GetItem(args, 2))); + debug("- Mode: %i\n", (mode)); + GS_Close_File(app_id, file_name, mode); + debug("File closed\n"); + Py_RETURN_NONE; +} + +/* + Given a PyCOMPSs-id file, get its corresponding last version COMPSs-id file +*/ +static PyObject* get_file(PyObject* self, PyObject* args) { + char* file_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Get file: %s\n", (file_name)); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + debug("- App id: %ld\n", (app_id)); + GS_Get_File(app_id, file_name); + debug("COMPSs file already get: %s\n", (file_name)); + Py_RETURN_NONE; +} + +/* + Given a PyCOMPSs-id directory, get its corresponding last version +*/ +static PyObject* get_directory(PyObject *self, PyObject *args) { + char* dir_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Get directory: %s\n", (dir_name)); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + debug("- App id: %ld\n", (app_id)); + GS_Get_Directory(app_id, dir_name); + debug("COMPSs directory already get: %s\n", (dir_name)); + Py_RETURN_NONE; +} + +/* + Notify the runtime that our current application wants to "execute" a barrier. + Program will be blocked in GS_BarrierNew until all running tasks have ended. + Notifies the 'no more tasks' boolean value. +*/ +static PyObject* barrier(PyObject* self, PyObject* args) { + debug("Barrier\n"); + long app_id; + PyObject* py_no_more_tasks; + bool no_more_tasks; + if(!PyArg_ParseTuple(args, "lO", &app_id, &py_no_more_tasks)) { + return NULL; + } + no_more_tasks = PyObject_IsTrue(py_no_more_tasks); // Verify that is a bool + debug("- App id: %ld \n", (app_id)); + debug("- No more tasks?: %s \n", (no_more_tasks ? "true" : "false")); + GS_BarrierNew(app_id, no_more_tasks); + debug("Barrier end\n"); + Py_RETURN_NONE; +} + +/* + Notify the runtime that our current application wants to "execute" a barrier for a group. + Program will be blocked in GS_BarrierGroup until all running tasks part of the group have ended. +*/ +static PyObject* barrier_group(PyObject* self, PyObject* args) { + char* group_name = _pystring_to_char(PyTuple_GetItem(args, 1)); + debug("Barrier group: %s\n", (group_name)); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + debug("- App id: %ld \n", (app_id)); + char* exception_message = NULL; + GS_BarrierGroup(app_id, group_name, &exception_message); + debug("Barrier group end: %s\n", (group_name)); + if (exception_message != NULL){ + PyObject* message_object = Py_BuildValue("s", exception_message); + debug("- COMPSs exception raised : %s \n", (exception_message)); + return message_object; + } else { + Py_RETURN_NONE; + } +} + +/* + Creates a new task group that will include all the subsequent tasks. +*/ +static PyObject* open_task_group(PyObject* self, PyObject* args) { + char *group_name = _pystring_to_char(PyTuple_GetItem(args, 0)); + debug("Open task group: %s\n", (group_name)); + bool implicit_barrier = PyObject_IsTrue(PyTuple_GetItem(args, 1)); + debug("- Implicit barrier?: %s \n", (implicit_barrier ? "true" : "false")); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 2))); + debug("- App id: %ld \n", (app_id)); + GS_OpenTaskGroup(group_name, implicit_barrier, app_id); + debug("Task group: %s created\n", (group_name)); + Py_RETURN_NONE; +} + +/* + Closes the opened task group. +*/ +static PyObject* close_task_group(PyObject* self, PyObject* args) { + char* group_name = _pystring_to_char(PyTuple_GetItem(args, 0)); + debug("Close task group: %s\n", (group_name)); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 1))); + debug("- App id: %ld \n", (app_id)); + GS_CloseTaskGroup(group_name, app_id); + debug("Task group: %s closed\n", (group_name)); + Py_RETURN_NONE; +} + +/* + Cancels a task group. +*/ +static PyObject* cancel_task_group(PyObject* self, PyObject* args) { + char* group_name = _pystring_to_char(PyTuple_GetItem(args, 0)); + debug("Cancel task group: %s\n", (group_name)); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 1))); + debug("- App id: %ld \n", (app_id)); + char* exception_message = NULL; + GS_CancelTaskGroup(group_name, app_id, &exception_message); + debug("Task group: %s cancelled\n", (group_name)); + debug("Barrier group end: %s\n", (group_name)); + if (exception_message != NULL) { + PyObject* message_object = Py_BuildValue("s", exception_message); + debug("- COMPSs exception raised : %s \n", (exception_message)); + return message_object; + } else { + Py_RETURN_NONE; + } + +} + +/* + Notify the runtime that our current application wants to "execute" a snapshot. +*/ +static PyObject* snapshot(PyObject* self, PyObject* args) { + debug("Snapshot\n"); + long app_id; + if(!PyArg_ParseTuple(args, "l", &app_id)) { + return NULL; + } + debug("- App id: %ld \n", (app_id)); + GS_Snapshot(app_id); + debug("Snapshot end\n"); + Py_RETURN_NONE; +} + +/* + Notify the event emission. +*/ +static PyObject* emit_event(PyObject *self, PyObject *args) { + int type = int(PyInt_AsLong(PyTuple_GetItem(args, 0))); + long value = long(PyInt_AsLong(PyTuple_GetItem(args, 1))); + debug("Emit Event: (%i, %ld)\n", type, value); + GS_EmitEvent(type, value); + Py_RETURN_NONE; +} + + +/* + Returns the logging path. +*/ +static PyObject* get_logging_path(PyObject* self, PyObject* args) { + debug("Get logs path\n"); + char* log_path; + GS_Get_AppDir(&log_path); + debug("- COMPSs log path %s\n", (log_path)); + // This makes log_path unallocatable + PyObject* ret = Py_BuildValue("s", log_path); + return ret; +} + +/* + Returns the master working path. +*/ +static PyObject* get_master_working_path(PyObject* self, PyObject* args) { + debug("Get master working path\n"); + char* master_working_path; + GS_Get_MasterWorkingDir(&master_working_path); + debug("- COMPSs master working path %s\n", (master_working_path)); + // This makes log_path unallocatable + PyObject* ret = Py_BuildValue("s", master_working_path); + return ret; +} + +/* + Requests the number of active resources to the runtime. +*/ +static PyObject* get_number_of_resources(PyObject* self, PyObject* args) { + debug("Get number of resources\n"); + long app_id; + if (!PyArg_ParseTuple(args, "l", &app_id)) { + return NULL; + } + debug("- App id: %ld\n", (app_id)); + int resources = GS_GetNumberOfResources(app_id); + debug("Number of resources: %i\n", (resources)); + PyObject* ret = Py_BuildValue("i", resources); + return ret; +} + +/* + Requests the runtime to increase a given number of resources. +*/ +static PyObject* request_resources(PyObject* self, PyObject* args) { + debug("Request resources creation\n"); + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + int num_resources = int(PyInt_AsLong(PyTuple_GetItem(args, 1))); + char* group_name = _pystring_to_char(PyTuple_GetItem(args, 2)); + + debug("- App id: %ld\n", (app_id)); + debug("- Number of resources: %i\n", (num_resources)); + debug("- Group name: %s\n", (group_name)); + + GS_RequestResources(app_id, num_resources, group_name); + Py_RETURN_NONE; +} + +/* + Requests the runtime to decrease a given number of resources. +*/ +static PyObject* free_resources(PyObject* self, PyObject* args) { + debug("Request resources destruction\n"); + + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + int num_resources = int(PyInt_AsLong(PyTuple_GetItem(args, 1))); + char *group_name = _pystring_to_char(PyTuple_GetItem(args, 2)); + + debug("- App id: %ld\n", (app_id)); + debug("- Number of resources: %i\n", (num_resources)); + debug("- Group name: %s\n", (group_name)); + + GS_FreeResources(app_id, num_resources, group_name); + Py_RETURN_NONE; +} + +/* + Requests the runtime to decrease a given number of resources. +*/ +static PyObject* set_wall_clock(PyObject* self, PyObject* args) { + debug("Setting wall clock limit\n"); + + long app_id = long(PyInt_AsLong(PyTuple_GetItem(args, 0))); + long wcl = long(PyInt_AsLong(PyTuple_GetItem(args, 1))); + + debug("- App id: %ld\n", (app_id)); + debug("- Number of resources: %ld\n", (wcl)); + + GS_Set_wall_clock(app_id, wcl, 0); + Py_RETURN_NONE; +} + +/* + Registers a new core element +*/ +static PyObject* register_core_element(PyObject* self, PyObject* args) { + debug("Register core element\n"); + char* CESignature; + char* ImplSignature; + char* ImplConstraints; + char* ImplType; + char* ImplLocal; + char* ImplIO; + PyObject* prolog; + PyObject* epilog; + PyObject* container; + PyObject* typeArgs; + if (!PyArg_ParseTuple(args, "ssssssOOOO", &CESignature, &ImplSignature, + &ImplConstraints, &ImplType, &ImplLocal, &ImplIO, &prolog, &epilog, &container, &typeArgs)) { + return NULL; + } + + debug("- Core Element Signature: %s\n", CESignature); + debug("- Implementation Signature: %s\n", ImplSignature); + debug("- Implementation Constraints: %s\n", ImplConstraints); + debug("- Implementation Type: %s\n", ImplType); + debug("- Implementation Local: %s\n", ImplLocal); + debug("- Implementation IO: %s\n", ImplIO); + char** pro = new char*[3]; + char** epi = new char*[3]; + pro[0] = _pystring_to_char(PyList_GetItem(prolog, 0)); + pro[1] = _pystring_to_char(PyList_GetItem(prolog, 1)); + pro[2] = _pystring_to_char(PyList_GetItem(prolog, 2)); + epi[0] = _pystring_to_char(PyList_GetItem(epilog, 0)); + epi[1] = _pystring_to_char(PyList_GetItem(epilog, 1)); + epi[2] = _pystring_to_char(PyList_GetItem(epilog, 2)); + debug("- Prolog: %s %s\n", pro[0], pro[1]); + debug("- Epilog: %s %s\n", epi[0], epi[1]); + + char** cont = new char*[3]; + cont[0] = _pystring_to_char(PyList_GetItem(container, 0)); + cont[1] = _pystring_to_char(PyList_GetItem(container, 1)); + cont[2] = _pystring_to_char(PyList_GetItem(container, 2)); + debug("- Container: %s %s\n", cont[0], cont[1], cont[2]); + + int num_params = PyList_Size(typeArgs); + debug("- Implementation Type num args: %i\n", num_params); + char** ImplTypeArgs = new char*[num_params]; + for(int i = 0; i < num_params; ++i) { + ImplTypeArgs[i] = _pystring_to_char(PyList_GetItem(typeArgs, i)); + debug("- Implementation Type Args: %s\n", ImplTypeArgs[i]); + } + // Invoke the C library + GS_RegisterCE(CESignature, + ImplSignature, + ImplConstraints, + ImplType, + ImplLocal, + ImplIO, + pro, + epi, + cont, + num_params, + ImplTypeArgs); + debug("Core element registered\n"); + // Free all allocated memory + delete ImplTypeArgs; + Py_RETURN_NONE; +} + +/* + Method definition, generic argument speficication, and __doc__ field value +*/ +static PyMethodDef CompssMethods[] = { + { "error_out", (PyCFunction)error_out, METH_NOARGS, NULL}, + { "set_debug", set_debug, METH_VARARGS, "Set debug mode." }, + { "start_runtime", start_runtime, METH_VARARGS, "Start the COMPSs runtime." }, + { "stop_runtime", stop_runtime, METH_VARARGS, "Stop the COMPSs runtime." }, + { "cancel_application_tasks", cancel_application_tasks, METH_VARARGS, "Cancel all tasks of an application." }, + { "process_task", process_task, METH_VARARGS, "Process a task call from the application." }, + { "process_http_task", process_http_task, METH_VARARGS, "Process an http task call from the application." }, + { "accessed_file", accessed_file, METH_VARARGS, "Check if a file has been already accessed. The file can contain an object." }, + { "open_file", open_file, METH_VARARGS, "Get a file for opening. The file can contain an object." }, + { "delete_file", delete_file, METH_VARARGS, "Delete a file." }, + { "close_file", close_file, METH_VARARGS, "Close a file." }, + { "get_file", get_file, METH_VARARGS, "Get last version of file with its original name." }, + { "get_directory", get_directory, METH_VARARGS, "Get last version of a directory with its original name." }, + { "barrier", barrier, METH_VARARGS, "Perform a barrier until the tasks already submitted have finished." }, + { "barrier_group", barrier_group, METH_VARARGS, "Barrier for a task group." }, + { "open_task_group", open_task_group, METH_VARARGS, "Opens a new task group." }, + { "close_task_group", close_task_group, METH_VARARGS, "Closes a new task group." }, + { "cancel_task_group", cancel_task_group, METH_VARARGS, "Cancels a new task group." }, + { "snapshot", snapshot, METH_VARARGS, "Perform a snapshot of the tasks and data." }, + { "get_logging_path", get_logging_path, METH_VARARGS, "Requests the app log path." }, + { "get_master_working_path", get_master_working_path, METH_VARARGS, "Requests the master working path." }, + { "get_number_of_resources", get_number_of_resources, METH_VARARGS, "Requests the number of active resources." }, + { "request_resources", request_resources, METH_VARARGS, "Requests the creation of a new resource."}, + { "free_resources", free_resources, METH_VARARGS, "Requests the destruction of a resource."}, + { "register_core_element", register_core_element, METH_VARARGS, "Registers a task in the Runtime." }, + { "emit_event", emit_event, METH_VARARGS, "Emit a event in the API Thread." }, + { "set_pipes", set_pipes, METH_VARARGS, "Set compss module to pipe comunication mode." }, + { "read_pipes", read_pipes, METH_VARARGS, "Reads a command using the pipe comunication mode." }, + { "set_wall_clock" , set_wall_clock, METH_VARARGS, "Set the application wall clock limit."}, + { NULL, NULL } /* sentinel */ +}; + +extern "C" { + + static int compss_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; + } + static int compss_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; + } + static struct PyModuleDef cModPy = { + PyModuleDef_HEAD_INIT, + "compss", /* name of module */ + NULL, /* module documentation, may be NULL */ + sizeof(struct module_state), /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */ + CompssMethods, + NULL, + compss_traverse, + compss_clear, + NULL + }; + #define INITERROR return NULL + PyMODINIT_FUNC PyInit_compss(void) { + PyObject *module = PyModule_Create(&cModPy); + + if (module == NULL) + INITERROR; + struct module_state *st = GETSTATE(module); + + st->error = PyErr_NewException("compss.Error", NULL, NULL); + if (st->error == NULL) { + Py_DECREF(module); + INITERROR; + } + + return module; + } +}; diff --git a/examples/dds/pycompss/ext/dlb_affinity.c b/examples/dds/pycompss/ext/dlb_affinity.c new file mode 100644 index 00000000..8fda734d --- /dev/null +++ b/examples/dds/pycompss/ext/dlb_affinity.c @@ -0,0 +1,307 @@ +/* + * Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include + +static PyObject * dlb_init(PyObject *self, PyObject *args) { + cpu_set_t mask; + CPU_ZERO(&mask); + int err = DLB_Init(0, &mask, 0); + //int err = DLB_Init(0, NULL, "--drom --lewi --ompt"); + if (err == DLB_SUCCESS) printf("\nInitializing DLB DROM\n"); + else printf("\nError initializing DLB DROM: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_finalize(PyObject *self, PyObject *args) { + printf("\nFinalizing DLB DROM\n"); + DLB_Finalize(); + Py_RETURN_NONE; +} + +static PyObject * dlb_pollDROM_Update(PyObject *self, PyObject *args) { + DLB_PollDROM_Update(); + Py_RETURN_NONE; +} + +static PyObject * dlb_attach(PyObject *self, PyObject *args) { + int err = DLB_DROM_Attach(); + Py_RETURN_NONE; +} + +static PyObject * dlb_detach(PyObject *self, PyObject *args) { + int err = DLB_DROM_Detach(); + if (err == DLB_SUCCESS) printf("\ndetached\n"); + else printf("\nerror ataching: %d\n", err); + Py_RETURN_NONE; +} + + +/* Lend */ + +static PyObject * dlb_lend(PyObject *self, PyObject *args) { + int err = DLB_Lend(); + if (err == DLB_SUCCESS) printf("\nlent\n"); + else printf("\nerror lending: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_lendCpu(PyObject *self, PyObject *args) { + int cpuid = 0; + if (!PyArg_ParseTuple(args, "i", &cpuid)) + return NULL; + int err = DLB_LendCpu(cpuid); + if (err == DLB_SUCCESS) printf("\nlent\n"); + else printf("\nerror lending: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_lendCpus(PyObject *self, PyObject *args) { + int ncpus = 0; + if (!PyArg_ParseTuple(args, "i", &ncpus)) + return NULL; + int err = DLB_LendCpus(ncpus); + if (err == DLB_SUCCESS) printf("\nlent\n"); + else printf("\nerror lending: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_lendCpuMask(PyObject *self, PyObject *args) { + int i; + PyObject* cpu_list; + if (!PyArg_ParseTuple(args, "O", &cpu_list)) + return NULL; + + cpu_set_t mask; + CPU_ZERO(&mask); + int num_params = PyList_Size(cpu_list); + for (i = 0; i < num_params; ++i) { + int cpu_id = PyLong_AsLong(PyList_GetItem(cpu_list, i)); + CPU_SET(cpu_id, &mask); + } + int err = DLB_LendCpuMask(&mask); + if (err == DLB_SUCCESS) printf("\nlent\n"); + else printf("\nerror lending: %d\n", err); + Py_RETURN_NONE; +} + + +/* Acquire */ + +static PyObject * dlb_acquireCpu(PyObject *self, PyObject *args) { + int cpuid = 0; + if (!PyArg_ParseTuple(args, "i", &cpuid)) + return NULL; + int err = DLB_AcquireCpu(cpuid); + if (err == DLB_SUCCESS) printf("\nacquired\n"); + else printf("\nerror acquiring: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_acquireCpus(PyObject *self, PyObject *args) { + int ncpus = 0; + if (!PyArg_ParseTuple(args, "i", &ncpus)) + return NULL; + int err = DLB_AcquireCpus(ncpus); + if (err == DLB_SUCCESS) printf("\nacquired\n"); + else printf("\nerror acquiring: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_acquireCpuMask(PyObject *self, PyObject *args) { + int i; + PyObject* cpu_list; + if (!PyArg_ParseTuple(args, "O", &cpu_list)) + return NULL; + + cpu_set_t mask; + CPU_ZERO(&mask); + int num_params = PyList_Size(cpu_list); + for (i = 0; i < num_params; ++i) { + int cpu_id = PyLong_AsLong(PyList_GetItem(cpu_list, i)); + CPU_SET(cpu_id, &mask); + } + int err = DLB_AcquireCpuMask(&mask); + if (err == DLB_SUCCESS) printf("\nacquired\n"); + else printf("\nerror acquiring: %d\n", err); + Py_RETURN_NONE; +} + + +/* Borrow */ + +static PyObject * dlb_borrow(PyObject *self, PyObject *args) { + int err = DLB_Borrow(); + if (err == DLB_SUCCESS) printf("\nborrowed\n"); + else printf("\nerror borrowing: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_borrowCpu(PyObject *self, PyObject *args) { + int cpuid = 0; + if (!PyArg_ParseTuple(args, "i", &cpuid)) + return NULL; + int err = DLB_BorrowCpu(cpuid); + if (err == DLB_SUCCESS) printf("\nborrowed\n"); + else printf("\nerror borrowing: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_borrowCpus(PyObject *self, PyObject *args) { + int ncpus = 0; + if (!PyArg_ParseTuple(args, "i", &ncpus)) + return NULL; + int err = DLB_BorrowCpus(ncpus); + if (err == DLB_SUCCESS) printf("\nborrowed\n"); + else printf("\nerror borrowing: %d\n", err); + Py_RETURN_NONE; +} + +static PyObject * dlb_borrowCpuMask(PyObject *self, PyObject *args) { + int i; + PyObject* cpu_list; + if (!PyArg_ParseTuple(args, "O", &cpu_list)) + return NULL; + + cpu_set_t mask; + CPU_ZERO(&mask); + int num_params = PyList_Size(cpu_list); + for (i = 0; i < num_params; ++i) { + int cpu_id = PyLong_AsLong(PyList_GetItem(cpu_list, i)); + CPU_SET(cpu_id, &mask); + } + int err = DLB_BorrowCpuMask(&mask); + if (err == DLB_SUCCESS) printf("\nborrowed\n"); + else printf("\nerror borrowing: %d\n", err); + Py_RETURN_NONE; +} + + +/* Get_Affinity */ + +static PyObject * dlb_getaffinity(PyObject *self, PyObject *args) { + int i, ii; + long long pid = 0ll; + + if (!PyArg_ParseTuple(args, "|l", &pid)) + return NULL; + + if (pid == 0ll) pid = getpid(); + cpu_set_t mask; + CPU_ZERO(&mask); + int err = DLB_DROM_GetProcessMask(pid, &mask, 0); + if (err == DLB_SUCCESS) + printf("\n(%d) getmask ok\n", pid);//, __CPU_SETSIZE); + else + printf("\n(%d) getmask fail: %d\n", pid, err); + + int size = 0; + for (i = 0; i < __CPU_SETSIZE; ++i) + if (CPU_ISSET(i, &mask)) size++; + + PyObject* py_ret = PyList_New(size); + for (i = 0, ii = 0; i < __CPU_SETSIZE; ++i) + if (CPU_ISSET(i, &mask)) + PyList_SetItem(py_ret, ii++, Py_BuildValue("i", i)); + + return py_ret; +} + + +/* Set_Affinity */ + +static PyObject * dlb_setaffinity(PyObject *self, PyObject *args) { + int i; + long long pid = 0ll; + PyObject* cpu_list; + + if (!PyArg_ParseTuple(args, "O|l", &cpu_list, &pid)) + return NULL; + + cpu_set_t mask; + CPU_ZERO(&mask); + int num_params = PyList_Size(cpu_list); + for (i = 0; i < num_params; ++i) { + int cpu_id = PyLong_AsLong(PyList_GetItem(cpu_list, i)); + CPU_SET(cpu_id, &mask); + } + int err = DLB_DROM_SetProcessMask(pid, &mask, 0); + if (err == DLB_SUCCESS) + printf("\n(%d) setmask ok\n", pid); + else + printf("\n(%d) setmask fail: %d\n", pid, err); + + Py_RETURN_NONE; +} + +static PyMethodDef DlbAffinityMethods[] = { + + {"init", dlb_init, METH_NOARGS, "Initialize DLB library and all its internal data structures."}, + + {"finalize", dlb_finalize, METH_NOARGS, "Finalize DLB library and clean up all its data structures."}, + + {"pollDROM_Update", dlb_pollDROM_Update, METH_NOARGS, "Poll DROM module to check if the process needs to adapt to a new mask or number of CPUs."}, + + {"attach", dlb_attach, METH_NOARGS, "Attach process to DLB as third party."}, + + {"detach", dlb_detach, METH_NOARGS, "Detach process from DLB."}, + + {"lend", dlb_lend, METH_NOARGS, "Lend all the current CPUs."}, + + {"lendCpu", dlb_lendCpu, METH_VARARGS, "Args: (cpuid) -> Lend a specific CPU."}, + + {"lendCpus", dlb_lendCpus, METH_VARARGS, "Args: (ncpus) -> Lend a specific amount of CPUs, only useful for systems that do not work with cpu masks."}, + + {"lendCpuMask", dlb_lendCpuMask, METH_VARARGS, "Args: (mask) -> Lend a set of CPUs."}, + + {"acquireCpu", dlb_acquireCpu, METH_VARARGS, "Args: (cpuid) -> Acquire a specific CPU."}, + + {"acquireCpus", dlb_acquireCpus, METH_VARARGS, "Args: (ncpu) -> Acquire a specific amount of CPUs."}, + + {"acquireCpuMask", dlb_acquireCpuMask, METH_VARARGS, "Args: (mask) -> Acquire a set of CPUs."}, + + {"borrow", dlb_borrow, METH_NOARGS, "Borrow all the possible CPUs registered on DLB."}, + + {"borrowCpu", dlb_borrowCpu, METH_VARARGS, "Args: (cpuid) -> Borrow a specific CPU."}, + + {"borrowCpus", dlb_borrowCpus, METH_VARARGS, "Args: (ncpus) -> Borrow a specific amount of CPUs."}, + + {"borrowCpuMask", dlb_borrowCpuMask, METH_VARARGS, "Args: (mask) -> Borrow a set of CPUs."}, + + {"getaffinity", dlb_getaffinity, METH_VARARGS, "Args: (mask, [pid=0]) -> Returns the process mask of the given PID."}, + + {"setaffinity", dlb_setaffinity, METH_VARARGS, "Args: ([pid=0]) -> Set the process mask of the given PID."}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef dlbmodule = { + PyModuleDef_HEAD_INIT, + "dlb_thread_affinity", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */ + DlbAffinityMethods +}; + +PyMODINIT_FUNC PyInit_dlb_affinity(void) +{ + return PyModule_Create(&dlbmodule); +} diff --git a/examples/dds/pycompss/ext/ear.c b/examples/dds/pycompss/ext/ear.c new file mode 100644 index 00000000..2a6b4d30 --- /dev/null +++ b/examples/dds/pycompss/ext/ear.c @@ -0,0 +1,47 @@ +/* + * Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include + + +static PyObject * ear_finalize(PyObject *self, PyObject *args) { + printf("\nFinalizing EAR - Calling destructor\n"); + ear_destructor(); + Py_RETURN_NONE; +} + + +static PyMethodDef EARMethods[] = { + {"finalize", ear_finalize, METH_NOARGS, "Finalize EAR library and clean up all its data structures."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef earmodule = { + PyModuleDef_HEAD_INIT, + "ear", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */ + EARMethods +}; + +PyMODINIT_FUNC PyInit_ear(void) +{ + return PyModule_Create(&earmodule); +} diff --git a/examples/dds/pycompss/ext/process_affinity.cc b/examples/dds/pycompss/ext/process_affinity.cc new file mode 100644 index 00000000..df5270d1 --- /dev/null +++ b/examples/dds/pycompss/ext/process_affinity.cc @@ -0,0 +1,128 @@ +/* + * Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "process_affinity.h" + +struct module_state { + PyObject *error; +}; + +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#define PyInt_AsLong PyLong_AsLong + +static PyObject * error_out(PyObject *m) { + struct module_state *st = GETSTATE(m); + PyErr_SetString(st->error, "Thread affinity extension module: Something bad happened"); + return NULL; +} + +////////////////////////////////////////////////////////////// + +cpu_set_t default_affinity; + +static PyObject* pysched_setaffinity(PyObject* self, PyObject* args) { + long long pid = 0ll; + PyObject* cpu_list; + if(!PyArg_ParseTuple(args, "O|l", &cpu_list, &pid)) { + return NULL; + } + cpu_set_t to_assign; + CPU_ZERO(&to_assign); + int num_params = PyList_Size(cpu_list); + for(int i = 0; i < num_params; ++i) { + int cpu_id = PyInt_AsLong(PyList_GetItem(cpu_list, i)); + CPU_SET(cpu_id, &to_assign); + } + if(sched_setaffinity(pid, sizeof(cpu_set_t), &to_assign) < 0) { + if(sched_setaffinity(pid, sizeof(cpu_set_t), &default_affinity) < 0) { + PyErr_SetString(PyExc_RuntimeError, "Cannot set default affinity (!)"); + } else { + PyErr_SetString(PyExc_RuntimeError, "setaffinity failed, setting default affinity"); + } + } + Py_RETURN_NONE; +} + +static PyObject* pysched_getaffinity(PyObject* self, PyObject* args) { + long long pid = 0ll; + if(!PyArg_ParseTuple(args, "|l", &pid)) { + return NULL; + } + if(pid == 0ll) pid = getpid(); + cpu_set_t set_cpus; + if(sched_getaffinity(pid, sizeof(cpu_set_t), &set_cpus) < 0) { + PyErr_SetString(PyExc_RuntimeError, "Error during sched_getaffinity call!"); + Py_RETURN_NONE; + } + std::vector< int > ret_val; + for(int i = 0; i < __CPU_SETSIZE; ++i) { + if(CPU_ISSET(i, &set_cpus)) ret_val.push_back(i); + } + PyObject* py_ret = PyList_New(int(ret_val.size())); + for(int i = 0; i < int(ret_val.size()); ++i) { + PyList_SetItem(py_ret, i, Py_BuildValue("i", ret_val[i])); + } + return py_ret; +} + +////////////////////////////////////////////////////////////// + + +static PyMethodDef ThreadAffinityMethods[] = { + { "error_out", (PyCFunction)error_out, METH_NOARGS, NULL}, + { "setaffinity", pysched_setaffinity, METH_VARARGS, "Args: (mask, [pid=0]) -> set the affinity for the thread with given pid to given mask. If pid equals zero, then the current thread's affinity will be changed." }, + { "getaffinity", pysched_getaffinity, METH_VARARGS, "Args: ([pid=0]) -> returns the affinity for the thread with given pid. If not specified, returns the affinity for the current thread."}, + { NULL, NULL } /* sentinel */ +}; + + +static int process_affinity_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} +static int process_affinity_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} +static struct PyModuleDef cModThAPy = { + PyModuleDef_HEAD_INIT, + "process_affinity", /* name of module */ + NULL, /* module documentation, may be NULL */ + sizeof(struct module_state), /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */ + ThreadAffinityMethods, + NULL, + process_affinity_traverse, + process_affinity_clear, + NULL +}; +#define INITERROR return NULL +PyMODINIT_FUNC PyInit_process_affinity(void) { + PyObject *module = PyModule_Create(&cModThAPy); + + if (module == NULL) + INITERROR; + struct module_state *st = GETSTATE(module); + + st->error = PyErr_NewException("process_affinity.Error", NULL, NULL); + if (st->error == NULL) { + Py_DECREF(module); + INITERROR; + } + + sched_getaffinity(0, sizeof(cpu_set_t), &default_affinity); + + return module; +} diff --git a/examples/dds/pycompss/ext/process_affinity.h b/examples/dds/pycompss/ext/process_affinity.h new file mode 100644 index 00000000..eed1539e --- /dev/null +++ b/examples/dds/pycompss/ext/process_affinity.h @@ -0,0 +1,51 @@ +/* + * Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * Wrappers that make possible to call thread (set|get)affinity from Python2 + */ +#pragma once +#include +#include +#include +#include +#include + +/* + Wrapper for sched_setaffinity. + Arguments: + - mask: a list of integers that denote the CPU identifiers (0-based) that we + want to allow + - pid: if zero, this will be transformed to the current pid + Returns None +*/ +static PyObject* pysched_setaffinity(PyObject* self, PyObject* args); + + +/* + Wrapper for sched_getaffinity. + Arguments: + - pid (OPTIONAL): if zero or ommited, this will be transformed to the current pid + Returns the list of allowed CPUs +*/ +static PyObject* pysched_getaffinity(PyObject* self, PyObject* args); + + +extern "C" { + PyMODINIT_FUNC + PyInit_process_affinity(void); +} diff --git a/examples/dds/pycompss/functions/__init__.py b/examples/dds/pycompss/functions/__init__.py new file mode 100644 index 00000000..97e11bb0 --- /dev/null +++ b/examples/dds/pycompss/functions/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains common functions that can be used by users.""" diff --git a/examples/dds/pycompss/functions/data.py b/examples/dds/pycompss/functions/data.py new file mode 100644 index 00000000..53617162 --- /dev/null +++ b/examples/dds/pycompss/functions/data.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Data generators. + +This file defines the common data producing functions. +""" + +from pycompss.functions.data_tasks import gen_normal as _gen_normal +from pycompss.functions.data_tasks import gen_random as _gen_random +from pycompss.functions.data_tasks import gen_uniform as _gen_uniform +from pycompss.util.typing_helper import typing + + +def generator( + size: typing.Tuple[int, int], + num_frag: int, + seed: int = 0, + distribution: str = "random", + wait: bool = False, +) -> typing.Optional[typing.List[typing.Any]]: + """Generate a list of fragments with random data. + + :param size: Size (numElements, dim) + :param num_frag: Dataset number of fragments + :param seed: Random seed. Default None, system time is used- + :param distribution: Random, normal, uniform + :param wait: If we want to wait for result. Default False + :return: Random dataset + """ + data = None + frag_size = int(size[0] / num_frag) + if distribution == "random": + data = [_gen_random(size[1], frag_size, seed) for _ in range(num_frag)] + elif distribution == "normal": + data = [_gen_normal(size[1], frag_size, seed) for _ in range(num_frag)] + elif distribution == "uniform": + data = [ + _gen_uniform(size[1], frag_size, seed) for _ in range(num_frag) + ] + if wait: + from pycompss.api.api import ( # pylint: disable=C0415 + # disable=import-outside-toplevel + compss_wait_on, + ) + + data = compss_wait_on(data) + return data + + +def chunks( + lst: list, number: int, balanced: bool = False +) -> typing.Iterator[typing.List[int]]: + """List splitter into fragments. + + WARNING: Not tested! + + :param lst: List of data to be chunked + :param number: length of the fragments + :param balanced: True to generate balanced fragments + :return: yield fragments of size n from lst + """ + if not balanced or not len(lst) % number: + for i in range(0, len(lst), number): + yield lst[i : i + number] # noqa: E203 + else: + rest = len(lst) % number + start = 0 + while rest: + yield lst[start : start + number + 1] # noqa: E203 + rest -= 1 + start += number + 1 + for i in range(start, len(lst), number): + yield lst[i : i + number] # noqa: E203 diff --git a/examples/dds/pycompss/functions/data_tasks.py b/examples/dds/pycompss/functions/data_tasks.py new file mode 100644 index 00000000..090af9c9 --- /dev/null +++ b/examples/dds/pycompss/functions/data_tasks.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Data generators. + +This file defines the common data producing functions. + +WARNING: This file can not be compiled with mypyc since contains tasks. +""" + +import random + +from pycompss.api.task import task +from pycompss.util.typing_helper import typing + + +@task(returns=list) +def gen_random( + size: int, frag_size: int, seed: int +) -> typing.List[typing.List[float]]: + """Generate random distribution fragment. + + :param size: Size + :param frag_size: Fragment size + :param seed: Random seed + :return: a fragment of elements + """ + random.seed(seed) + return [[random.random() for _ in range(size)] for _ in range(frag_size)] + + +@task(returns=list) +def gen_normal( + size: int, frag_size: int, seed: int +) -> typing.List[typing.List[float]]: + """Generate normal distribution fragment. + + :param size: Size + :param frag_size: Fragment size + :param seed: Random seed + :return: a fragment of elements + """ + random.seed(seed) + return [ + [random.gauss(mu=0.0, sigma=1.0) for _ in range(size)] + for _ in range(frag_size) + ] + + +@task(returns=list) +def gen_uniform( + size: int, frag_size: int, seed: int +) -> typing.List[typing.List[float]]: + """Generate uniform distribution fragment. + + :param size: Size + :param frag_size: Fragment size + :param seed: Random seed + :return: a fragment of elements + """ + random.seed(seed) + return [ + [random.uniform(-1.0, 1.0) for _ in range(size)] + for _ in range(frag_size) + ] diff --git a/examples/dds/pycompss/functions/elapsed_time.py b/examples/dds/pycompss/functions/elapsed_time.py new file mode 100644 index 00000000..4aee601b --- /dev/null +++ b/examples/dds/pycompss/functions/elapsed_time.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Elapsed time decorator. + +This file defines the time it decorator to be used over the task decorator. +""" + +import time +from functools import wraps + +from pycompss.util.typing_helper import typing + + +class TimeIt: + """TimeIT decorator class.""" + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Store arguments passed to the decorator. + + :param args: Arguments. + :param kwargs: Keyword arguments. + """ + self.args = args + self.kwargs = kwargs + + def __call__(self, function: typing.Any) -> typing.Any: + """Elapsed time decorator. + + :param f: Function to be time measured (can be a decorated function, + usually with @task decorator). + :return: the decorator wrapper. + """ + + @wraps(function) + def wrapped_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """Elapsed time decorator body. + + :param args: args + :param kwargs: kwargs + :return: a list with [the function result, The elapsed time] + """ + start_time = time.time() + result = function(*args, **kwargs) + end_time = time.time() + return [result, (end_time - start_time)] + + return wrapped_f + + def __str__(self) -> str: + """Represent the elapsed time decorator as string.""" + return "Elapsed time decorator object" + + +# ########################################################################### # +# ################### TimeIT DECORATOR ALTERNATIVE NAMES #################### # +# ########################################################################### # + +timeit = TimeIt # pylint: disable=invalid-name +TimeIT = TimeIt +TIMEIT = TimeIt diff --git a/examples/dds/pycompss/functions/map.py b/examples/dds/pycompss/functions/map.py new file mode 100644 index 00000000..b2b8aa24 --- /dev/null +++ b/examples/dds/pycompss/functions/map.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Map. + +This file defines the common map functions. +""" + +from pycompss.util.typing_helper import typing + + +def __get_chunks( + iterable: typing.Union[list, tuple], chunksize: int +) -> typing.Iterator[typing.Union[list, tuple]]: + """Yield n-sized chunks from iterable. + + :param iterable: Iterable of items to be reduced. + :param chunksize: Elements to consider in a chunk. + :return: List of lists that contain chunksize elements. + """ + for i in range(0, len(list(iterable)), chunksize): + yield iterable[i : i + chunksize] # noqa: E203 + + +def map( # pylint: disable=redefined-builtin + func: typing.Callable, + iterable: typing.Union[list, tuple], + chunksize: typing.Optional[int] = None, +) -> typing.Union[list, tuple]: + """Apply function cumulatively to the items of data. + + Provides the multiprocessing pool map interface. + + :param func: function to apply to reduce data. + :param iterable: Iterable of items to be reduced. + :param chunksize: Elements to consider in a chunk. + :return: result of applying func to all iterable elements. + """ + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_wait_on, + ) + + result = [] + if chunksize: + for i in __get_chunks(iterable, chunksize): + result.append(func(i)) + else: + for i in iterable: + result.append(func(i)) + result = compss_wait_on(result) + return result diff --git a/examples/dds/pycompss/functions/profile.py b/examples/dds/pycompss/functions/profile.py new file mode 100644 index 00000000..c411e5f0 --- /dev/null +++ b/examples/dds/pycompss/functions/profile.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Profiling decorator. + +This file defines the profiling decorator to be used below the task decorator. +""" + +import os +import sys +import time +from functools import wraps +from io import StringIO + +import psutil +from memory_profiler import profile as mem_profile +from pycompss.util.typing_helper import typing +from pycompss.util.context import CONTEXT + +if __debug__: + import logging + + LOGGER = logging.getLogger(__name__) + +# Global environment variable name +__PROFILE_REDIRECT_ENV_VAR__ = "COMPSS_PROFILING_FILE" + +FD_PATH = "/proc/self/fd/" +if sys.platform == "darwin": + FD_PATH = "/dev/fd/" + + +class Profile: + """Profile decorator class.""" + + __slots__ = ("stream", "full_report", "precision", "backend", "redirect") + + def __init__( + self, + full_report: bool = True, + precision: int = 1, + backend: str = "psutil", + ) -> None: + """Store arguments passed to the decorator. + + :param full_report: If provide the full memory usage report. + :param precision: Memory usage precision. + :param backend: Memory measure backend. + """ + self.stream = StringIO() + self.full_report = full_report + self.precision = precision + self.backend = backend + if __PROFILE_REDIRECT_ENV_VAR__ in os.environ: + self.redirect = os.environ[__PROFILE_REDIRECT_ENV_VAR__] + else: + self.redirect = "" + + def __call__(self, function: typing.Callable) -> typing.Any: + """Memory profiler decorator. + + :param function: Function to be profiled (can be a decorated function, + usually with @task decorator). + :return: the decorator wrapper. + """ + + @wraps(function) + def wrapped_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """Memory profiler decorator. + + :param args: Task arguments. + :param kwargs: Task keyword arguments. + :return: The result of executing function with the given *args and + **kwargs. + """ + # Get job name from log file + if CONTEXT.in_master(): + job_name = "master" + else: + stdout_fd = sys.stdout.fileno() + job_log_file_path = os.readlink(f"{FD_PATH}{stdout_fd}") + job_log_file_name = os.path.basename(job_log_file_path) + job_name = job_log_file_name.split(".")[0] + # Get initial memory usage + initial_memory = psutil.virtual_memory().used + # Get start time stamp + start_time = time.time() + # Run the user code + result = mem_profile( + func=function, + stream=self.stream, + precision=self.precision, + backend=self.backend, + )(*args, **kwargs) + # Get elapsed time + elapsed_time = time.time() - start_time + # Get final memory usage + final_memory = psutil.virtual_memory().used + # Report before returning the result. + report = self.stream.getvalue().strip() + # Reset the stream StringIO + self.stream.truncate(0) + self.stream.seek(0) + self.__print_report__( + start_time, + job_name, + function.__name__, + report, + initial_memory, + final_memory, + elapsed_time, + ) + return result + + return wrapped_f + + def __print_report__( # pylint: disable=R0913,R0914 + # disable=too-many-arguments, too-many-locals + self, + start_time: float, + job_name: str, + function_name: str, + report: str, + initial_memory: int, + final_memory: int, + elapsed_time: float, + ) -> None: + """Show the memory usage report. + + :param start_time: Task start time. + :param job_name: Job name. + :param function_name: Function name. + :param report: Memory usage report. + :param initial_memory: Memory used before the task is executed. + :param final_memory: Memory used after the task is executed. + :param elapsed_time: Task elapsed time. + :return: None + """ + if self.full_report: + job_name = f"Job name: {job_name}" + st_time = f"Task start time: {start_time}" + el_time = f"Elapsed time: {elapsed_time}" + pre_mem = f"Initial memory: {initial_memory}" + post_mem = f"Final memory: {final_memory}" + report_info = ( + f"{report}\n" + f"{job_name}\n" + f"{st_time}\n" + f"{el_time}\n" + f"{pre_mem}\n" + f"{post_mem}" + ) + self.__redirect__(report_info) + else: + report_lines = report.splitlines() + peak_memory = 0.0 + file_name = report_lines[0].split()[1] + for line in report_lines[4:]: + # 4: removes the header lines + info = line.split() + has_mem = True + try: + usage = float(info[1].replace(",", "")) + except ValueError: + has_mem = False + usage = 0.0 + if len(info) > 5 and has_mem: + # Is a full line with memory usage + if usage > peak_memory: + peak_memory = usage + mem_usage = f"{initial_memory} {final_memory} {peak_memory} MiB" + info_line = ( + f"{start_time:.7f} {job_name} {file_name} {function_name}" + ) + info_line = f"{info_line} {elapsed_time} {mem_usage}" + self.__redirect__(info_line) + # Clear the stream for the next usage + self.stream.truncate(0) + + def __redirect__(self, info: str) -> None: + """Redirect the given info to the required output file or std. + + :param info: Information to redirect. + :return: None + """ + if self.redirect == "": + print(info) + else: + with open(self.redirect, "a", encoding="utf-8") as file_descriptor: + file_descriptor.write(f"{info}\n") + file_descriptor.flush() + + def __del__(self) -> None: + """Destructor. + + Closes the internal stream where the profiling is being redirected. + """ + self.stream.close() + + +# ########################################################################### # +# ################### Profile DECORATOR ALTERNATIVE NAME #################### # +# ########################################################################### # + +profile = Profile # pylint: disable=invalid-name +PROFILE = Profile diff --git a/examples/dds/pycompss/functions/reduce.py b/examples/dds/pycompss/functions/reduce.py new file mode 100644 index 00000000..d517910a --- /dev/null +++ b/examples/dds/pycompss/functions/reduce.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Functions: Reduce. + +This file defines the common reduction functions. +""" + +from collections import deque + +from pycompss.util.typing_helper import typing + + +def merge_reduce(function: typing.Callable, data: list) -> typing.Any: + """Reduce the given data applying f as an inverted binary tree. + + Apply function cumulatively to the items of data, from left to right in + binary tree structure, so as to reduce the data to a single value. + + :param function: function to apply to reduce data + :param data: List of items to be reduced + :return: result of reduce the data to a single value + """ + queue = deque(range(len(data))) + while len(queue): + x_value = queue.popleft() + if len(queue): + y_value = queue.popleft() + data[x_value] = function(data[x_value], data[y_value]) + queue.append(x_value) + else: + return data[x_value] + + +def merge_n_reduce( + function: typing.Callable, arity: int, data: list +) -> typing.Any: + """Reduce the given data applying f as an inverted N-ary tree. + + Apply f cumulatively to the items of data, from left to right in n-tree + structure, to reduce the data. + + :param function: function to apply to reduce data + :param arity: Number of elements in group + :param data: List of items to be reduced + :return: List of results + """ + while len(data) > 1: + data_chunk = data[:arity] + data = data[arity:] + data.append(function(*data_chunk)) + return data[0] diff --git a/examples/dds/pycompss/interactive.py b/examples/dds/pycompss/interactive.py new file mode 100644 index 00000000..7f30422b --- /dev/null +++ b/examples/dds/pycompss/interactive.py @@ -0,0 +1,887 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Interactive API. + +Provides the functions for the use of PyCOMPSs interactively. +""" + +import logging +import os +import sys +import time + +import pycompss.util.interactive.helpers as interactive_helpers +from pycompss.util.context import CONTEXT +from pycompss.runtime.binding import get_log_path +from pycompss.runtime.binding import get_tmp_path +from pycompss.runtime.commons import CONSTANTS +from pycompss.runtime.commons import GLOBALS +from pycompss.runtime.start.initialization import LAUNCH_STATUS +from pycompss.runtime.start.interactive_initialization import ( + EXTRA_LAUNCH_STATUS, +) +from pycompss.runtime.management.classes import Future +from pycompss.runtime.management.object_tracker import OT + +# Streaming imports +from pycompss.streams.environment import init_streaming +from pycompss.streams.environment import stop_streaming +from pycompss.util.environment.configuration import ( + export_current_flags, + prepare_environment, + prepare_loglevel_graph_for_monitoring, + updated_variables_in_sc, + prepare_tracing_environment, + check_infrastructure_variables, + create_init_config_file, +) +from pycompss.util.interactive.events import release_event_manager +from pycompss.util.interactive.events import setup_event_manager +from pycompss.util.interactive.flags import check_flags +from pycompss.util.interactive.flags import print_flag_issues +from pycompss.util.interactive.graphs import show_graph +from pycompss.util.interactive.monitor import show_monitoring_information +from pycompss.util.interactive.outwatcher import STDW +from pycompss.util.interactive.state import check_monitoring_file +from pycompss.util.interactive.state import show_resources_status +from pycompss.util.interactive.state import show_statistics +from pycompss.util.interactive.state import show_tasks_info +from pycompss.util.interactive.state import show_tasks_status +from pycompss.util.interactive.utils import parameters_to_dict +from pycompss.util.logger.helpers import init_logging +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.process.manager import initialize_multiprocessing + +# Storage imports +from pycompss.util.storages.persistent import master_init_storage +from pycompss.util.storages.persistent import master_stop_storage + +# Tracing imports +from pycompss.util.tracing.helpers import emit_manual_event +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.typing_helper import typing + + +def start( # pylint: disable=too-many-arguments, too-many-locals + log_level: str = "off", + debug: bool = False, + o_c: bool = False, + graph: bool = False, + trace: bool = False, + monitor: int = -1, + project_xml: str = "", + resources_xml: str = "", + summary: bool = False, + task_execution: str = "compss", + storage_impl: str = "", + storage_conf: str = "", + streaming_backend: str = "", + streaming_master_name: str = "", + streaming_master_port: str = "", + task_count: int = 50, + app_name: str = CONSTANTS.interactive_file_name, + uuid: str = "", + log_dir: str = "", + master_working_dir: str = "", + extrae_cfg: str = "", + extrae_final_directory: str = "", + comm: str = "NIO", + conn: str = CONSTANTS.default_conn, + master_name: str = "", + master_port: str = "", + scheduler: str = CONSTANTS.default_sched, + jvm_workers: str = CONSTANTS.default_jvm_workers, + cpu_affinity: str = "automatic", + gpu_affinity: str = "automatic", + fpga_affinity: str = "automatic", + fpga_reprogram: str = "", + profile_input: str = "", + profile_output: str = "", + scheduler_config: str = "", + external_adaptation: bool = False, + propagate_virtual_environment: bool = True, + mpi_worker: bool = False, + worker_cache: typing.Union[bool, str] = False, + shutdown_in_node_failure=False, + io_executors: int = 0, + env_script: str = "", + reuse_on_block: bool = True, + nested_enabled: bool = False, + tracing_task_dependencies: bool = False, + trace_label: str = "", + extrae_cfg_python: str = "", + wcl: int = 0, + cache_profiler: bool = False, + ear: bool = False, + data_provenance: bool = False, + checkpoint_policy: str = CONSTANTS.default_checkpoint_policy, + checkpoint_params: str = "", + checkpoint_folder: str = "", + verbose: bool = False, + disable_external: bool = False, +) -> None: + """Start the runtime in interactive mode. + + :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ] + (default: "off") + :param debug: Debug mode [ True | False ] + (default: False) (overrides log-level) + :param o_c: Objects to string conversion [ True|False ] + (default: False) + :param graph: Generate graph [ True|False ] + (default: False) + :param trace: Generate trace [ True | False ] + (default: False) + :param monitor: Monitor refresh rate + (default: None) + :param project_xml: Project xml file path + (default: None) + :param resources_xml: Resources xml file path + (default: None) + :param summary: Execution summary [ True | False ] + (default: False) + :param task_execution: Task execution + (default: "compss") + :param storage_impl: Storage implementation path + (default: None) + :param storage_conf: Storage configuration file path + (default: None) + :param streaming_backend: Streaming backend + (default: None) + :param streaming_master_name: Streaming master name + (default: None) + :param streaming_master_port: Streaming master port + (default: None) + :param task_count: Task count + (default: 50) + :param app_name: Application name + (default: CONSTANTS.interactive_file_name) + :param uuid: UUId + (default: None) + :param log_dir: Logging directory + (default: None) + :param master_working_dir: Master working directory + (default: None) + :param extrae_cfg: Extrae configuration file path + (default: None) + :param extrae_final_directory: Extrae final directory (default: "") + :param comm: Communication library + (default: NIO) + :param conn: Connector + (default: DefaultSSHConnector) + :param master_name: Master Name + (default: "") + :param master_port: Master port + (default: "") + :param scheduler: Scheduler (see runcompss) + (default: es.bsc.compss.scheduler. + lookahead.locality.LocalityTS) + :param jvm_workers: Java VM parameters + (default: "-Xms1024m,-Xmx1024m,-Xmn400m") + :param cpu_affinity: CPU Core affinity + (default: "automatic") + :param gpu_affinity: GPU affinity + (default: "automatic") + :param fpga_affinity: FPGA affinity + (default: "automatic") + :param fpga_reprogram: FPGA reprogram command + (default: "") + :param profile_input: Input profile + (default: "") + :param profile_output: Output profile + (default: "") + :param scheduler_config: Scheduler configuration + (default: "") + :param external_adaptation: External adaptation [ True|False ] + (default: False) + :param propagate_virtual_environment: Propagate virtual environment + [ True|False ] + (default: False) + :param mpi_worker: Use the MPI worker [ True|False ] + (default: False) + :param worker_cache: Use the worker cache [ True | int(size) | False] + (default: False) + :param shutdown_in_node_failure: Shutdown in node failure [ True | False] + (default: False) + :param io_executors: Number of IO executors + :param env_script: Environment script to be sourced in workers + :param reuse_on_block: Reuse on block [ True | False] + (default: True) + :param nested_enabled: Nested enabled [ True | False] + (default: True) + :param tracing_task_dependencies: Include task dependencies in trace + [ True | False] (default: False) + :param trace_label: Add trace label + :param extrae_cfg_python: Extrae configuration file for the + workers + :param wcl: Wall clock limit. Stops the runtime if reached. + 0 means forever. + :param cache_profiler: Use the cache profiler [ True | False] + (default: False) + :param ear: Use EAR [ True | False ] (default: False) + :param data_provenance: Enable data provenance [ True | False ] + (default: False) + :param checkpoint_policy: Checkpointing policy. + (default: "es.bsc.compss.checkpoint. + policies.NoCheckpoint") + :param checkpoint_params: Checkpointing parameters. + (default: "") + :param checkpoint_folder: Checkpointing folder. + (default: "") + :param verbose: Verbose mode [ True|False ] + (default: False) + :param disable_external: To avoid to load compss in external process + [ True | False ] + Necessary in scenarios like pytest which fails + with multiprocessing. It also disables the + outwatcher since pytest also captures stdout + and stderr. + (default: False) + :return: None + """ + # Initialize multiprocessing + initialize_multiprocessing() + + if CONTEXT.in_pycompss(): + print("The runtime is already running") + return None + + EXTRA_LAUNCH_STATUS.set_graphing(graph) + EXTRA_LAUNCH_STATUS.set_disable_external(disable_external) + + interactive_helpers.DEBUG = debug + if debug: + log_level = "debug" + + __show_flower() + + # Let the Python binding know we are at master + CONTEXT.set_master() + # Then we can import the appropriate start and stop functions from the API + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_start, + ) + + ############################################################## + # INITIALIZATION + ############################################################## + + # Initial dictionary with the user defined parameters + all_vars = parameters_to_dict( + log_level, + debug, + o_c, + graph, + trace, + monitor, + project_xml, + resources_xml, + summary, + task_execution, + storage_impl, + storage_conf, + streaming_backend, + streaming_master_name, + streaming_master_port, + task_count, + app_name, + uuid, + log_dir, + master_working_dir, + extrae_cfg, + extrae_final_directory, + comm, + conn, + master_name, + master_port, + scheduler, + jvm_workers, + cpu_affinity, + gpu_affinity, + fpga_affinity, + fpga_reprogram, + profile_input, + profile_output, + scheduler_config, + external_adaptation, + propagate_virtual_environment, + mpi_worker, + worker_cache, + shutdown_in_node_failure, + io_executors, + env_script, + reuse_on_block, + nested_enabled, + tracing_task_dependencies, + trace_label, + extrae_cfg_python, + wcl, + cache_profiler, + ear, + data_provenance, + checkpoint_policy, + checkpoint_params, + checkpoint_folder, + ) + # Save all vars in global current flags so that events.py can restart + # the notebook with the same flags + export_current_flags(all_vars) + + # Check the provided flags + flags, issues = check_flags(all_vars) + if not flags: + print_flag_issues(issues) + return None + + # Prepare the environment + env_vars = prepare_environment( + True, o_c, storage_impl, "undefined", debug, mpi_worker + ) + all_vars.update(env_vars) + + # Update the log level and graph values if monitoring is enabled + monitoring_vars = prepare_loglevel_graph_for_monitoring( + monitor, graph, debug, log_level + ) + all_vars.update(monitoring_vars) + + # Check if running in supercomputer and update the variables accordingly + # with the defined in the launcher and exported in environment variables. + if CONSTANTS.running_in_supercomputer: + updated_vars = updated_variables_in_sc() + if verbose: + print( + f"- Overridden project xml with: {updated_vars['project_xml']}" + ) + print( + f"- Overridden resources xml with: " + f"{updated_vars['resources_xml']}" + ) + print( + f"- Overridden master name with: {updated_vars['master_name']}" + ) + print( + f"- Overridden master port with: {updated_vars['master_port']}" + ) + print(f"- Overridden uuid with: {updated_vars['uuid']}") + print(f"- Overridden log dir with: {updated_vars['log_dir']}") + print( + f"- Overridden master working dir with: " + f"{updated_vars['master_working_dir']}" + ) + print( + f"- Overridden storage conf with: " + f"{updated_vars['storage_conf']}" + ) + print( + f"- Overridden log level with: " + f"{str(updated_vars['log_level'])}" + ) + print(f"- Overridden debug with: {str(updated_vars['debug'])}") + print(f"- Overridden trace with: {str(updated_vars['trace'])}") + all_vars.update(updated_vars) + + # Update the tracing environment if set and set the appropriate trace + # integer value + all_vars["ld_library_path"] = prepare_tracing_environment( + all_vars["trace"], all_vars["extrae_lib"], all_vars["ld_library_path"] + ) + + # Update the infrastructure variables if necessary + inf_vars = check_infrastructure_variables( + all_vars["project_xml"], + all_vars["resources_xml"], + all_vars["compss_home"], + all_vars["app_name"], + all_vars["file_name"], + all_vars["external_adaptation"], + ) + all_vars.update(inf_vars) + + # With all this information, create the configuration file for the + # runtime start + create_init_config_file(**all_vars) + + # Start the event manager (ipython hooks) + ipython = globals()["__builtins__"]["get_ipython"]() + setup_event_manager(ipython) + + ############################################################## + # RUNTIME START + ############################################################## + + print("* - Starting COMPSs runtime... *") + sys.stdout.flush() # Force flush + compss_start(log_level, all_vars["trace"], True, disable_external) + + log_path = get_log_path() + GLOBALS.set_log_directory(log_path) + print("* - Log path : " + log_path) + + # Setup logging + init_logging(LOG_REMITTENT.MASTER, log_level, log_path) + logger = logging.getLogger("pycompss.runtime.launch") + + # Setup tmp path + tmp_path = get_tmp_path() # master.workingDir + GLOBALS.set_temporary_directory(tmp_path) + + # Setup interactive file + interactive_tmp_files_path = os.path.join( + tmp_path, "compss_interactive_app" + ) + if not os.path.exists(interactive_tmp_files_path): + os.makedirs(interactive_tmp_files_path) + temp_app_filename = "".join( + ( + CONSTANTS.interactive_file_name, + "_", + str(time.strftime("%d%m%y_%H%M%S")), + ".py", + ) + ) + temp_app_file_path = os.path.join( + interactive_tmp_files_path, temp_app_filename + ) + LAUNCH_STATUS.set_app_path(temp_app_file_path) + + __print_setup(verbose, all_vars) + + logger.debug("--- START ---") + logger.debug("PyCOMPSs Log path: %s", log_path) + + logger.debug("Starting storage") + persistent_storage = master_init_storage(all_vars["storage_conf"], logger) + LAUNCH_STATUS.set_persistent_storage(persistent_storage) + + logger.debug("Starting streaming") + streaming = init_streaming( + all_vars["streaming_backend"], + all_vars["streaming_master_name"], + all_vars["streaming_master_port"], + ) + LAUNCH_STATUS.set_streaming(streaming) + + # Start monitoring the stdout and stderr + if not disable_external: + STDW.start_watching() + + # MAIN EXECUTION + # let the user write an interactive application + print("* - PyCOMPSs Runtime started... Have fun! *") + print(EXTRA_LAUNCH_STATUS.get_line_separator()) + + # Emit the application start event (the 0 is in the stop function) + emit_manual_event(TRACING_MASTER.application_running_event) + return None + + +def __show_flower() -> None: + """Show the flower and version through stdout. + + :return: None + """ + line_separator = EXTRA_LAUNCH_STATUS.get_line_separator() + print(line_separator) + print(r"**************** PyCOMPSs Interactive ******************") + print(line_separator) + print(r"* .-~~-.--. ______ ______ *") + print(r"* : ) |____ \ |____ \ *") + print(r"* .~ ~ -.\ /.- ~~ . __) | __) | *") + print(r"* > `. .' < |__ | |__ | *") + print(r"* ( .- -. ) ____) | _ ____) | *") + print(r"* `- -.-~ `- -' ~-.- -' |______/ |_| |______/ *") + print(r"* ( : ) _ _ .-: *") + print(r"* ~--. : .--~ .-~ .-~ } *") + print(r"* ~-.-^-.-~ \_ .~ .-~ .~ *") + print(r"* \ \ ' \ '_ _ -~ *") + print(r"* \`.\`. // *") + print(r"* . - ~ ~-.__\`.\`-.// *") + print(r"* .-~ . - ~ }~ ~ ~-.~-. *") + print(r"* .' .-~ .-~ :/~-.~-./: *") + print(r"* /_~_ _ . - ~ ~-.~-._ *") + print(r"* ~-.< *") + print(line_separator) + + +def __print_setup( + verbose: bool, all_vars: typing.Dict[str, typing.Any] +) -> None: + """Print the setup variables through stdout (only if verbose is True). + + :param verbose: Verbose mode [True | False] + :param all_vars: Dictionary containing all variables. + :return: None + """ + line_separator = EXTRA_LAUNCH_STATUS.get_line_separator() + logger = logging.getLogger(__name__) + output = "" + output += line_separator + "\n" + output += " CONFIGURATION: \n" + for key, value in sorted(all_vars.items()): + output += f" - {key} : {value} \n" + output += line_separator + if verbose: + print(output) + logger.debug(output) + + +def stop(sync: bool = False, _hard_stop: bool = False) -> None: + """Runtime stop. + + :param sync: Scope variables synchronization [ True | False ] + (default: False) + :param _hard_stop: Stop COMPSs when runtime has died [ True | False ]. + (default: False) + :return: None + """ + logger = logging.getLogger(__name__) + ipython = globals()["__builtins__"]["get_ipython"]() + + if not CONTEXT.in_pycompss(): + return __hard_stop(interactive_helpers.DEBUG, sync, logger, ipython) + + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_stop, + ) + + line_separator = EXTRA_LAUNCH_STATUS.get_line_separator() + print(line_separator) + print("***************** STOPPING PyCOMPSs ********************") + print(line_separator) + disable_external = EXTRA_LAUNCH_STATUS.get_disable_external() + if not disable_external: + # Wait 5 seconds to give some time to process the remaining messages + # of the STDW and check if there is some error that could have stopped + # the runtime before continuing. + print("Checking if any issue happened.") + time.sleep(5) + messages = STDW.get_messages() + if messages: + for message in messages: + sys.stderr.write("".join((message, "\n"))) + + # Uncomment the following lines to see the ipython dictionary + # in a structured way: + # import pprint + # pprint.pprint(ipython.__dict__, width=1) + if sync and not _hard_stop: + sync_msg = "Synchronizing all future objects left on the user scope." + print(sync_msg) + logger.debug(sync_msg) + from pycompss.api.api import ( # pylint: disable=C0415 + # disable=import-outside-toplevel + compss_wait_on, + ) + + reserved_names = ( + "quit", + "exit", + "get_ipython", + "ipycompss", + "In", + "Out", + ) + raw_code = ipython.__dict__["user_ns"] + for k in raw_code: + obj_k = raw_code[k] + if not k.startswith("_"): # not internal objects + if isinstance(obj_k, Future): + print(f"Found a future object: {str(k)}") + logger.debug("Found a future object: %s", str(k)) + new_obj_k = compss_wait_on(obj_k) + if new_obj_k == obj_k: + print(f"\t - Could not retrieve object: {str(k)}") + logger.debug( + "\t - Could not retrieve object: %s", str(k) + ) + else: + ipython.__dict__["user_ns"][k] = new_obj_k + elif k not in reserved_names: + try: + if OT.is_pending_to_synchronize(obj_k): + print(f"Found an object to synchronize: {str(k)}") + logger.debug( + "Found an object to synchronize: %s", str(k) + ) + ipython.__dict__["user_ns"][k] = compss_wait_on( + obj_k + ) + except TypeError: + # Unhashable type: List - could be a collection + if isinstance(obj_k, list): + print(f"Found a list to synchronize: {str(k)}") + logger.debug( + "Found a list to synchronize: %s", str(k) + ) + ipython.__dict__["user_ns"][k] = compss_wait_on( + obj_k + ) + else: + print("Warning: some of the variables used with PyCOMPSs may") + print(" have not been brought to the master.") + + # Stop streaming + if LAUNCH_STATUS.get_streaming(): + stop_streaming() + + # Stop persistent storage + if LAUNCH_STATUS.get_persistent_storage(): + master_stop_storage(logger) + + # Emit the 0 for the TRACING_MASTER.*_event emitted on start function. + emit_manual_event(0) + + # Stop runtime + compss_stop(_hard_stop=_hard_stop) + + # Cleanup events and files + release_event_manager(ipython) + __clean_temp_files() + + # Stop watching stdout and stderr + if not disable_external: + STDW.stop_watching(clean=True) + # Retrieve the remaining messages that could have been captured. + last_messages = STDW.get_messages() + if last_messages: + for message in last_messages: + print(message) + + # Let the Python binding know we are not at master anymore + CONTEXT.set_out_of_scope() + + print(line_separator) + logger.debug("--- END ---") + # --- Execution finished --- + return None + + +def __hard_stop( + debug: bool, sync: bool, logger: logging.Logger, ipython: typing.Any +) -> None: + """Stop the binding securely when the runtime crashes. + + If the runtime has been stopped by any error, this method stops the + remaining things in the binding. + + :param debug: If debugging. + :param sync: Scope variables synchronization [ True | False ]. + :param logger: Logger where to put the logging messages. + :param ipython: Ipython instance. + :return: None + """ + print("The runtime is not running.") + # Check that everything is stopped as well: + + # Stop streaming + if LAUNCH_STATUS.get_streaming(): + stop_streaming() + + # Stop persistent storage + if LAUNCH_STATUS.get_persistent_storage(): + master_stop_storage(logger) + + # Clean any left object in the object tracker + OT.clean_object_tracker(hard_stop=True) + + # Cleanup events and files + release_event_manager(ipython) + __clean_temp_files() + + # Stop watching stdout and stderr + if not EXTRA_LAUNCH_STATUS.get_disable_external(): + STDW.stop_watching(clean=not debug) + # Retrieve the remaining messages that could have been captured. + last_messages = STDW.get_messages() + if last_messages: + for message in last_messages: + print(message) + + if sync: + print("* Can not synchronize any future object.") + + +def current_task_graph( + fit: bool = False, + refresh_rate: int = 1, + timeout: int = 0, + widget=None, +) -> typing.Any: + """Show current graph. + + :param fit: Fit to width [ True | False ] (default: False) + :param refresh_rate: Update the current task graph every "refresh_rate" + seconds. Default 1 second if timeout != 0. + :param timeout: Time during the current task graph is going to be updated. + :param widget: Widget where to show the current task graph. + :return: None + """ + if not EXTRA_LAUNCH_STATUS.get_graphing(): + print("Oops! Graph is not enabled in this execution.") + print( + " Please, enable it by setting the graph flag when" + + " starting PyCOMPSs." + ) + return None + return show_graph( + log_path=GLOBALS.get_log_directory(), + name="current_graph", + fit=fit, + refresh_rate=refresh_rate, + timeout=timeout, + widget=widget, + ) + + +def complete_task_graph( + fit: bool = False, + refresh_rate: int = 1, + timeout: int = 0, + widget=None, +) -> typing.Any: + """Show complete graph. + + :param fit: Fit to width [ True | False ] (default: False) + :param refresh_rate: Update the current task graph every "refresh_rate" + seconds. Default 1 second if timeout != 0 + :param timeout: Time during the current task graph is going to be updated. + :param widget: Widget where to show the current task graph. + :return: None + """ + if not EXTRA_LAUNCH_STATUS.get_graphing(): + print("Oops! Graph is not enabled in this execution.") + print( + " Please, enable it by setting the graph flag when" + + " starting PyCOMPSs." + ) + return None + elif CONTEXT.in_pycompss(): + print("Oops! The complete graph is not available yet.") + print( + " In order to see the complete task graph you must stop" + + " the COMPSs runtime with ipycompss.stop()." + ) + return None + return show_graph( + log_path=GLOBALS.get_log_directory(), + name="complete_graph", + fit=fit, + refresh_rate=refresh_rate, + timeout=timeout, + widget=widget, + ) + + +def tasks_info() -> None: + """Show tasks info. + + :return: None + """ + log_path = GLOBALS.get_log_directory() + if check_monitoring_file(log_path): + show_tasks_info(log_path) + else: + print("Oops! Monitoring is not enabled in this execution.") + print( + " Please, enable it by setting the monitor flag when" + + " starting PyCOMPSs." + ) + + +def tasks_status() -> None: + """Show tasks status. + + :return: None + """ + log_path = GLOBALS.get_log_directory() + if check_monitoring_file(log_path): + show_tasks_status(log_path) + else: + print("Oops! Monitoring is not enabled in this execution.") + print( + " Please, enable it by setting the monitor flag when" + + " starting PyCOMPSs." + ) + + +def statistics() -> None: + """Show statistics info. + + :return: None + """ + log_path = GLOBALS.get_log_directory() + if check_monitoring_file(log_path): + show_statistics(log_path) + else: + print("Oops! Monitoring is not enabled in this execution.") + print( + " Please, enable it by setting the monitor flag when" + + " starting PyCOMPSs." + ) + + +def resources_status() -> None: + """Show resources status info. + + :return: None + """ + log_path = GLOBALS.get_log_directory() + if check_monitoring_file(log_path): + show_resources_status(log_path) + else: + print("Oops! Monitoring is not enabled in this execution.") + print( + " Please, enable it by setting the monitor flag when" + + " starting PyCOMPSs." + ) + + +def monitoring_information() -> None: + """Show monitoring evolution information. + + :return: None + """ + log_path = GLOBALS.get_log_directory() + show_monitoring_information(log_path) + + +# ########################################################################### # +# ########################################################################### # +# ########################################################################### # + + +def __clean_temp_files() -> None: + """Remove any temporary files that may exist. + + Currently: APP_PATH, which contains the file path where all interactive + code required by the worker is. + + :return: None + """ + app_path = LAUNCH_STATUS.get_app_path() + try: + if os.path.exists(app_path): + os.remove(app_path) + if os.path.exists(app_path + "c"): + os.remove(app_path + "c") + except OSError: + print("[ERROR] An error has occurred when cleaning temporary files.") diff --git a/examples/dds/pycompss/runtime/__init__.py b/examples/dds/pycompss/runtime/__init__.py new file mode 100644 index 00000000..5a5e3f9c --- /dev/null +++ b/examples/dds/pycompss/runtime/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime.""" diff --git a/examples/dds/pycompss/runtime/binding.py b/examples/dds/pycompss/runtime/binding.py new file mode 100644 index 00000000..0fd843c6 --- /dev/null +++ b/examples/dds/pycompss/runtime/binding.py @@ -0,0 +1,1075 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Binding core module. + +This file contains the Python binding core classes and methods. +""" + +import os +import re +import signal +from shutil import rmtree + +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.context import CONTEXT +from pycompss.runtime.commons import GLOBALS +from pycompss.runtime.management.classes import EmptyReturn +from pycompss.runtime.management.direction import get_compss_direction +from pycompss.runtime.management.object_tracker import OT +from pycompss.runtime.management.synchronization import wait_on_object +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.runtime.task.definitions.arguments import TaskArguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.logger.helpers import add_new_logger + +# Tracing imports +from pycompss.util.tracing.helpers import enable_trace_master +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.helpers import EventMaster +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + LOGGER = logging.getLogger(__name__) + + +# ########################################################################### # +# ############ FUNCTIONS THAT COMMUNICATE WITH THE RUNTIME ################## # +# ########################################################################### # + + +def start_runtime( + log_level: str = "off", + tracing: bool = False, + interactive: bool = False, + disable_external: bool = False, +) -> None: + """Start the COMPSs runtime. + + Starts the runtime by calling the external python library that calls + the bindings-common. + + :param log_level: Log level [ "trace" | "debug" | "info" | "api" | "off" ]. + :param tracing: Tracing level [ True | False ]. + :param interactive: Boolean if interactive (ipython or jupyter). + :param disable_external: To avoid to load compss in external process. + :return: None. + """ + if __debug__: + LOGGER.info("Starting COMPSs...") + + if tracing and not interactive: + # Enabled only if not interactive - extrae issues within jupyter. + enable_trace_master() + + with EventMaster(TRACING_MASTER.start_runtime_event): + if interactive and CONTEXT.in_master() and not disable_external: + COMPSs.load_runtime(external_process=True) + else: + COMPSs.load_runtime(external_process=False) + + if log_level == "trace": + # Could also be "debug" or True, but we only show the C extension + # debug in the maximum tracing level. + COMPSs.set_debug(True) + OT.enable_report() + + COMPSs.start_runtime() + + if __debug__: + LOGGER.info("COMPSs started") + + +def stop_runtime(code: int = 0, hard_stop: bool = False) -> None: + """Stop the COMPSs runtime. + + Stops the runtime by calling the external python library that calls + the bindings-common. + Also cleans objects and temporary files created during runtime. + If the code is different from 0, all running or waiting tasks will be + cancelled. + + :parameter code: Stop code (if code != 0 ==> cancel application tasks). + :param hard_stop: Stop compss when runtime has died. + :return: None. + """ + with EventMaster(TRACING_MASTER.stop_runtime_event): + app_id = 0 + if __debug__: + LOGGER.info("Stopping runtime...") + + # Stopping a possible wall clock limit + signal.alarm(0) + + if code != 0: + if __debug__: + LOGGER.info("Canceling all application tasks...") + COMPSs.cancel_application_tasks(app_id, 0) + + if __debug__: + LOGGER.info("Cleaning objects...") + _clean_objects(hard_stop=hard_stop) + + if __debug__: + reporting = OT.is_report_enabled() + if reporting: + LOGGER.info("Generating Object tracker report...") + target_path = get_log_path() + OT.generate_report(target_path) + OT.clean_report() + + if __debug__: + LOGGER.info("Stopping COMPSs...") + COMPSs.stop_runtime(code) + + if __debug__: + LOGGER.info("Cleaning temps...") + _clean_temps() + + CONTEXT.set_out_of_scope() + if __debug__: + LOGGER.info("COMPSs stopped") + + +def file_exists(*file_name: typing.Union[list, tuple, str]) -> typing.Any: + """Check if one or more files exists (has/have been accessed). + + :param file_name: File/s name. + :return: True if accessed, False otherwise. + """ + if __debug__: + LOGGER.debug( + "Checking if file/s: %s has/have been accessed.", file_name + ) + return __apply_recursively_to_file( + __file_exists, TRACING_MASTER.accessed_file_event, True, *file_name + ) + + +def __file_exists(app_id: int, file_name: str) -> bool: + """Check if one files exists (has been accessed). + + Calls the external python library (that calls the bindings-common) + in order to check if a file has been accessed. + + :param app_id: Application identifier. + :param file_name: File name. + :return: True if accessed, False otherwise. + """ + if os.path.exists(file_name): + return True + return COMPSs.accessed_file(app_id, file_name) + + +def open_file(file_name: str, mode: str) -> str: + """Open a file (retrieves if necessary). + + Calls the external python library (that calls the bindings-common) + in order to request a file. + + :param file_name: File name. + :param mode: Open file mode ('r', 'rw', etc.). + :return: The current name of the file requested (that may have been + renamed during runtime). + """ + with EventMaster(TRACING_MASTER.open_file_event): + app_id = 0 + compss_mode = get_compss_direction(mode) + if __debug__: + LOGGER.debug( + "Getting file %s with mode %s", file_name, compss_mode + ) + compss_name = COMPSs.open_file(app_id, file_name, compss_mode) + if __debug__: + LOGGER.debug("COMPSs file name is %s", compss_name) + return compss_name + + +def delete_file(*file_name: typing.Union[list, tuple, str]) -> typing.Any: + """Remove one or more files. + + :param file_name: File/s name to remove. + :return: True if success. False otherwise. With the same file_name + structure. + """ + if __debug__: + LOGGER.debug("Deleting file/s: %s", file_name) + return __apply_recursively_to_file( + __delete_file, TRACING_MASTER.delete_file_event, True, *file_name + ) + + +def __delete_file(app_id: int, file_name: str) -> bool: + """Remove one or more files. + + Calls the external python library (that calls the bindings-common) + in order to request a file removal. + + :param app_id: Application identifier. + :param file_name: File/s name to remove. + :return: True if success. False otherwise. With the same file_name + structure. + """ + result = COMPSs.delete_file(app_id, file_name, True) + if __debug__: + if result: + LOGGER.debug("File %s successfully deleted.", file_name) + else: + LOGGER.error("Failed to remove file %s.", file_name) + return result + + +def wait_on_file(*file_name: typing.Union[list, tuple, str]) -> typing.Any: + """Retrieve one or more files. + + :param file_name: File name/s to retrieve (can contain lists and tuples + of strings). + :return: The file name/s (with the same structure). + """ + if __debug__: + LOGGER.debug("Getting file/s: %s", file_name) + return __apply_recursively_to_file( + COMPSs.get_file, TRACING_MASTER.get_file_event, False, *file_name + ) + + +def wait_on_directory( + *directory_name: typing.Union[list, tuple, str] +) -> typing.Any: + """Retrieve one or more directories. + + :param directory_name: Directory name/s to retrieve (can contain lists + and tuples of strings). + :return: The directory name/s (with the same structure). + """ + if __debug__: + LOGGER.debug("Getting directory/s: %s", directory_name) + return __apply_recursively_to_file( + COMPSs.get_directory, + TRACING_MASTER.get_directory_event, + False, + *directory_name, + ) + + +def __apply_recursively_to_file( + function: typing.Callable, + event: int, + get_results: bool, + *name: typing.Union[list, tuple, str], +) -> typing.Any: + """Apply the given function recursively over the given names. + + Calls the external python library (that calls the bindings-common) + in order to request last version of a file. + Iterates recursively over file_name lists and tuples. + Emits an event per name processed. + + :param function: Function to apply. + :param event: Event to emit. + :param get_results: If get the results of the function, or the given name. + :param name: File/s or directory/ies name to apply the given function. + :return: The result of applying the given function. + """ + app_id = 0 + ret = [] # type: typing.List[typing.Union[list, tuple, str, bool]] + for f_name in name: + if isinstance(f_name, str): + with EventMaster(event): + result = function(app_id, f_name) + if get_results: + ret.append(result) + else: + ret.append(f_name) + elif isinstance(f_name, list): + files_list = list( + [ + __apply_recursively_to_file( + function, event, get_results, name + ) + for name in f_name + ] + ) + ret.append(files_list) + elif isinstance(f_name, tuple): + files_tuple = tuple( + [ + __apply_recursively_to_file( + function, event, get_results, name + ) + for name in f_name + ] + ) + ret.append(files_tuple) + else: + raise PyCOMPSsException( + "Unsupported type in apply_recursively. " + "Must be str, list or tuple" + ) + if len(ret) == 1: + return ret[0] + return ret + + +def delete_object( + *objs: typing.Any, +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Remove object/s. + + :param objs: Object/s to remove. + :return: True if success. False otherwise. Keeps structure if lists or + tuples are provided. + """ + if __debug__: + LOGGER.debug("Deleting object/s: %r", objs) + app_id = 0 + ret = [] # type: typing.List[typing.Union[bool, list]] + for obj in objs: + with EventMaster(TRACING_MASTER.delete_object_event): + result = __delete_object(app_id, obj) + ret.append(result) + if len(ret) == 1: + return ret[0] + return ret + + +def __delete_object(app_id: int, obj: typing.Any) -> bool: + """Remove object function. + + Removes a used object from the internal structures and calls the + external python library (that calls the bindings-common) + in order to request its corresponding file removal. + + :param app_id: Application identifier. + :param obj: Object to remove. + :return: True if success. False otherwise. + """ + obj_id = OT.is_tracked(obj) + if obj_id is None: + # Not being tracked + return False + try: + file_name = OT.get_file_name(obj_id) + COMPSs.delete_file(app_id, file_name, False) + OT.stop_tracking(obj) + except KeyError: + pass + return True + + +def barrier(no_more_tasks: bool = False) -> None: + """Wait for all submitted tasks. + + Calls the external python library (that calls the bindings-common) + in order to request a barrier. + + :param no_more_tasks: If no more tasks are going to be submitted, remove + all objects. + :return: None. + """ + with EventMaster(TRACING_MASTER.barrier_event): + if __debug__: + LOGGER.debug("Barrier. No more tasks? %s", str(no_more_tasks)) + # If noMoreFlags is set, clean up the objects + if no_more_tasks: + _clean_objects() + app_id = 0 + # Call the Runtime barrier (appId 0, not needed for the signature) + COMPSs.barrier(app_id, no_more_tasks) + + +def barrier_group(group_name: str) -> str: + """Wait for all tasks of the given group. + + Calls the external python library (that calls the bindings-common) + in order to request a barrier of a group. + + :param group_name: Group name. + :return: None or string with exception message. + """ + with EventMaster(TRACING_MASTER.barrier_group_event): + app_id = 0 + # Call the Runtime group barrier + return str(COMPSs.barrier_group(app_id, group_name)) + + +def open_task_group(group_name: str, implicit_barrier: bool) -> None: + """Open task group. + + Calls the external python library (that calls the bindings-common) + in order to request an opening of a group. + + :param group_name: Group name. + :param implicit_barrier: Perform a wait on all group tasks before closing. + :return: None. + """ + with EventMaster(TRACING_MASTER.open_task_group_event): + app_id = 0 + COMPSs.open_task_group(group_name, implicit_barrier, app_id) + + +def close_task_group(group_name: str) -> None: + """Close task group. + + Calls the external python library (that calls the bindings-common) + in order to request a group closure. + + :param group_name: Group name. + :return: None. + """ + with EventMaster(TRACING_MASTER.close_task_group_event): + app_id = 0 + COMPSs.close_task_group(group_name, app_id) + + +def cancel_task_group(group_name: str) -> str: + """Cancel task group. + + Calls the external python library (that calls the bindings-common) + in order to request a group cancellation. + + :param group_name: Group name. + :return: None or string with exception message. + """ + with EventMaster(TRACING_MASTER.cancel_task_group_event): + app_id = 0 + return str(COMPSs.cancel_task_group(group_name, app_id)) + + +def snapshot() -> None: + """Make a snapshot of the tasks. + + Calls the external python library (that calls the bindings-common) + in order to request a snapshot. + + :return: None + """ + with EventMaster(TRACING_MASTER.snapshot_event): + app_id = 0 + COMPSs.snapshot(app_id) + + +def get_log_path() -> str: + """Get logging path. + + Requests the logging path to the external python library (that calls + the bindings-common). + + :return: The path where to store the logs. + """ + with EventMaster(TRACING_MASTER.get_log_path_event): + if __debug__: + LOGGER.debug("Requesting log path") + log_path = COMPSs.get_logging_path() + if __debug__: + LOGGER.debug("Log path received: %s", log_path) + return log_path + + +def get_tmp_path() -> str: + """Get tmp path. + + Requests the master working path to the external python library (that + calls the bindings-common). + + :return: The path where to store the master tmp files. + """ + with EventMaster(TRACING_MASTER.get_tmp_path_event): + if __debug__: + LOGGER.debug("Requesting tmp path (master working dir)") + tmp_path = COMPSs.get_master_working_path() + if __debug__: + LOGGER.debug( + "Tmp path (master working dir) received: %s", tmp_path + ) + return tmp_path + + +def get_number_of_resources() -> int: + """Get the number of resources. + + Calls the external python library (that calls the bindings-common) + in order to request for the number of active resources. + + :return: Number of active resources. + """ + with EventMaster(TRACING_MASTER.get_number_resources_event): + app_id = 0 + if __debug__: + LOGGER.debug("Request the number of active resources") + # Call the Runtime + return COMPSs.get_number_of_resources(app_id) + + +def request_resources( + num_resources: int, group_name: typing.Optional[str] +) -> None: + """Request new resources. + + Calls the external python library (that calls the bindings-common) + in order to request for the creation of the given resources. + + :param num_resources: Number of resources to create. + :param group_name: Task group to notify upon resource creation. + :return: None. + """ + with EventMaster(TRACING_MASTER.request_resources_event): + app_id = 0 + if group_name is None: + group_name = "NULL" + if __debug__: + LOGGER.debug( + "Request the creation of %s resources with notification to " + "task group %s", + str(num_resources), + str(group_name), + ) + # Call the Runtime + COMPSs.request_resources(app_id, num_resources, group_name) + + +def free_resources( + num_resources: int, group_name: typing.Optional[str] +) -> None: + """Liberate resources. + + Calls the external python library (that calls the bindings-common) + in order to request for the destruction of the given resources. + + :param num_resources: Number of resources to destroy. + :param group_name: Task group to notify upon resource creation. + :return: None. + """ + with EventMaster(TRACING_MASTER.free_resources_event): + app_id = 0 + if group_name is None: + group_name = "NULL" + if __debug__: + LOGGER.debug( + "Request the destruction of %s resources with notification to " + "task group %s", + str(num_resources), + str(group_name), + ) + # Call the Runtime + COMPSs.free_resources(app_id, num_resources, group_name) + + +def set_wall_clock(wall_clock_limit: int) -> None: + """Set the application wall clock limit. + + :param wall_clock_limit: Wall clock limit in seconds. + :return: None. + """ + with EventMaster(TRACING_MASTER.wall_clock_limit_event): + app_id = 0 + if __debug__: + LOGGER.debug("Set a wall clock limit of %s", str(wall_clock_limit)) + # Activate wall clock limit alarm + signal.signal(signal.SIGALRM, _wall_clock_exceed) + signal.alarm(wall_clock_limit) + # Call the Runtime to set a timer in case wall clock is + # reached in a synch + COMPSs.set_wall_clock(app_id, wall_clock_limit) + + +def register_ce(core_element: CE) -> None: + """Register a core element. + + Calls the external python library (that calls the bindings-common) + in order to notify the runtime about a core element that needs to be + registered. + + Java Examples: + + // METHOD + System.out.println('Registering METHOD implementation'); + String core_elementSignature = 'methodClass.methodName'; + String impl_signature = 'methodClass.methodName'; + String impl_constraints = 'ComputingUnits:2'; + String impl_type = 'METHOD'; + String[] impl_type_args = new String[] { 'methodClass', + 'methodName' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // MPI + System.out.println('Registering MPI implementation'); + core_elementSignature = 'methodClass1.methodName1'; + impl_signature = 'mpi.MPI'; + impl_constraints = 'StorageType:SSD'; + impl_type = 'MPI'; + impl_type_args = new String[] { 'mpiBinary', + 'mpiWorkingDir', + 'mpiRunner' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // PYTHON MPI + System.out.println('Registering PYTHON MPI implementation'); + core_elementSignature = 'methodClass1.methodName1'; + impl_signature = 'MPI.methodClass1.methodName'; + impl_constraints = 'ComputingUnits:2'; + impl_type = 'PYTHON_MPI'; + impl_type_args = new String[] { 'methodClass', + 'methodName', + 'mpiWorkingDir', + 'mpiRunner' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // BINARY + System.out.println('Registering BINARY implementation'); + core_elementSignature = 'methodClass2.methodName2'; + impl_signature = 'binary.BINARY'; + impl_constraints = 'MemoryType:RAM'; + impl_type = 'BINARY'; + impl_type_args = new String[] { 'binary', + 'binaryWorkingDir' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // OMPSS + System.out.println('Registering OMPSS implementation'); + core_elementSignature = 'methodClass3.methodName3'; + impl_signature = 'ompss.OMPSS'; + impl_constraints = 'ComputingUnits:3'; + impl_type = 'OMPSS'; + impl_type_args = new String[] { 'ompssBinary', + 'ompssWorkingDir' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // OPENCL + System.out.println('Registering OPENCL implementation'); + core_elementSignature = 'methodClass4.methodName4'; + impl_signature = 'opencl.OPENCL'; + impl_constraints = 'ComputingUnits:4'; + impl_type = 'OPENCL'; + impl_type_args = new String[] { 'openclKernel', + 'openclWorkingDir' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + // VERSIONING + System.out.println('Registering METHOD implementation'); + core_elementSignature = 'methodClass.methodName'; + impl_signature = 'anotherClass.anotherMethodName'; + impl_constraints = 'ComputingUnits:1'; + impl_type = 'METHOD'; + impl_type_args = new String[] { 'anotherClass', + 'anotherMethodName' }; + rt.registerCoreElement(coreElementSignature, + impl_signature, + impl_constraints, + impl_type, + impl_type_args); + + --------------------- + + Core Element fields: + + ce_signature: Core Element signature + (e.g.- "methodClass.methodName") + impl_signature: Implementation signature + (e.g.- "methodClass.methodName") + impl_constraints: Implementation constraints + (e.g.- "{ComputingUnits:2}") + impl_type: Implementation type + ("METHOD" | "MPI" | "BINARY" | "OMPSS" | "OPENCL") + impl_io: IO Implementation + impl_type_args: Implementation arguments + (e.g.- ["methodClass", "methodName"]) + + :param core_element: Core Element to register. + :return: None. + """ + with EventMaster(TRACING_MASTER.register_core_element_event): + # Retrieve Core element fields + ce_signature = core_element.get_ce_signature() + impl_signature_base = core_element.get_impl_signature() + impl_signature = ( + None if impl_signature_base == "" else impl_signature_base + ) + impl_constraints_base = core_element.get_impl_constraints() + impl_constraints = None # type: typing.Any + if impl_constraints_base == "": + impl_constraints = {} + else: + impl_constraints = impl_constraints_base + impl_type_base = core_element.get_impl_type() + impl_type = None if impl_type_base == "" else str(impl_type_base) + impl_local = str(core_element.get_impl_local()) + impl_io = str(core_element.get_impl_io()) + impl_type_args = core_element.get_impl_type_args() + prolog = core_element.get_impl_prolog() + epilog = core_element.get_impl_epilog() + container = core_element.get_impl_container() + + if __debug__: + LOGGER.debug("Registering CE with signature: %s", ce_signature) + LOGGER.debug("\t - Implementation signature: %s", impl_signature) + + # Build constraints string from constraints dictionary + impl_constraints_lst = [] + for key, value in impl_constraints.items(): + if isinstance(value, int): + val = str(value) + elif isinstance(value, str): + val = value + elif isinstance(value, list): + val = str(value).replace("'", "") + else: + raise PyCOMPSsException( + "Implementation constraints items must be " + "str, int or list." + ) + kv_constraint = "".join((key, ":", str(val), ";")) + impl_constraints_lst.append(kv_constraint) + impl_constraints_str = "".join(impl_constraints_lst) + + if __debug__: + LOGGER.debug( + "\t - Implementation constraints: %s", impl_constraints_str + ) + LOGGER.debug("\t - Implementation type: %s", impl_type) + LOGGER.debug( + "\t - Implementation type arguments: %s", + " ".join(impl_type_args), + ) + # import pdb; pdb.set_trace() + # Call runtime with the appropriate parameters + COMPSs.register_core_element( + ce_signature, + impl_signature, + impl_constraints_str, + impl_type, + impl_local, + impl_io, + prolog, + epilog, + container, + impl_type_args, + ) + if __debug__: + LOGGER.debug("CE with signature %s registered.", ce_signature) + + +def wait_on(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """Wait on a set of objects. + + Waits on a set of objects defined in args with the options defined in + kwargs. + + :param args: Objects to wait on. + :param kwargs: Options: Write enable? [True | False] Default = True. + May include: master_event: Emit master event. [Default: True | False] + False will emit the event inside task + (for nested). + :return: Real value of the objects requested. + """ + master_event = True + if "master_event" in kwargs: # pylint: disable=consider-using-get + master_event = kwargs["master_event"] + if master_event: + with EventMaster(TRACING_MASTER.wait_on_event): + return __wait_on(*args, **kwargs) + else: + with EventInsideWorker(TRACING_WORKER.wait_on_event): + return __wait_on(*args, **kwargs) + + +def __wait_on(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """Wait on a set of objects. + + Waits on a set of objects defined in args with the options defined in + kwargs. + + :param args: Objects to wait on. + :param kwargs: Options: Write enable? [True | False] Default = True. + :return: Real value of the objects requested. + """ + ret = list( + map(wait_on_object, args, [kwargs.get("mode", "rw")] * len(args)) + ) + if len(ret) == 1: + ret_lst = ret[0] + else: + ret_lst = ret + # Check if there are empty elements return elements that need to be removed + if isinstance(ret_lst, list): + # Look backwards the list removing the first EmptyReturn elements + for elem in reversed(ret_lst): + if isinstance(elem, EmptyReturn): + ret_lst.remove(elem) + return ret_lst + + +def process_task( + signature: str, + has_target: bool, + names: list, + values: list, + num_returns: int, + compss_types: list, + compss_directions: list, + compss_streams: list, + compss_prefixes: list, + content_types: list, + weights: list, + keep_renames: list, + decorator_arguments: TaskArguments, + is_http: bool = False, +) -> None: + """Submit a task to the runtime. + + :param signature: Task signature. + :param has_target: Boolean if the task has self. + :param names: Task parameter names. + :param values: Task parameter values. + :param num_returns: Number of returns. + :param compss_types: List of parameter types. + :param compss_directions: List of parameter directions. + :param compss_streams: List of parameter streams. + :param compss_prefixes: List of parameter prefixes. + :param content_types: Content types. + :param weights: List of parameter weights. + :param keep_renames: Boolean keep renaming. + :param decorator_arguments: TaskArguments object containing all information + contained in the @task decorator. + :param is_http: If it is a http task (service). + :return: The future object related to the task return. + """ + with EventMaster(TRACING_MASTER.process_task_event): + app_id = 0 + has_priority = decorator_arguments.priority + num_nodes = decorator_arguments.computing_nodes + reduction = decorator_arguments.is_reduce + chunk_size = decorator_arguments.chunk_size + replicated = decorator_arguments.is_replicated + distributed = decorator_arguments.is_distributed + on_failure = decorator_arguments.on_failure + time_out = decorator_arguments.time_out + if __debug__: + # Log the task submission values for debugging purposes. + values_str = " ".join(str(v) for v in values) + types_str = " ".join(str(t) for t in compss_types) + direct_str = " ".join(str(d) for d in compss_directions) + streams_str = " ".join(str(s) for s in compss_streams) + prefixes_str = " ".join(str(p) for p in compss_prefixes) + names_str = " ".join(x for x in names) + ct_str = " ".join(str(x) for x in content_types) + weights_str = " ".join(str(x) for x in weights) + keep_renames_str = " ".join(str(x) for x in keep_renames) + LOGGER.debug("Processing task:") + LOGGER.debug("\t- App id: %s", str(app_id)) + LOGGER.debug("\t- Signature: %s", signature) + LOGGER.debug("\t- Has target: %s", str(has_target)) + LOGGER.debug("\t- Names: %s", names_str) + LOGGER.debug("\t- Values: %s", values_str) + LOGGER.debug("\t- COMPSs types: %s", types_str) + LOGGER.debug("\t- COMPSs directions: %s", direct_str) + LOGGER.debug("\t- COMPSs streams: %s", streams_str) + LOGGER.debug("\t- COMPSs prefixes: %s", prefixes_str) + LOGGER.debug("\t- Content Types: %s", ct_str) + LOGGER.debug("\t- Weights: %s", weights_str) + LOGGER.debug("\t- Keep_renames: %s", keep_renames_str) + LOGGER.debug("\t- Priority: %s", str(has_priority)) + LOGGER.debug("\t- Num nodes: %s", str(num_nodes)) + LOGGER.debug("\t- Reduce: %s", str(reduction)) + LOGGER.debug("\t- Chunk Size: %s", str(chunk_size)) + LOGGER.debug("\t- Replicated: %s", str(replicated)) + LOGGER.debug("\t- Distributed: %s", str(distributed)) + LOGGER.debug("\t- On failure behavior: %s", on_failure) + LOGGER.debug("\t- Task time out: %s", str(time_out)) + LOGGER.debug("\t- Is http: %s", str(is_http)) + + # Check that there is the same amount of values as their types, as well + # as their directions, streams and prefixes. + if not ( + len(values) + == len(compss_types) + == len(compss_directions) + == len(compss_streams) + == len(compss_prefixes) + == len(content_types) + == len(weights) + == len(keep_renames) + ): + raise PyCOMPSsException( + "Issue with the amount of values, types, " + "directions, streams, prefixes, etc." + ) + + # Submit task to the runtime (call to the C extension): + # Parameters: + # 0 - - application id (by default always 0 due to it + # is not currently needed for the signature) + # 1 - - path of the module where the task is + # + # 2 - - behavior if the task fails + # + # 3 - - function name of the task (to be called from + # the worker) + # 4 - - priority flag (true|false) + # + # 5 - - has target (true|false). If the task is within + # an object or not. + # 6 - [] - task parameters (basic types or file paths for + # objects) + # 7 - [] - parameters types (number corresponding to the + # type of each parameter) + # 8 - [] - parameters directions (number corresponding to + # the direction of each parameter) + # 9 - [] - parameters streams (number corresponding to the + # stream of each parameter) + # 10 - [] - parameters prefixes (string corresponding to + # the prefix of each parameter) + # 11 - [] - parameters extra type (string corresponding to + # the extra type of each parameter) + # 12 - [] - parameters weights (string corresponding to the + # weight of each parameter + # 13 - - Keep renames flag (true|false) + # + + if not is_http: + COMPSs.process_task( + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + else: + COMPSs.process_http_task( + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + + +def add_logger(logger_name: str) -> None: + """Add a new logger for the user. + + :param logger_name: New logger name. + :returns: None + """ + add_new_logger(logger_name) + + +# ########################################################################### # +# ####################### AUXILIARY FUNCTIONS ############################### # +# ########################################################################### # + + +def _clean_objects(hard_stop: bool = False) -> None: + """Clean all objects. + + Clean the objects stored in the global dictionaries from the object + tracker. + + :param hard_stop: avoid call to delete_file when the runtime has died. + :return: None. + """ + OT.clean_object_tracker(hard_stop=hard_stop) + + +def _clean_temps() -> None: + """Clean temporary files. + + The temporary files end with the IT extension. + + :return: None. + """ + temp_directory = GLOBALS.get_temporary_directory() + rmtree(temp_directory, True) + cwd = os.getcwd() + for temp_file in os.listdir(cwd): + if re.search(r"d\d+v\d+_\d+\.IT", temp_file): + os.remove(os.path.join(cwd, temp_file)) + + +def _wall_clock_exceed(signum: int, frame: typing.Any) -> None: + """Task wall clock exceeded action: raise PyCOMPSs exception. + + Do not remove the parameters. + + :param signum: Signal number. + :param frame: Frame. + :return: None. + :raises: PyCOMPSsException exception. + """ + raise PyCOMPSsException("Application has reached its wall clock limit") diff --git a/examples/dds/pycompss/runtime/commons.py b/examples/dds/pycompss/runtime/commons.py new file mode 100644 index 00000000..bcd9f761 --- /dev/null +++ b/examples/dds/pycompss/runtime/commons.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Commons. + +This file contains the common definitions of the Python binding. +""" + +import os +from tempfile import mkdtemp + +from pycompss.util.typing_helper import typing # noqa: F401 + + +####################################### +# Global variables set in this module # +####################################### + + +class Constants: # pylint: disable=R0902,R0903 + # disable=too-many-instance-attributes, too-few-public-methods + """Common constants definitions.""" + + __slots__ = ( + "empty_string_key", + "python_interpreter", + "str_escape", + "environment", + "is_interactive", + "running_in_supercomputer", + "tracing_hook_env_var", + "extra_content_type_format", + "interactive_file_name", + "compss_interactive_source_file", + "default_sched", + "default_conn", + "default_jvm_workers", + "default_checkpoint_policy", + "temp_dir_prefix", + "temp_obj_prefix", + ) + + def __init__(self) -> None: + """Constant constructor. + + :returns: None. + """ + # Empty string substitution key + self.empty_string_key = "3mPtY57r1Ng" + # Default python interpreter + self.python_interpreter = "python3" + # Coding/decoding escape + self.str_escape = "unicode_escape" + # Determine the environment + environment = "terminal" + is_interactive = False + try: + from IPython import get_ipython # noqa # pylint: disable=C0415 + + # disable=import-outside-toplevel + + ipy_str = str(type(get_ipython())) + if "zmqshell" in ipy_str: + environment = "jupyter" + is_interactive = True + if "terminal" in ipy_str: + environment = "ipython" + is_interactive = True + except ImportError: + environment = "terminal" + is_interactive = False + self.environment = environment + self.is_interactive = is_interactive + # Determine if running in a supercomputer + self.running_in_supercomputer = False + if ( + "COMPSS_RUNNING_IN_SC" in os.environ + and os.environ["COMPSS_RUNNING_IN_SC"] == "true" + ): + self.running_in_supercomputer = True + elif ( + "BSC_MACHINE" in os.environ and os.environ["BSC_MACHINE"] == "mn4" + ): + # Only supported in MN4 currently + self.running_in_supercomputer = True + # Tracing hook environment variable + self.tracing_hook_env_var = "COMPSS_TRACING_HOOK" + # Extra content type format + self.extra_content_type_format = "{}:{}" # : + # Interactive mode file name prefix + self.interactive_file_name = "InteractiveMode" + # Interactive source file that will be used + self.compss_interactive_source_file = "compss_interactive_source_file" + # LONG DEFAULTS + self.default_sched = ( + "es.bsc.compss.scheduler.lookahead.locality.LocalityTS" + ) + self.default_conn = "es.bsc.compss.connectors.DefaultSSHConnector" + self.default_jvm_workers = "-Xms1024m,-Xmx1024m,-Xmn400m" + self.default_checkpoint_policy = ( + "es.bsc.compss.checkpoint.policies.NoCheckpoint" + ) + # Temporary directory/objects info + self.temp_dir_prefix = "pycompss" + self.temp_obj_prefix = "/compss-serialized-obj_" + + +# Placeholder for all constant variables +CONSTANTS = Constants() + + +############################################### +# Global variables set from different modules # +############################################### + + +class Globals: + """Common global definitions.""" + + __slots__ = ( + "log_dir", + "temp_dir", + "analysis_dir", + "object_conversion", + "tracing_task_name_to_id", + ) + + def __init__(self) -> None: + """Global object constructor. + + :returns: None. + """ + self.log_dir = "" + self.temp_dir = "" + self.analysis_dir = "" + self.object_conversion = False + self.tracing_task_name_to_id = {} # type: typing.Dict[str, int] + + def get_log_directory(self) -> str: + """Log directory getter. + + :return: Log directory path. + """ + return self.log_dir + + def set_log_directory(self, folder: str) -> None: + """Set the log directory. + + :param folder: Log directory path. + :return: None. + """ + if not os.path.exists(folder): + os.mkdir(folder) + self.log_dir = folder + + def get_temporary_directory(self) -> str: + """Temporary directory getter. + + :return: Temporary directory path. + """ + return self.temp_dir + + def set_temporary_directory(self, folder: str) -> None: + """Set the temporary directory. + + Creates the temporary directory from the folder parameter and + sets the temporary directory variable. + + :param folder: Temporary directory path. + :return: None. + """ + temp_dir = mkdtemp(prefix=CONSTANTS.temp_dir_prefix, dir=folder) + self.temp_dir = temp_dir + + def get_analysis_directory(self) -> str: + """Analysis directory getter. + + :return: Analysis directory path. + """ + return self.analysis_dir + + def set_analysis_directory(self, folder: str) -> None: + """Set the analysis directory. + + Creates the analysis directory from the folder parameter and + sets the analysis directory variable. + + :param folder: Analysis directory path. + :return: None. + """ + if not os.path.exists(folder): + os.mkdir(folder) + self.analysis_dir = folder + + def in_tracing_task_name_to_id(self, task_name: str) -> bool: + """Check if task_name is in tracing_task_name_to_id dictionary. + + :param task_name: Traced task name. + :return: Boolean if exists. + """ + return task_name in self.tracing_task_name_to_id + + def get_tracing_task_name_id(self, task_name: str) -> int: + """Retrieve the identifier of the given task_name. + + :param task_name: Traced task name. + :return: The task_name identifier. + """ + return self.tracing_task_name_to_id[task_name] + + def set_tracing_task_name_to_id(self, task_name: str, value: int) -> None: + """Set value as the identifier for the given task_name. + + :param task_name: Traced task name. + :param value: Traced task identifier. + :return: None. + """ + self.tracing_task_name_to_id[task_name] = value + + def len_tracing_task_name_to_id(self) -> int: + """Retrieve the amount of identifier registered. + + :return: The number of entries. + """ + return len(self.tracing_task_name_to_id) + + +GLOBALS = Globals() diff --git a/examples/dds/pycompss/runtime/launch.py b/examples/dds/pycompss/runtime/launch.py new file mode 100644 index 00000000..7c6630ef --- /dev/null +++ b/examples/dds/pycompss/runtime/launch.py @@ -0,0 +1,709 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Launch. + +This file contains the __main__ method. +It is called from the runcompss/enqueue_compss/cli script with the user and +environment parameters. +""" + +import argparse +import gc +import logging + +# Imports +import os +import sys +import traceback + +# Project imports +from pycompss.util.context import CONTEXT +from pycompss.util.context import loading_context +from pycompss.api.exceptions import COMPSsException +from pycompss.runtime.binding import get_log_path +from pycompss.runtime.binding import get_tmp_path +from pycompss.runtime.commons import CONSTANTS +from pycompss.runtime.commons import GLOBALS +from pycompss.runtime.start.initialization import LAUNCH_STATUS +from pycompss.runtime.task.features import TASK_FEATURES + +# Streaming imports +from pycompss.streams.environment import init_streaming +from pycompss.streams.environment import stop_streaming +from pycompss.util.environment.configuration import ( + preload_user_code, + export_current_flags, + prepare_environment, + prepare_loglevel_graph_for_monitoring, + updated_variables_in_sc, + prepare_tracing_environment, + check_infrastructure_variables, + create_init_config_file, +) +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.exceptions import SerializerException +from pycompss.util.interactive.flags import check_flags +from pycompss.util.interactive.flags import print_flag_issues +from pycompss.util.interactive.utils import parameters_to_dict +from pycompss.util.logger.helpers import clean_log_configs +from pycompss.util.logger.helpers import init_logging +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.process.manager import initialize_multiprocessing +from pycompss.util.process.preloader import preimports +from pycompss.util.process.preloader import preload_imports +from pycompss.util.storages.persistent import master_init_storage +from pycompss.util.storages.persistent import master_stop_storage + +# Storage imports +from pycompss.util.storages.persistent import use_storage + +# Tracing imports +from pycompss.util.tracing.helpers import EventMaster +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.typing_helper import typing +from pycompss.util.warnings.modules import show_optional_module_warnings + +# Spend less time in gc; do this before significant computation +gc.set_threshold(150000) + + +def stop_all(exit_code: int) -> None: + """Stop everything smoothly. + + :param exit_code: Exit code. + :return: None. + """ + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_stop, + ) + + # Stop streaming + if LAUNCH_STATUS.get_streaming(): + stop_streaming() + # Stop persistent storage + if LAUNCH_STATUS.get_persistent_storage(): + master_stop_storage(LAUNCH_STATUS.get_logger()) + compss_stop(exit_code) + sys.stdout.flush() + sys.stderr.flush() + sys.exit(exit_code) + + +def parse_arguments() -> argparse.Namespace: + """Parse PyCOMPSs arguments. + + :return: Argument's parser. + """ + parser = argparse.ArgumentParser( + description="PyCOMPSs application launcher" + ) + parser.add_argument( + "wall_clock", + help="Application Wall Clock limit " + "[wall_clock<=0 deactivated|wall_clock>0 max duration in seconds]", + ) + parser.add_argument( + "log_level", help="Logging level [trace|debug|api|info|off]" + ) + parser.add_argument("tracing", help="Tracing [True | False]") + parser.add_argument( + "object_conversion", help="Object_conversion [true|false]" + ) + parser.add_argument( + "storage_configuration", help="Storage configuration [null|*]" + ) + parser.add_argument("streaming_backend", help="Streaming Backend [null|*]") + parser.add_argument( + "streaming_master_name", help="Streaming Master Name [*]" + ) + parser.add_argument( + "streaming_master_port", help="Streaming Master Port [*]" + ) + parser.add_argument("app_path", help="Application path") + return parser.parse_args() + + +def __load_user_module(app_path: str, log_level: str) -> None: + """Load the user module (resolve all user imports). + + This has shown to be necessary before doing "start_compss" in order + to avoid segmentation fault in some libraries. + + :param app_path: Path to the file to be imported. + :param log_level: Logging level. + :return: None. + """ + app_name = os.path.basename(app_path).split(".")[0] + try: + from importlib.machinery import SourceFileLoader # noqa + + _ = SourceFileLoader(app_name, app_path).load_module() + except Exception: # pylint: disable=broad-except + # Ignore any exception to try to run. + # This exception can be produce for example with applications + # that have code replacer and have imports to code that does not + # exist (e.g. using autoparallel) + if log_level != "off": + print( + "WARNING: Could not load the application " + "(this may be the cause of a running exception." + ) + + +def __register_implementation_core_elements() -> None: + """Register all implementations accumulated during initialization. + + Register the @implements core elements accumulated during the + initialization of the @implements decorators. They have not been + registered because the runtime was not started. And the load is + necessary to resolve all user imports before starting the runtime (it has + been found that starting the runtime and loading the user code may lead + to import errors with some libraries - reason: unknown). + + :return: None + """ + task_list = CONTEXT.get_to_register() + for task, impl_signature in task_list: + task.register_task() + task.decorated_function.registered = True + task.decorated_function.signature = impl_signature + + +def compss_main() -> None: + """Execute the given application and flags with PyCOMPSs. + + General call: + python $PYCOMPSS_HOME/pycompss/runtime/launch.py $wall_clock $log_level + $PyObject_serialize $storage_conf $streaming_backend + $streaming_master_name $streaming_master_port + $fullAppPath $application_args + + :return: None. + """ + # Let the Python binding know we are at master + CONTEXT.set_master() + # Then we can import the appropriate start and stop functions from the API + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_start, + ) + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_set_wall_clock, + ) + + # See parse_arguments, defined above + # In order to avoid parsing user arguments, we are going to remove user + # args from sys.argv + user_sys_argv = sys.argv[10:] + sys.argv = sys.argv[:10] + args = parse_arguments() + # We are done, now sys.argv must contain user args only + sys.argv = [args.app_path] + user_sys_argv + + # Get log_level + log_level = args.log_level + + # Setup tracing + tracing = args.tracing == "true" + + # Get storage configuration at master + storage_conf = args.storage_configuration + + # Load user imports before starting the runtime (can be avoided if + # ENVIRONMENT_VARIABLE_LOAD -- defined in configuration.py -- + # is set to false). + # Reason: some cases like autoparallel can require to avoid loading. + # It is disabled if using storage (with dataClay this can not be done) + if preload_user_code() and not use_storage(storage_conf): + with loading_context(): + __load_user_module(args.app_path, log_level) + + # Start the runtime + compss_start(log_level, tracing, False) + + # Register @implements core elements (they can not be registered in + # __load_user__module__). + __register_implementation_core_elements() + + # Get application wall clock limit + wall_clock = int(args.wall_clock) + if wall_clock > 0: + compss_set_wall_clock(wall_clock) + + # Get object_conversion boolean + TASK_FEATURES.set_object_conversion(args.object_conversion == "true") + + # Get application execution path + LAUNCH_STATUS.set_app_path(args.app_path) + + # Setup logging + binding_log_path = get_log_path() + GLOBALS.set_log_directory(binding_log_path) + init_logging(LOG_REMITTENT.MASTER, log_level, binding_log_path) + logger = logging.getLogger("pycompss.runtime.launch") + LAUNCH_STATUS.set_logger(logger) + + # Setup tmp path + binding_tmp_path = get_tmp_path() # master.workingDir + GLOBALS.set_temporary_directory(binding_tmp_path) + + # Pre-load imports + if preimports(): + if __debug__: + logger.debug("Preloading imports") + preload_imports(logger, "", "") + + # Get JVM options + # jvm_opts = os.environ["JVM_OPTIONS_FILE"] + # from pycompss.util.jvm.parser import convert_to_dict + # opts = convert_to_dict(jvm_opts) + # storage_conf = opts.get("-Dcompss.storage.conf") + + exit_code = 0 + try: + if __debug__: + logger.debug("--- START ---") + logger.debug("PyCOMPSs Log path: %s", binding_log_path) + + # Start persistent storage + persistent_storage = master_init_storage(storage_conf, logger) + LAUNCH_STATUS.set_persistent_storage(persistent_storage) + + # Start streaming + streaming = init_streaming( + args.streaming_backend, + args.streaming_master_name, + args.streaming_master_port, + ) + LAUNCH_STATUS.set_streaming(streaming) + + # Show module warnings + if __debug__: + show_optional_module_warnings() + + # MAIN EXECUTION + app_path = args.app_path + with EventMaster(TRACING_MASTER.application_running_event): + # MAIN EXECUTION + with open(app_path) as user_file: + exec( # nosec B102 # need to run the user application + compile(user_file.read(), app_path, "exec"), globals() + ) + + # End + if __debug__: + logger.debug("--- END ---") + except SystemExit as system_exit: + # Re-raising would not allow to stop the runtime gracefully. + if system_exit.code != 0: + print( + "[ ERROR ]: User program ended with exitcode %s.", + system_exit.code, + ) + print("\t\tShutting down runtime...") + if system_exit.code is None: + exit_code = -1 + elif isinstance(system_exit.code, str): + exit_code = -1 + else: + exit_code = system_exit.code + except SerializerException: + exit_code = 1 + # If an object that can not be serialized has been used as a parameter. + print("[ ERROR ]: Serialization exception") + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + for line in lines: + if app_path in line: + print("[ ERROR ]: In: %s", line) + exit_code = 1 + except COMPSsException as compss_exception: + # Any other exception occurred + print( + "[ ERROR ]: A COMPSs exception occurred: %s", str(compss_exception) + ) + traceback.print_exc() + exit_code = 0 # COMPSs exception is not considered an error + except Exception as general_exception: # pylint: disable=broad-except + # Any other exception occurred + print("[ ERROR ]: An exception occurred: %s" + str(general_exception)) + traceback.print_exc() + exit_code = 1 + finally: + # Stop runtime + stop_all(exit_code) + clean_log_configs() + # --- Execution finished --- + + +# ###################################################### # +# ------ FOR EXTERNAL EXECUTION ------ # +# Starts a new COMPSs runtime and calls the application. # +# ###################################################### # + + +def launch_pycompss_application( + app: str, + func: typing.Optional[str], + log_level: str = "off", + o_c: bool = False, + debug: bool = False, + graph: bool = False, + trace: bool = False, + monitor: int = -1, + project_xml: str = "", + resources_xml: str = "", + summary: bool = False, + task_execution: str = "compss", + storage_impl: str = "", + storage_conf: str = "", + streaming_backend: str = "", + streaming_master_name: str = "", + streaming_master_port: str = "", + task_count: int = 50, + app_name: str = "", + uuid: str = "", + log_dir: str = "", + master_working_dir: str = "", + extrae_cfg: str = "", + extrae_final_directory: str = "", + comm: str = "NIO", + conn: str = CONSTANTS.default_conn, + master_name: str = "", + master_port: str = "", + scheduler: str = CONSTANTS.default_sched, + jvm_workers: str = CONSTANTS.default_jvm_workers, + cpu_affinity: str = "automatic", + gpu_affinity: str = "automatic", + fpga_affinity: str = "automatic", + fpga_reprogram: str = "", + profile_input: str = "", + profile_output: str = "", + scheduler_config: str = "", + external_adaptation: bool = False, + propagate_virtual_environment: bool = True, + mpi_worker: bool = False, + worker_cache: typing.Union[bool, str] = False, + shutdown_in_node_failure: bool = False, + io_executors: int = 0, + env_script: str = "", + reuse_on_block: bool = True, + nested_enabled: bool = False, + tracing_task_dependencies: bool = False, + trace_label: str = "", + extrae_cfg_python: str = "", + wcl: int = 0, + cache_profiler: bool = False, + ear: bool = False, + data_provenance: bool = False, + checkpoint_policy: str = "es.bsc.compss.checkpoint.policies.NoCheckpoint", + checkpoint_params: str = "", + checkpoint_folder: str = "", + *args: typing.Any, + **kwargs: typing.Any +) -> typing.Any: + """Launch PyCOMPSs application from function. + + :param app: Application path + :param func: Function + :param log_level: Logging level [ "trace"|"debug"|"info"|"api"|"off" ] + (default: "off") + :param o_c: Objects to string conversion [ True | False ] (default: False) + :param debug: Debug mode [ True | False ] (default: False) + (overrides log_level) + :param graph: Generate graph [ True | False ] (default: False) + :param trace: Generate trace [ True | False ] (default: False) + :param monitor: Monitor refresh rate (default: None) + :param project_xml: Project xml file path + :param resources_xml: Resources xml file path + :param summary: Execution summary [ True | False ] (default: False) + :param task_execution: Task execution (default: "compss") + :param storage_impl: Storage implementation path + :param storage_conf: Storage configuration file path + :param streaming_backend: Streaming backend (default: None) + :param streaming_master_name: Streaming master name (default: None) + :param streaming_master_port: Streaming master port (default: None) + :param task_count: Task count (default: 50) + :param app_name: Application name (default: Interactive_date) + :param uuid: UUId + :param log_dir: Logging directory + :param master_working_dir: Master working directory + :param extrae_cfg: Extrae configuration file path + :param extrae_final_directory: Extrae final directory (default: "") + :param comm: Communication library (default: NIO) + :param conn: Connector (default: DefaultSSHConnector) + :param master_name: Master Name (default: "") + :param master_port: Master port (default: "") + :param scheduler: Scheduler (default: + es.bsc.compss.scheduler.lookahead.locality.LocalityTS) + :param jvm_workers: Java VM parameters + (default: "-Xms1024m,-Xmx1024m,-Xmn400m") + :param cpu_affinity: CPU Core affinity (default: "automatic") + :param gpu_affinity: GPU Core affinity (default: "automatic") + :param fpga_affinity: FPA Core affinity (default: "automatic") + :param fpga_reprogram: FPGA reprogram command (default: "") + :param profile_input: Input profile (default: "") + :param profile_output: Output profile (default: "") + :param scheduler_config: Scheduler configuration (default: "") + :param external_adaptation: External adaptation [ True | False ] + (default: False) + :param propagate_virtual_environment: Propagate virtual environment + [ True | False ] (default: False) + :param mpi_worker: Use the MPI worker [ True | False ] (default: False) + :param worker_cache: Use the worker cache [ True | int(size) | False] + (default: False) + :param shutdown_in_node_failure: Shutdown in node failure [ True | False] + (default: False) + :param io_executors: Number of IO executors + :param env_script: Environment script to be sourced in workers + :param reuse_on_block: Reuse on block [ True | False] (default: True) + :param nested_enabled: Nested enabled [ True | False] (default: False) + :param tracing_task_dependencies: Include task dependencies in trace + [ True | False] (default: False) + :param trace_label: Add trace label + :param extrae_cfg_python: Extrae configuration file for the + workers + :param wcl: Wallclock limit. Stops the runtime if reached. + 0 means forever. + :param cache_profiler: Use the cache profiler [ True | False ] + (default: False) + :param ear: Use EAR [ True | False ] (default: False) + :param data_provenance: Enable data provenance [ True | False ] + (default: False) + :param checkpoint_policy: Checkpointing policy. + (default: "es.bsc.compss.checkpoint. + policies.NoCheckpoint") + :param checkpoint_params: Checkpointing parameters. + (default: "") + :param checkpoint_folder: Checkpointing folder. + (default: "") + :param args: Positional arguments + :param kwargs: Named arguments + :return: Execution result + """ + # Check that COMPSs is available + if "COMPSS_HOME" not in os.environ: + # Do not allow to continue if COMPSS_HOME is not defined + raise PyCOMPSsException( + "ERROR: COMPSS_HOME is not defined in the environment" + ) + + # Let the Python binding know we are at master + CONTEXT.set_master() + # Then we can import the appropriate start and stop functions from the API + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_start, + ) + from pycompss.api.api import ( # pylint: disable=import-outside-toplevel + compss_stop, + ) + + ############################################################## + # INITIALIZATION + ############################################################## + + if debug: + log_level = "debug" + + # Initial dictionary with the user defined parameters + all_vars = parameters_to_dict( + log_level, + debug, + o_c, + graph, + trace, + monitor, + project_xml, + resources_xml, + summary, + task_execution, + storage_impl, + storage_conf, + streaming_backend, + streaming_master_name, + streaming_master_port, + task_count, + app_name, + uuid, + log_dir, + master_working_dir, + extrae_cfg, + extrae_final_directory, + comm, + conn, + master_name, + master_port, + scheduler, + jvm_workers, + cpu_affinity, + gpu_affinity, + fpga_affinity, + fpga_reprogram, + profile_input, + profile_output, + scheduler_config, + external_adaptation, + propagate_virtual_environment, + mpi_worker, + worker_cache, + shutdown_in_node_failure, + io_executors, + env_script, + reuse_on_block, + nested_enabled, + tracing_task_dependencies, + trace_label, + extrae_cfg_python, + wcl, + cache_profiler, + ear, + data_provenance, + checkpoint_policy, + checkpoint_params, + checkpoint_folder, + ) + # Save all vars in global current flags so that events.py can restart + # the notebook with the same flags + export_current_flags(all_vars) + + # Check the provided flags + flags, issues = check_flags(all_vars) + if not flags: + print_flag_issues(issues) + return None + + # Prepare the environment + env_vars = prepare_environment( + False, o_c, storage_impl, app, debug, mpi_worker + ) + all_vars.update(env_vars) + + monitoring_vars = prepare_loglevel_graph_for_monitoring( + monitor, graph, debug, log_level + ) + all_vars.update(monitoring_vars) + + if CONSTANTS.running_in_supercomputer: + updated_vars = updated_variables_in_sc() + all_vars.update(updated_vars) + + all_vars["ld_library_path"] = prepare_tracing_environment( + all_vars["trace"], all_vars["extrae_lib"], all_vars["ld_library_path"] + ) + + inf_vars = check_infrastructure_variables( + all_vars["project_xml"], + all_vars["resources_xml"], + all_vars["compss_home"], + all_vars["app_name"], + all_vars["file_name"], + all_vars["external_adaptation"], + ) + all_vars.update(inf_vars) + + create_init_config_file(**all_vars) + + ############################################################## + # RUNTIME START + ############################################################## + + # Runtime start + compss_start(log_level, all_vars["trace"], True) + + # Setup logging + binding_log_path = get_log_path() + GLOBALS.set_log_directory(binding_log_path) + init_logging(LOG_REMITTENT.MASTER, log_level, binding_log_path) + logger = logging.getLogger("pycompss.runtime.launch") + + # Setup tmp path + binding_tmp_path = get_tmp_path() # master.workingDir + GLOBALS.set_temporary_directory(binding_tmp_path) + + logger.debug("--- START ---") + logger.debug("PyCOMPSs Log path: %s", binding_log_path) + + if storage_impl and storage_conf: + logger.debug("Starting storage") + persistent_storage = master_init_storage( + all_vars["storage_conf"], logger + ) + else: + persistent_storage = False + + logger.debug("Starting streaming") + streaming = init_streaming( + all_vars["streaming_backend"], + all_vars["streaming_master_name"], + all_vars["streaming_master_port"], + ) + + saved_argv = sys.argv + sys.argv = list(args) + # Execution: + with EventMaster(TRACING_MASTER.application_running_event): + if func is None or func == "__main__": + with open(app, "r") as app_fd: + exec( # nosec B102 # need to run the user application + app_fd.read() + ) + result = None + else: + from importlib.machinery import SourceFileLoader # noqa + + imported_module = SourceFileLoader( + all_vars["file_name"], app + ).load_module() + method_to_call = getattr(imported_module, func) + try: + result = method_to_call(*args, **kwargs) + except TypeError: + result = method_to_call() + # Recover the system arguments + sys.argv = saved_argv + + # Stop streaming + if streaming: + stop_streaming() + + # Stop persistent storage + if persistent_storage: + master_stop_storage(logger) + + logger.debug("--- END ---") + + ############################################################## + # RUNTIME STOP + ############################################################## + + # Stop runtime + compss_stop() + clean_log_configs() + + return result + + +if __name__ == "__main__": + # This is the PyCOMPSs entry point. + + # Initialize multiprocessing + initialize_multiprocessing() + + compss_main() diff --git a/examples/dds/pycompss/runtime/management/COMPSs.py b/examples/dds/pycompss/runtime/management/COMPSs.py new file mode 100644 index 00000000..48e2a829 --- /dev/null +++ b/examples/dds/pycompss/runtime/management/COMPSs.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - COMPSs Runtime. + +This file contains the COMPSs runtime connection. +Loads the external C module. +""" + +import logging +from pycompss.runtime.management.link.separate import ( + establish_interactive_link, +) +from pycompss.runtime.management.link.direct import establish_link +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + LOGGER = logging.getLogger(__name__) + + +class COMPSsModule: # pylint: disable=invalid-name, too-many-public-methods + """C module extension for the communication with the runtime. + + See ext/compssmodule.cc + + Keep the COMPSs runtime link in this module so that any module can access + it through the module methods. + """ + + __slots__ = ["compss", "stdout", "stderr"] + + def __init__(self) -> None: + """Create a new COMPSs object.""" + # COMPSs connection + self.compss = None # type: typing.Any + # Files where the std may be redirected with interactive + self.stdout = "" + self.stderr = "" + + ###################################################### + # INTERNAL FUNCTIONS # + ###################################################### + + def load_runtime( + self, + external_process: bool = False, + _logger: typing.Optional[logging.Logger] = None, + ) -> None: + """Load the external C extension module. + + :param external_process: Loads the runtime in an external process + if true. + Within this python process if false. + :param _logger: Use this logger instead of the module LOGGER. + :return: None + """ + if external_process: + # For interactive python environments + self.compss, self.stdout, self.stderr = establish_interactive_link( + _logger, True + ) + else: + # Normal python environments + self.compss = establish_link(_logger) + + def is_redirected(self) -> bool: + """Check if the stdout and stderr are being redirected. + + :return: If stdout/stderr are being redirected. + """ + if self.stdout == "" and self.stderr == "": + return False + if self.stdout != "" and self.stderr != "": + return True + message = "Inconsistent status of STDOUT and STDERR" + raise PyCOMPSsException(message) + + def get_redirection_file_names(self) -> typing.Tuple[str, str]: + """Retrieve the stdout and stderr file names. + + :return: The stdout and stderr file names. + """ + if self.is_redirected(): + return self.stdout, self.stderr + message = "The runtime STDOUT and STDERR are not being redirected." + raise PyCOMPSsException(message) + + ###################################################### + # COMPSs API EXPOSED FUNCTIONS # + ###################################################### + + def start_runtime(self) -> None: + """Call to start_runtime. + + :return: None + """ + self.compss.start_runtime() + + def set_debug(self, mode: bool) -> None: + """Call to set_debug. + + :param mode: Debug mode ( True | False ). + :return: None. + """ + self.compss.set_debug(mode) + + def stop_runtime(self, code: int) -> None: + """Call to stop_runtime. + + :param code: Stopping code. + :return: None. + """ + self.compss.stop_runtime(code) + + def cancel_application_tasks(self, app_id: int, value: int) -> None: + """Call to cancel_application_tasks. + + :param app_id: Application identifier. + :param value: Task identifier. + :return: None. + """ + self.compss.cancel_application_tasks(app_id, value) + + def accessed_file(self, app_id: int, file_name: str) -> bool: + """Call to accessed_file. + + :param app_id: Application identifier. + :param file_name: File name to check if accessed. + :return: If the file has been accessed. + """ + return self.compss.accessed_file(app_id, file_name) + + def open_file(self, app_id: int, file_name: str, mode: int) -> str: + """Call to open_file. + + Synchronizes if necessary. + + :param app_id: Application identifier. + :param file_name: File name reference to open. + :param mode: Open mode. + :return: The real file name. + """ + return self.compss.open_file(app_id, file_name, mode) + + def close_file(self, app_id: int, file_name: str, mode: int) -> None: + """Call to close_file. + + :param app_id: Application identifier. + :param file_name: File name reference to close. + :param mode: Close mode. + :return: None + """ + self.compss.close_file(app_id, file_name, mode) + + def delete_file( + self, app_id: int, file_name: str, mode: bool, application_delete=True + ) -> bool: + """Call to delete_file. + + :param app_id: Application identifier. + :param file_name: File name reference to delete. + :param mode: Delete mode. + :param application_delete: Application delete. + :return: The deletion result. + """ + result = self.compss.delete_file( + app_id, file_name, mode, application_delete + ) + if result is None: + return False + return result + + def get_file(self, app_id: int, file_name: str) -> None: + """Call to (synchronize) get_file. + + :param app_id: Application identifier. + :param file_name: File name reference to get. + :return: None. + """ + self.compss.get_file(app_id, file_name) + + def get_directory(self, app_id: int, directory_name: str) -> None: + """Call to (synchronize) get_directory. + + :param app_id: Application identifier. + :param directory_name: Directory name reference to get. + :return: None. + """ + self.compss.get_directory(app_id, directory_name) + + def barrier(self, app_id: int, no_more_tasks: bool) -> None: + """Call to barrier. + + :param app_id: Application identifier. + :param no_more_tasks: No more tasks boolean. + :return: None + """ + self.compss.barrier(app_id, no_more_tasks) + + def barrier_group(self, app_id: int, group_name: str) -> str: + """Call to barrier_group. + + :param app_id: Application identifier. + :param group_name: Group name. + :return: Exception message. + """ + return str(self.compss.barrier_group(app_id, group_name)) + + def open_task_group( + self, group_name: str, implicit_barrier: bool, app_id: int + ) -> None: + """Call to open_task_group. + + :param group_name: Group name. + :param implicit_barrier: Implicit barrier boolean. + :param app_id: Application identifier. + :return: None. + """ + self.compss.open_task_group(group_name, implicit_barrier, app_id) + + def close_task_group(self, group_name: str, app_id: int) -> None: + """Call to close_task_group. + + :param group_name: Group name. + :param app_id: Application identifier. + :return: None. + """ + self.compss.close_task_group(group_name, app_id) + + def cancel_task_group(self, group_name: str, app_id: int) -> str: + """Call to cancel_task_group. + + :param group_name: Group name. + :param app_id: Application identifier. + :return: Exception message. + """ + return str(self.compss.cancel_task_group(group_name, app_id)) + + def snapshot(self, app_id) -> None: + """Call to snapshot. + + :param app_id: Application identifier. + :return: None + """ + self.compss.snapshot(app_id) + + def get_logging_path(self) -> str: + """Call to get_logging_path. + + :return: The COMPSs log path. + """ + return self.compss.get_logging_path() + + def get_master_working_path(self) -> str: + """Call to get_master_working_path. + + :return: The COMPSs log path. + """ + return self.compss.get_master_working_path() + + def get_number_of_resources(self, app_id: int) -> int: + """Call to number_of_resources. + + :param app_id: Application identifier. + :return: Number of resources. + """ + return self.compss.get_number_of_resources(app_id) + + def request_resources( + self, app_id: int, num_resources: int, group_name: str + ) -> None: + """Call to request_resources. + + :param app_id: Application identifier. + :param num_resources: Number of resources. + :param group_name: Group name. + :return: None. + """ + self.compss.request_resources(app_id, num_resources, group_name) + + def free_resources( + self, app_id: int, num_resources: int, group_name: str + ) -> None: + """Call to free_resources. + + :param app_id: Application identifier. + :param num_resources: Number of resources. + :param group_name: Group name. + :return: None. + """ + self.compss.free_resources(app_id, num_resources, group_name) + + def set_wall_clock(self, app_id: float, wcl: float) -> None: + """Call to set_wall_clock. + + :param app_id: Application identifier. + :param wcl: Wall Clock limit in seconds. + :return: None. + """ + self.compss.set_wall_clock(app_id, wcl) + + def register_core_element( # pylint: disable=too-many-arguments + self, + ce_signature: str, + impl_signature: typing.Optional[str], + impl_constraints: typing.Optional[str], + impl_type: typing.Optional[str], + impl_local: str, + impl_io: str, + impl_prolog: typing.List[str], + impl_epilog: typing.List[str], + impl_container: typing.List[str], + impl_type_args: typing.List[str], + ) -> None: + """Call to register_core_element. + + :param ce_signature: Core element signature. + :param impl_signature: Implementation signature. + :param impl_constraints: Implementation constraints. + :param impl_type: Implementation type. + :param impl_local: Implementation Local. + :param impl_io: Implementation IO. + :param impl_prolog: [binary, params, fail_by_exit_value] of the prolog. + :param impl_epilog: [binary, params, fail_by_exit_value] of the epilog. + :param impl_container: [engine, image, options] of the container. + :param impl_type_args: Implementation type arguments. + :return: None. + """ + self.compss.register_core_element( + ce_signature, + impl_signature, + impl_constraints, + impl_type, + impl_local, + impl_io, + impl_prolog, + impl_epilog, + impl_container, + impl_type_args, + ) + + def process_task( # pylint: disable=too-many-arguments, too-many-locals + self, + app_id: int, + signature: str, + on_failure: str, + time_out: int, + has_priority: bool, + num_nodes: int, + reduction: bool, + chunk_size: int, + replicated: bool, + distributed: bool, + has_target: bool, + num_returns: int, + values: list, + names: list, + compss_types: list, + compss_directions: list, + compss_streams: list, + compss_prefixes: list, + content_types: list, + weights: list, + keep_renames: list, + ) -> None: + """Call to process_task. + + :param app_id: Application identifier. + :param signature: Task signature. + :param on_failure: On failure action. + :param time_out: Task time out. + :param has_priority: Boolean has priority. + :param num_nodes: Number of nodes. + :param reduction: Boolean indicating if the task is of type reduce. + :param chunk_size: Size of chunks for executing the reduce operation. + :param replicated: Boolean is replicated. + :param distributed: Boolean is distributed. + :param has_target: Boolean has target. + :param num_returns: Number of returns. + :param values: Values. + :param names: Names. + :param compss_types: COMPSs types. + :param compss_directions: COMPSs directions. + :param compss_streams: COMPSs streams. + :param compss_prefixes: COMPSs prefixes. + :param content_types: COMPSs types. + :param weights: Parameter weights. + :param keep_renames: Boolean keep renames. + :return: None. + """ + self.compss.process_task( + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + + def process_http_task( # pylint: disable=too-many-arguments, R0914 + self, + app_id: int, + signature: str, + on_failure: str, + time_out: int, + has_priority: bool, + num_nodes: int, + reduction: bool, + chunk_size: int, + replicated: bool, + distributed: bool, + has_target: bool, + num_returns: int, + values: list, + names: list, + compss_types: list, + compss_directions: list, + compss_streams: list, + compss_prefixes: list, + content_types: list, + weights: list, + keep_renames: list, + ) -> None: + """Call to process_http_task. + + :param app_id: Application identifier. + :param signature: Task signature. + :param on_failure: On failure action. + :param time_out: Task time out. + :param has_priority: Boolean has priority. + :param num_nodes: Number of nodes. + :param reduction: Boolean indicating if the task is of type reduce. + :param chunk_size: Size of chunks for executing the reduce operation. + :param replicated: Boolean is replicated. + :param distributed: Boolean is distributed. + :param has_target: Boolean has target. + :param num_returns: Number of returns. + :param values: Values. + :param names: Names. + :param compss_types: COMPSs types. + :param compss_directions: COMPSs directions. + :param compss_streams: COMPSs streams. + :param compss_prefixes: COMPSs prefixes. + :param content_types: COMPSs types. + :param weights: Parameter weights. + :param keep_renames: Boolean keep renames. + :return: None. + """ + self.compss.process_http_task( + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + + def set_pipes(self, pipe_in: str, pipe_out: str) -> None: + """Set nesting pipes. + + :param pipe_in: Input pipe. + :param pipe_out: Output pipe. + :return: None. + """ + self.compss.set_pipes(pipe_in, pipe_out) + + def read_pipes(self) -> str: + """Call to read_pipes. + + :return: The command read from the pipe. + """ + command = self.compss.read_pipes() + return command + + +# ################################################# # +# ########## MAIN EXTERNAL COMPSS MODULE ########## # +# ################################################# # + + +COMPSs = COMPSsModule() diff --git a/examples/dds/pycompss/runtime/management/__init__.py b/examples/dds/pycompss/runtime/management/__init__.py new file mode 100644 index 00000000..7c1022bf --- /dev/null +++ b/examples/dds/pycompss/runtime/management/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime object management helpers.""" diff --git a/examples/dds/pycompss/runtime/management/classes.py b/examples/dds/pycompss/runtime/management/classes.py new file mode 100644 index 00000000..0603d36e --- /dev/null +++ b/examples/dds/pycompss/runtime/management/classes.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Classes. + +This file contains the internal classes. +""" + + +class SupportedFunctionTypes: # pylint: disable=too-few-public-methods + """Used as enum to identify the function type.""" + + FUNCTION = 1 + INSTANCE_METHOD = 2 + CLASS_METHOD = 3 + + +class Future: # pylint: disable=too-few-public-methods + """Future object class definition.""" + + +class EmptyReturn: # pylint: disable=too-few-public-methods + """For functions with empty return.""" + + +FunctionType = SupportedFunctionTypes() diff --git a/examples/dds/pycompss/runtime/management/direction.py b/examples/dds/pycompss/runtime/management/direction.py new file mode 100644 index 00000000..6bcee32f --- /dev/null +++ b/examples/dds/pycompss/runtime/management/direction.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Direction. + +This file contains the Direction management functions. +""" + +from pycompss.api.parameter import DIRECTION + + +def get_compss_direction(access_mode: str) -> int: + """Get the COMPSs direction of the given access_mode string. + + :param access_mode: String to parse and return the direction. + :return: Direction object (IN/INOUT/OUT). + """ + if access_mode.startswith("w"): + return DIRECTION.OUT + if access_mode.startswith("r+") or access_mode.startswith("a"): + return DIRECTION.INOUT + if access_mode.startswith("cv"): + return DIRECTION.COMMUTATIVE + if access_mode.startswith("c"): + return DIRECTION.CONCURRENT + return DIRECTION.IN diff --git a/examples/dds/pycompss/runtime/management/link/__init__.py b/examples/dds/pycompss/runtime/management/link/__init__.py new file mode 100644 index 00000000..3e94ac27 --- /dev/null +++ b/examples/dds/pycompss/runtime/management/link/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime management link helpers.""" diff --git a/examples/dds/pycompss/runtime/management/link/direct.py b/examples/dds/pycompss/runtime/management/link/direct.py new file mode 100644 index 00000000..6aa864d4 --- /dev/null +++ b/examples/dds/pycompss/runtime/management/link/direct.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Link Direct. + +This file contains the functions to link with the binding-commons directly. +""" + +import logging +from pycompss.util.typing_helper import typing + +if __debug__: + link_logger = logging.getLogger(__name__) + + +def establish_link( + logger: typing.Optional[logging.Logger] = None, +) -> typing.Any: + """Load the compss C extension within the same process. + + Does not implement support for stdout and stderr redirecting as the + establish_interactive_link. + + :param logger: Use this logger instead of the module logger. + :return: The COMPSs C extension link. + """ + if __debug__: + message = "Loading compss extension" + if logger: + logger.debug(message) + else: + link_logger.debug(message) + import compss # pylint: disable=import-outside-toplevel + + if __debug__: + message = "Loaded compss extension" + if logger: + logger.debug(message) + else: + link_logger.debug(message) + return compss diff --git a/examples/dds/pycompss/runtime/management/link/messages.py b/examples/dds/pycompss/runtime/management/link/messages.py new file mode 100644 index 00000000..bf98b70e --- /dev/null +++ b/examples/dds/pycompss/runtime/management/link/messages.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Link messages. + +This file contains the messages needed to link with the compss extension. +""" + + +class LinkMessages: + """Link messages definitions (used through the queues).""" + + __slots__ = [ + "start", + "set_debug", + "stop", + "cancel_tasks", + "accessed_file", + "open_file", + "close_file", + "delete_file", + "get_file", + "get_directory", + "barrier", + "barrier_group", + "open_task_group", + "close_task_group", + "cancel_task_group", + "snapshot", + "get_logging_path", + "get_master_working_path", + "get_number_of_resources", + "request_resources", + "free_resources", + "register_core_element", + "process_http_task", + "process_task", + "set_pipes", + "read_pipes", + "set_wall_clock", + "command_done", + ] + + def __init__(self) -> None: + """Instantiate a new link messages object.""" + self.start = "START" + self.set_debug = "SET_DEBUG" + self.stop = "STOP" + self.cancel_tasks = "CANCEL_TASKS" + self.accessed_file = "ACCESSED_FILE" + self.open_file = "OPEN_FILE" + self.close_file = "CLOSE_FILE" + self.delete_file = "DELETE_FILE" + self.get_file = "GET_FILE" + self.get_directory = "GET_DIRECTORY" + self.barrier = "BARRIER" + self.barrier_group = "BARRIER_GROUP" + self.open_task_group = "OPEN_TASK_GROUP" + self.close_task_group = "CLOSE_TASK_GROUP" + self.cancel_task_group = "CANCEL_TASK_GROUP" + self.snapshot = "SNAPSHOT" + self.get_logging_path = "GET_LOGGING_PATH" + self.get_master_working_path = "GET_MASTER_WORKING_PATH" + self.get_number_of_resources = "GET_NUMBER_OF_RESOURCES" + self.request_resources = "REQUEST_RESOURCES" + self.free_resources = "FREE_RESOURCES" + self.register_core_element = "REGISTER_CORE_ELEMENT" + self.process_http_task = "PROCESS_HTTP_TASK" + self.process_task = "PROCESS_TASK" + self.set_pipes = "SET_PIPES" + self.read_pipes = "READ_PIPES" + self.set_wall_clock = "SET_WALL_CLOCK" + self.command_done = "COMMAND_DONE" # Default response message + + +LINK_MESSAGES = LinkMessages() diff --git a/examples/dds/pycompss/runtime/management/link/separate.py b/examples/dds/pycompss/runtime/management/link/separate.py new file mode 100644 index 00000000..ca3ae067 --- /dev/null +++ b/examples/dds/pycompss/runtime/management/link/separate.py @@ -0,0 +1,813 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Link. + +This file contains the functions to link with the binding-commons. +In particular, manages a separate process which handles the compss +extension, so that the process can be removed when shutting off +and restarted (interactive usage of PyCOMPSs - ipython and jupyter). +""" + +import os +import logging +import signal + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.process.manager import Queue +from pycompss.util.process.manager import create_process +from pycompss.util.process.manager import new_process +from pycompss.util.process.manager import new_queue +from pycompss.util.std.redirects import ipython_std_redirector +from pycompss.util.std.redirects import not_std_redirector +from pycompss.util.typing_helper import typing +from pycompss.runtime.management.link.messages import LINK_MESSAGES + +if __debug__: + link_logger = logging.getLogger(__name__) + + +def shutdown_handler( + signal: int, # pylint: disable=unused-argument, redefined-outer-name + frame: typing.Any, # pylint: disable=unused-argument +) -> None: + """Shutdown handler. + + Do not remove the parameters. + + :param signal: shutdown signal. + :param frame: Frame. + :return: None + """ + if EXTERNAL_LINK.link_process.is_alive(): + EXTERNAL_LINK.terminate_interactive_link() + + +def establish_interactive_link( + logger: typing.Optional[logging.Logger] = None, redirect_std: bool = False +) -> typing.Tuple[typing.Any, str, str]: + """Start a process that communicates with the C-extension. + + Starts a new process which will be in charge of communicating with + the C-extension. + + It will return stdout file name and stderr file name as None if + redirect_std is False. Otherwise, returns the names which are the + current process pid followed by the out/err extension. + + :param logger: Use this logger instead of the module logger. + :param redirect_std: Decide whether to store the stdout and stderr into + files or not. + :return: The COMPSs C extension link, stdout file name and stderr file + name. + """ + return EXTERNAL_LINK.establish_interactive_link(logger, redirect_std) + + +class ExternalLink: + """External link class.""" + + __slots__ = ["link_process", "in_queue", "out_queue", "reload"] + + def __init__(self) -> None: + """Instantiate a new ExternalLink class.""" + self.link_process = new_process() + self.in_queue = new_queue() + self.out_queue = new_queue() + self.reload = False + + def establish_interactive_link( + self, + logger: typing.Optional[logging.Logger] = None, + redirect_std: bool = False, + ) -> typing.Tuple[typing.Any, str, str]: + """Start a process that communicates with the C-extension. + + Start a new process which will be in charge of communicating with + the C-extension. + + It will return stdout file name and stderr file name as None if + redirect_std is False. Otherwise, returns the names which are the + current process pid followed by the out/err extension. + + :param logger: Use this logger instead of the module logger. + :param redirect_std: Decide whether to store the stdout and stderr into + files or not. + :return: The COMPSs C extension link, stdout file name and stderr file + name. + """ + out_file_name = "" + err_file_name = "" + if redirect_std: + pid = str(os.getpid()) + out_file_name = "compss-" + pid + ".out" + err_file_name = "compss-" + pid + ".err" + + if self.reload: + self.in_queue = new_queue() + self.out_queue = new_queue() + self.reload = False + + if __debug__: + message = "Starting new process linking with the C-extension" + if logger: + logger.debug(message) + else: + link_logger.debug(message) + + self.link_process = create_process( + target=c_extension_link, + args=( + self.in_queue, + self.out_queue, + redirect_std, + out_file_name, + err_file_name, + ), + ) + signal.signal(signal.SIGTERM, shutdown_handler) + self.link_process.start() + + if __debug__: + message = "Established link with C-extension" + if logger: + logger.debug(message) + else: + link_logger.debug(message) + + # Create object that mimics compss library + compss_link = _COMPSs(self.in_queue, self.out_queue) + return compss_link, out_file_name, err_file_name + + def wait_for_interactive_link(self) -> None: + """Wait for interactive link finalization. + + :return: None + """ + # Wait for the link to finish + self.in_queue.close() + self.out_queue.close() + self.in_queue.join_thread() + self.out_queue.join_thread() + self.link_process.join() + # Notify that if terminated, the queues must be reopened to start again + self.reload = True + + def terminate_interactive_link(self) -> None: + """Terminate the compss C extension process. + + :return: None + """ + self.link_process.terminate() + + +# ############################################################# # +# #################### EXTERNAL LINK CLASS #################### # +# ############################################################# # + +EXTERNAL_LINK = ExternalLink() + + +def c_extension_link( # pylint: disable=too-many-locals + in_queue: Queue, + out_queue: Queue, + redirect_std: bool, + out_file_name: str, + err_file_name: str, +) -> None: + """Establish C extension within external process. + + Establish C extension within an external process and communicates + through queues. + + :param lock: Global lock for + :param in_queue: Queue to receive messages. + :param out_queue: Queue to send messages. + :param redirect_std: Decide whether to store the stdout and stderr into + files or not. + :param out_file_name: File where to store the stdout (only required if + redirect_std is True). + :param err_file_name: File where to store the stderr (only required if + redirect_std is True). + :return: None + """ + # Import C extension within the external process + import compss # pylint: disable=import-outside-toplevel + + command_done = LINK_MESSAGES.command_done + + with ( + ipython_std_redirector(out_file_name, err_file_name) + if redirect_std + else not_std_redirector() + ): + alive = True + while alive: + message = list(in_queue.get()) + command = str(message[0]) + parameters = [] # type: list + if len(message) > 0: + parameters = list(message[1:]) + if command == LINK_MESSAGES.start: + compss.start_runtime() + out_queue.put(command_done) + elif command == LINK_MESSAGES.set_debug: + compss.set_debug(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.stop: + compss.stop_runtime(*parameters) + alive = False + out_queue.put(command_done) + elif command == LINK_MESSAGES.cancel_tasks: + compss.cancel_application_tasks(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.accessed_file: + accessed = compss.accessed_file(*parameters) + out_queue.put(accessed) + elif command == LINK_MESSAGES.open_file: + compss_name = compss.open_file(*parameters) + out_queue.put(compss_name) + elif command == LINK_MESSAGES.close_file: + compss.close_file(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.delete_file: + result = compss.delete_file(*parameters) + out_queue.put(result) + elif command == LINK_MESSAGES.get_file: + compss.get_file(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.get_directory: + compss.get_directory(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.barrier: + compss.barrier(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.barrier_group: + exception_message = compss.barrier_group(*parameters) + out_queue.put(exception_message) + elif command == LINK_MESSAGES.open_task_group: + compss.open_task_group(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.close_task_group: + compss.close_task_group(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.cancel_task_group: + compss.cancel_task_group(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.snapshot: + compss.snapshot(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.get_logging_path: + log_path = compss.get_logging_path() + out_queue.put(log_path) + elif command == LINK_MESSAGES.get_master_working_path: + master_working_path = compss.get_master_working_path() + out_queue.put(master_working_path) + elif command == LINK_MESSAGES.get_number_of_resources: + num_resources = compss.get_number_of_resources(*parameters) + out_queue.put(num_resources) + elif command == LINK_MESSAGES.request_resources: + compss.request_resources(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.free_resources: + compss.free_resources(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.register_core_element: + compss.register_core_element(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.process_task: + compss.process_task(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.process_http_task: + compss.process_http_task(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.set_pipes: + compss.set_pipes(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.read_pipes: + compss.read_pipes(*parameters) + out_queue.put(command_done) + elif command == LINK_MESSAGES.set_wall_clock: + compss.set_wall_clock(*parameters) + out_queue.put(command_done) + else: + raise PyCOMPSsException("Unknown link command") + + +class _COMPSs: + """Class that mimics the compss extension library. + + Each function puts into the queue a list or set composed by: + (COMMAND_TAG, parameter1, parameter2, ...) + + IMPORTANT: methods must be exactly the same. + """ + + __slots__ = ["in_queue", "out_queue"] + + def __init__(self, in_queue: Queue, out_queue: Queue) -> None: + """Instantiate a new _COMPSs object.""" + self.in_queue = in_queue + self.out_queue = out_queue + + def start_runtime(self) -> None: + """Call to start_runtime. + + :return: None + """ + self.in_queue.put([LINK_MESSAGES.start]) + _ = self.out_queue.get(block=True) + + def set_debug(self, mode: bool) -> None: + """Call to set_debug. + + :param mode: Debug mode ( True | False ). + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.set_debug, mode)) + _ = self.out_queue.get(block=True) + + def stop_runtime(self, code: int) -> None: + """Call to stop_runtime. + + :param code: Stopping code. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.stop, code)) + _ = self.out_queue.get(block=True) + EXTERNAL_LINK.wait_for_interactive_link() + # EXTERNAL_LINK.terminate_interactive_link() + + def cancel_application_tasks(self, app_id: int, value: int) -> None: + """Call to cancel_application_tasks. + + :param app_id: Application identifier. + :param value: Task identifier. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.cancel_tasks, app_id, value)) + _ = self.out_queue.get(block=True) + + def accessed_file(self, app_id: int, file_name: str) -> bool: + """Call to accessed_file. + + :param app_id: Application identifier. + :param file_name: File name to check if accessed. + :return: If the file has been accessed. + """ + self.in_queue.put((LINK_MESSAGES.accessed_file, app_id, file_name)) + accessed = self.out_queue.get(block=True) + return accessed + + def open_file(self, app_id: int, file_name: str, mode: int) -> str: + """Call to open_file. + + Synchronizes if necessary. + + :param app_id: Application identifier. + :param file_name: File name to open. + :param mode: Open mode. + :return: The real file name. + """ + self.in_queue.put((LINK_MESSAGES.open_file, app_id, file_name, mode)) + compss_name = self.out_queue.get(block=True) + return compss_name + + def close_file(self, app_id: int, file_name: str, mode: int) -> None: + """Call to close_file. + + :param app_id: Application identifier. + :param file_name: File name reference to close. + :param mode: Close mode. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.close_file, app_id, file_name, mode)) + _ = self.out_queue.get(block=True) + + def delete_file( + self, app_id: int, file_name: str, mode: bool, application_delete=True + ) -> bool: + """Call to delete_file. + + :param app_id: Application identifier. + :param file_name: File name reference to delete. + :param mode: Delete mode. + :param application_delete: Application delete. + :return: The deletion result. + """ + self.in_queue.put( + ( + LINK_MESSAGES.delete_file, + app_id, + file_name, + mode, + application_delete, + ) + ) + result = self.out_queue.get(block=True) + if result is None: + return False + return result + + def get_file(self, app_id: int, file_name: str) -> None: + """Call to (synchronize file) get_file. + + :param app_id: Application identifier. + :param file_name: File name reference to get. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.get_file, app_id, file_name)) + _ = self.out_queue.get(block=True) + + def get_directory(self, app_id: int, directory_name: str) -> None: + """Call to (synchronize directory) get_directory. + + :param app_id: Application identifier. + :param directory_name: Directory name reference to get. + :return: None. + """ + self.in_queue.put( + (LINK_MESSAGES.get_directory, app_id, directory_name) + ) + _ = self.out_queue.get(block=True) + + def barrier(self, app_id: int, no_more_tasks: bool) -> None: + """Call to barrier. + + :param app_id: Application identifier. + :param no_more_tasks: No more tasks boolean. + :return: None + """ + self.in_queue.put((LINK_MESSAGES.barrier, app_id, no_more_tasks)) + _ = self.out_queue.get(block=True) + + def barrier_group( + self, app_id: int, group_name: str + ) -> typing.Optional[str]: + """Call to barrier_group. + + :param app_id: Application identifier. + :param group_name: Group name. + :return: Exception message. + """ + self.in_queue.put((LINK_MESSAGES.barrier_group, app_id, group_name)) + exception_message = self.out_queue.get(block=True) + return exception_message + + def open_task_group( + self, group_name: str, implicit_barrier: bool, app_id: int + ) -> None: + """Call to open_task_group. + + :param group_name: Group name. + :param implicit_barrier: Implicit barrier boolean. + :param app_id: Application identifier. + :return: None. + """ + self.in_queue.put( + ( + LINK_MESSAGES.open_task_group, + group_name, + implicit_barrier, + app_id, + ) + ) + _ = self.out_queue.get(block=True) + + def close_task_group(self, group_name: str, app_id: int) -> None: + """Call to close_task_group. + + :param group_name: Group name. + :param app_id: Application identifier. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.close_task_group, group_name, app_id)) + _ = self.out_queue.get(block=True) + + def cancel_task_group(self, group_name: str, app_id: int) -> None: + """Call to cancel_task_group. + + :param group_name: Group name. + :param app_id: Application identifier. + :return: None. + """ + self.in_queue.put( + (LINK_MESSAGES.cancel_task_group, group_name, app_id) + ) + _ = self.out_queue.get(block=True) + + def snapshot(self, code: int) -> None: + """Call to snapshot. + + :param code: Stopping code. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.snapshot, code)) + _ = self.out_queue.get(block=True) + + def get_logging_path(self) -> str: + """Call to get_logging_path. + + :return: The COMPSs log path. + """ + self.in_queue.put([LINK_MESSAGES.get_logging_path]) + log_path = self.out_queue.get(block=True) + return log_path + + def get_master_working_path(self) -> str: + """Call to master_working_path. + + :return: The COMPSs master working path. + """ + self.in_queue.put([LINK_MESSAGES.get_master_working_path]) + master_working_path = self.out_queue.get(block=True) + return master_working_path + + def get_number_of_resources(self, app_id: int) -> int: + """Call to number_of_resources. + + :param app_id: Application identifier. + :return: Number of resources. + """ + self.in_queue.put((LINK_MESSAGES.get_number_of_resources, app_id)) + num_resources = self.out_queue.get(block=True) + return num_resources + + def request_resources( + self, app_id: int, num_resources: int, group_name: str + ) -> None: + """Call to request_resources. + + :param app_id: Application identifier. + :param num_resources: Number of resources. + :param group_name: Group name. + :return: None. + """ + self.in_queue.put( + ( + LINK_MESSAGES.request_resources, + app_id, + num_resources, + group_name, + ) + ) + _ = self.out_queue.get(block=True) + + def free_resources( + self, app_id: int, num_resources: int, group_name: str + ) -> None: + """Call to free_resources. + + :param app_id: Application identifier. + :param num_resources: Number of resources. + :param group_name: Group name. + :return: None. + """ + self.in_queue.put( + (LINK_MESSAGES.free_resources, app_id, num_resources, group_name) + ) + _ = self.out_queue.get(block=True) + + def set_wall_clock(self, app_id: int, wcl: int) -> None: + """Call to set_wall_clock. + + :param app_id: Application identifier. + :param wcl: Wall Clock limit in seconds. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.set_wall_clock, app_id, wcl)) + _ = self.out_queue.get(block=True) + + def register_core_element( + self, + ce_signature: str, + impl_signature: typing.Optional[str], + impl_constraints: typing.Optional[str], + impl_type: typing.Optional[str], + impl_local: str, + impl_io: str, + impl_prolog: typing.List[str], + impl_epilog: typing.List[str], + impl_container: typing.List[str], + impl_type_args: typing.List[str], + ) -> None: + """Call to register_core_element. + + :param ce_signature: Core element signature. + :param impl_signature: Implementation signature. + :param impl_constraints: Implementation constraints. + :param impl_type: Implementation type. + :param impl_local: Implementation Local. + :param impl_io: Implementation IO. + :param impl_prolog: [binary, params, fail_by_exit_value] of the prolog. + :param impl_epilog: [binary, params, fail_by_exit_value] of the epilog. + :param impl_container: [engine, image, options] of the container. + :param impl_type_args: Implementation type arguments. + :return: None. + """ + self.in_queue.put( + ( + LINK_MESSAGES.register_core_element, + ce_signature, + impl_signature, + impl_constraints, + impl_type, + impl_local, + impl_io, + impl_prolog, + impl_epilog, + impl_container, + impl_type_args, + ) + ) + _ = self.out_queue.get(block=True) + + def process_task( + self, + app_id: int, + signature: str, + on_failure: str, + time_out: int, + has_priority: bool, + num_nodes: int, + reduction: bool, + chunk_size: int, + replicated: bool, + distributed: bool, + has_target: bool, + num_returns: int, + values: list, + names: list, + compss_types: list, + compss_directions: list, + compss_streams: list, + compss_prefixes: list, + content_types: list, + weights: list, + keep_renames: list, + ) -> None: + """Call to process_task. + + :param app_id: Application identifier. + :param signature: Task signature. + :param on_failure: On failure action. + :param time_out: Task time out. + :param has_priority: Boolean has priority. + :param num_nodes: Number of nodes. + :param reduction: Boolean indicating if the task is of type reduce. + :param chunk_size: Size of chunks for executing the reduce operation. + :param replicated: Boolean is replicated. + :param distributed: Boolean is distributed. + :param has_target: Boolean has target. + :param num_returns: Number of returns. + :param values: Values. + :param names: Names. + :param compss_types: COMPSs types. + :param compss_directions: COMPSs directions. + :param compss_streams: COMPSs streams. + :param compss_prefixes: COMPSs prefixes. + :param content_types: COMPSs types. + :param weights: Parameter weights. + :param keep_renames: Boolean keep renames. + :return: None. + """ + self.in_queue.put( + ( + LINK_MESSAGES.process_task, + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + ) + _ = self.out_queue.get(block=True) + + def process_http_task( + self, + app_id: int, + signature: str, + on_failure: str, + time_out: int, + has_priority: bool, + num_nodes: int, + reduction: bool, + chunk_size: int, + replicated: bool, + distributed: bool, + has_target: bool, + num_returns: int, + values: list, + names: list, + compss_types: list, + compss_directions: list, + compss_streams: list, + compss_prefixes: list, + content_types: list, + weights: list, + keep_renames: list, + ) -> None: + """Call to process_http_task. + + :param app_id: Application identifier. + :param signature: Task signature. + :param on_failure: On failure action. + :param time_out: Task time out. + :param has_priority: Boolean has priority. + :param num_nodes: Number of nodes. + :param reduction: Boolean indicating if the task is of type reduce. + :param chunk_size: Size of chunks for executing the reduce operation. + :param replicated: Boolean is replicated. + :param distributed: Boolean is distributed. + :param has_target: Boolean has target. + :param num_returns: Number of returns. + :param values: Values. + :param names: Names. + :param compss_types: COMPSs types. + :param compss_directions: COMPSs directions. + :param compss_streams: COMPSs streams. + :param compss_prefixes: COMPSs prefixes. + :param content_types: COMPSs types. + :param weights: Parameter weights. + :param keep_renames: Boolean keep renames. + :return: None. + """ + self.in_queue.put( + ( + LINK_MESSAGES.process_http_task, + app_id, + signature, + on_failure, + time_out, + has_priority, + num_nodes, + reduction, + chunk_size, + replicated, + distributed, + has_target, + num_returns, + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) + ) + _ = self.out_queue.get(block=True) + + def set_pipes(self, pipe_in: str, pipe_out: str) -> None: + """Set nesting pipes. + + :param pipe_in: Input pipe. + :param pipe_out: Output pipe. + :return: None. + """ + self.in_queue.put((LINK_MESSAGES.set_pipes, pipe_in, pipe_out)) + _ = self.out_queue.get(block=True) + + def read_pipes(self) -> str: + """Call to read_pipes. + + :return: The command read from the pipe. + """ + self.in_queue.put([LINK_MESSAGES.read_pipes]) + command = self.out_queue.get(block=True) + return command diff --git a/examples/dds/pycompss/runtime/management/object_tracker.py b/examples/dds/pycompss/runtime/management/object_tracker.py new file mode 100644 index 00000000..52cdce6f --- /dev/null +++ b/examples/dds/pycompss/runtime/management/object_tracker.py @@ -0,0 +1,605 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Object tracker. + +This file contains the object tracking functionality. +""" + +import os +import time +import uuid + +from pycompss.runtime.commons import GLOBALS +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +if __debug__: + import logging + + LOGGER = logging.getLogger(__name__) + + +class ObjectTracker: # pylint: disable=too-many-instance-attributes + """Object tracker class. + + This class has all needed data structures and functionalities + to keep track of the objects within the python binding. + """ + + __slots__ = [ + "file_names", + "obj_names", + "pending_to_synchronize", + "written_objects", + "current_id", + "runtime_id", + "obj_id_to_obj", + "address_to_obj_id", + "reporting", + "reporting_info", + "initial_time", + ] + + def __init__(self) -> None: + """Object tracker constructor. + + :returns: None. + """ + # Dictionary to contain the conversion from object id to the + # filename where it is stored (mapping). + # The filename will be used for requesting an object to + # the runtime (its corresponding version). + self.file_names = {} # type: typing.Dict[str, str] + # Dictionary to contain the conversion from object id to the + # parameter name (object name mapping). + # The object name will be used to map the tracked objects with the + # updated within a task (e.g. synchronize within task). + self.obj_names = {} # type: typing.Dict[str, typing.Optional[str]] + # Set that contains the object identifiers of the objects to pending + # to be synchronized. + self.pending_to_synchronize = set() # type: typing.Set[str] + # Set of identifiers of the objects that have been accessed by the + # main program + self.written_objects = set() # type: typing.Set[str] + # Identifier handling + self.current_id = 1 + # Object identifiers will be of the form _runtime_id-_current_id + # This way we avoid having two objects from different applications + # having the same identifier + self.runtime_id = str(uuid.uuid1()) + # Dictionary to contain the conversion from object identifier to + # the object (address pointer). + # NOTE: it can not be done in the other way since the memory addresses + # can be reused, not guaranteeing their uniqueness, and causing + # weird behaviour. + self.obj_id_to_obj = {} # type: typing.Dict[str, typing.Any] + # Dictionary to contain the object address (currently the id(obj)) to + # the identifier provided by the binding. + self.address_to_obj_id = {} # type: typing.Dict[typing.Any, str] + + # Boolean to store tracking information + # CAUTION: Enabling reporting increases the memory usage since + # it requires storing internally the object tracker status + # when a new object is tracked or stopped tracking. + self.reporting = False + # Report info: Contains tuples composed by the values to be reported. + self.reporting_info = [] # type: typing.List[tuple] + # Store the initial time as reference for the reporting. + self.initial_time = 0.0 + + def track( + self, + obj: typing.Any, + obj_name: typing.Optional[str] = None, + collection: bool = False, + ) -> typing.Tuple[str, str]: + """Start tracking an object. + + Collections are not stored into a file. Consequently, we just register + it to keep track of the identifier, but no file is stored. However, + the collection elements are stored into files. + + :param obj: Object to track. + :param obj_name: Object name (variable placeholder name). + :param collection: If the object to be tracked is a collection. + :return: Object identifier and its corresponding file name. + """ + if collection: + obj_id = self._register_object(obj, True) + file_name = "None" + if __debug__: + LOGGER.debug("Tracking collection %s", obj_id) + else: + obj_id = self._register_object(obj, True) + file_name = f"{GLOBALS.get_temporary_directory()}/{str(obj_id)}" + self._set_file_name(obj_id, file_name) + self._set_obj_name(obj_id, obj_name) + self.set_pending_to_synchronize(obj_id) + if __debug__: + LOGGER.debug( + "Tracking object %s to file %s", obj_id, file_name + ) + address = self._get_object_address(obj) + self.address_to_obj_id[address] = obj_id + if self.reporting: + self.report_now() + return obj_id, file_name + + def not_track(self, collection: bool = False) -> typing.Tuple[str, str]: + """Retrieve a not tracked identifier and file_name. + + :param collection: If the object is a collection. + :returns: Object identifier and file name. + """ + obj_id = f"{self.runtime_id}-{self.current_id}" + if collection: + file_name = "None" + else: + file_name = f"{GLOBALS.get_temporary_directory()}/{str(obj_id)}" + self.current_id += 1 + return obj_id, file_name + + def stop_tracking(self, obj: typing.Any, collection: bool = False) -> None: + """Stop tracking the given object. + + :param obj: Object to stop tracking. + :param collection: If the object to stop tracking is a collection. + :return: None. + """ + obj_id = self.is_tracked(obj) + if obj_id != "": + if collection: + if __debug__: + LOGGER.debug("Stop tracking collection %s", obj_id) + self._pop_object_id(obj_id) + else: + if __debug__: + LOGGER.debug("Stop tracking object %s", obj_id) + self._delete_file_name(obj_id) + self._delete_obj_name(obj_id) + self._remove_from_pending_to_synchronize(obj_id) + self._pop_object_id(obj_id) + self.report_now() + + def get_object_id(self, obj: typing.Any) -> str: + """Return the object identifier of the given object. + + This function is a wrapper of is_tracked. + + :param obj: Object to check. + :return: Object identifier if under tracking. Empty string otherwise. + """ + return self.is_tracked(obj) + + def is_tracked(self, obj: typing.Any) -> str: + """Check if the given object is being tracked. + + Due to the length that the obj_id_to_address dictionary can reach, if + is tracked we return the identifier in order to avoid to search again + into the dictionary. + + :param obj: Object to check. + :return: Object identifier if under tracking. Empty string otherwise. + """ + address = self._get_object_address(obj) + if address in self.address_to_obj_id: + return self.address_to_obj_id[address] + return "" + + def get_all_file_names(self) -> tuple: + """Return all used files names. + + Useful for cleanup. + + :return: List of file name that are currently available. + """ + return tuple(self.file_names.values()) + + def get_file_name(self, obj_id: str) -> str: + """Get the file name associated to the given object identifier. + + :param obj_id: Object identifier. + :return: File name. + """ + return self.file_names[obj_id] + + def get_obj_name(self, obj_id: str) -> typing.Optional[str]: + """Get the object name associated to the given object identifier. + + :param obj_id: Object identifier. + :return: Object name. + """ + if obj_id: + return self.obj_names[obj_id] + return None + + def is_obj_pending_to_synchronize(self, obj: typing.Any) -> bool: + """Check if the given object is pending to be synchronized. + + :param obj: Object to check. + :return: True if pending. False otherwise. + """ + obj_id = self.is_tracked(obj) + if obj_id == "": + return False + return self.is_pending_to_synchronize(obj_id) + + def is_pending_to_synchronize(self, obj_id: str) -> bool: + """Check if the given object id is in pending to be synchronized dict. + + :param obj_id: Object identifier. + :return: True if pending. False otherwise. + """ + return obj_id in self.pending_to_synchronize + + def set_pending_to_synchronize(self, obj_id: str) -> None: + """Set the given filename with object id as pending to synchronize. + + :param obj_id: Object identifier. + :return: None + """ + self.pending_to_synchronize.add(obj_id) + + def has_been_written(self, obj_id: str) -> bool: + """Check if the given object id has been written by the main program. + + :param obj_id: Object identifier. + :return: True if written. False otherwise. + """ + return obj_id in self.written_objects + + def pop_written_obj(self, obj_id: str) -> str: + """Pop a written filename with the given obj_id from written objects. + + :param obj_id: Object identifier. + :return: The file name. + """ + self.written_objects.remove(obj_id) + return self.get_file_name(obj_id) + + def update_mapping(self, obj_id: str, obj: typing.Any) -> None: + """Update the object into the object tracker. + + :param obj_id: Object identifier. + :param obj: New object to track. + :return: None. + """ + # The main program won't work with the old object anymore, update + # mapping + new_obj_id = self._register_object(obj, True, True) + old_file_name = self.get_file_name(obj_id) + new_file_name = old_file_name.replace(obj_id, str(new_obj_id)) + self._set_file_name(new_obj_id, new_file_name, written=True) + + def clean_object_tracker(self, hard_stop: bool = False) -> None: + """Clear all object tracker internal structures. + + :param hard_stop: avoid call to delete_file when the runtime has died. + :return: None + """ + app_id = 0 + if not hard_stop: + for filename in OT.get_all_file_names(): + COMPSs.delete_file(app_id, filename, False) + + self.file_names.clear() + self.obj_names.clear() + self.pending_to_synchronize.clear() + self.written_objects.clear() + self.current_id = 1 + self.runtime_id = str(uuid.uuid1()) + self.obj_id_to_obj.clear() + self.address_to_obj_id.clear() + self.report_now() + + def clean_report(self) -> None: + """Clear the reporting data. + + :return: None. + """ + del self.reporting_info[:] + + ############################################# + # PRIVATE FUNCTIONS # + ############################################# + + def _register_object( + self, + obj: typing.Any, + assign_new_key: bool = False, + force_insertion: bool = False, + ) -> str: + """Register an object into the object tracker. + + If not found or we are forced to, we create a new identifier for this + object, deleting the old one if necessary. We can also query for some + object without adding it in case of failure. + + Identifiers are of the form _runtime_id-_current_id in order to avoid + having two objects from different applications with the same identifier + (and thus file name). + This function updates the internal self.current_id to guarantee + that each time returns a new identifier. + + :param obj: Object to analyse. + :param assign_new_key: Assign new key. + :param force_insertion: force insertion. + :return: Object id. + """ + # Force_insertion implies assign_new_key + if not (not force_insertion or assign_new_key): + raise PyCOMPSsException("Force insertion implies assign new key") + + identifier = self.is_tracked(obj) + if identifier != "": + if force_insertion: + self.obj_id_to_obj.pop(identifier) + address = self._get_object_address(obj) + self.address_to_obj_id.pop(address) + else: + return identifier + + if assign_new_key: + # This object was not in our object database or we were forced to + # remove it, lets assign it an identifier and store it. + # Generate a new identifier + new_id = f"{self.runtime_id}-{self.current_id}" + self.current_id += 1 + self.obj_id_to_obj[new_id] = obj + address = self._get_object_address(obj) + self.address_to_obj_id[address] = new_id + return new_id + + raise PyCOMPSsException("Reached unexpected object registry case.") + + def _set_file_name( + self, obj_id: str, filename: str, written: bool = False + ) -> None: + """Set a filename for the given object identifier. + + :param obj_id: Object identifier. + :param filename: File name. + :param written: If the file has been written by main program + :return: None + """ + self.file_names[obj_id] = filename + if written: + self.written_objects.add(obj_id) + + def _delete_file_name(self, obj_id: str) -> None: + """Remove the file name of the given object identifier. + + :param obj_id: Object identifier. + :return: None + """ + del self.file_names[obj_id] + + def _set_obj_name( + self, obj_id: str, obj_name: typing.Optional[str] + ) -> None: + """Set the object name for the given object identifier. + + :param obj_id: Object identifier. + :param obj_name: Object name. + :return: None + """ + self.obj_names[obj_id] = obj_name + + def _delete_obj_name(self, obj_id: str) -> None: + """Remove the object name of the given object identifier. + + :param obj_id: Object identifier. + :return: None + """ + del self.obj_names[obj_id] + + def _remove_from_pending_to_synchronize(self, obj_id: str) -> None: + """Pop the filename of the given object id from pending to synchronize. + + :param obj_id: Object identifier. + :return: None + """ + self.pending_to_synchronize.remove(obj_id) + + def _pop_object_id(self, obj_id: str) -> typing.Any: + """Pop an object from the dictionary. + + :param obj_id: Object identifier to pop. + :return: Popped object. + """ + obj = self.obj_id_to_obj.pop(obj_id) + address = self._get_object_address(obj) + return self.address_to_obj_id.pop(address) + + @staticmethod + def _get_object_address(obj: typing.Any) -> int: + """Retrieve the object memory address. + + :param obj: Object to get the memory address. + :return: Object identifier. + """ + return id(obj) + # # If we want to detect automatically IN objects modification we need + # # to ensure uniqueness of the identifier. At this point, obj is a + # # reference to the object that we want to compute its identifier. + # # This means that we do not have the previous object to compare + # # directly. + # # So the only way would be to ensure the uniqueness by calculating + # # an id which depends on the object. + # # BUT THIS IS REALLY EXPENSIVE. So: Use the id and unregister the + # # object (IN) to be modified + # # explicitly. + # immutable_types = [bool, int, float, complex, str, + # tuple, frozenset, bytes] + # obj_type = type(obj) + # if obj_type in immutable_types: + # obj_address = id(obj) # Only guarantees uniqueness with + # # immutable objects + # else: + # # For all the rest, use hash of: + # # - The object id + # # - The size of the object (object increase/decrease) + # # - The object representation (object size is the same but has + # # been modified(e.g. list element)) + # # WARNING: Caveat: + # # - IN User defined object with parameter change without + # # __repr__ + # # INOUT parameters to be modified require a synchronization, so + # # they are not affected. + # import hashlib + # hash_id = hashlib.md5() + # # Consider the memory pointer: + # hash_id.update(str(id(obj)).encode()) + # # Include the object size: + # hash_id.update(str(total_sizeof(obj)).encode()) + # # Include the object representation: + # hash_id.update(repr(obj).encode()) + # obj_address = str(hash_id.hexdigest()) + # return obj_address + + ############################################# + # REPORTING FUNCTIONS # + ############################################# + + def enable_report(self) -> None: + """Enable reporting. + + Enables to keep the status in internal infrastructure so that + the report can be generated afterwards. + + :return: None + """ + self.reporting = True + # Get initial reporting status + self.report_now(first=True) + + def is_report_enabled(self) -> bool: + """Retrieve if the reporting is enabled. + + :return: If the object tracker is keeping track of the status. + """ + return self.reporting + + def report_now(self, first: bool = False) -> None: + """Update the report with the current Object Tracker status. + + WARNING: This function only works if log_level=debug. + + :param first: If it is the first time reporting the status. + :return: None + """ + if __debug__ and self.reporting: + # Log the object tracker status + self.__log_object_tracker_status__() + self.__update_report__(first) + + def __log_object_tracker_status__(self) -> None: + """Log the object tracker status. + + :return: None + """ + LOGGER.debug( + "Object tracker status:\n" + " - File_names=%s\n" + " - Pending_to_synchronize=%s\n" + " - Written_objs=%s\n" + " - Obj_id_to_obj=%s\n" + " - Address_to_obj_id=%s\n" + " - Current_id=%s", + str(len(self.file_names)), + str(len(self.pending_to_synchronize)), + str(len(self.written_objects)), + str(len(self.obj_id_to_obj)), + str(len(self.address_to_obj_id)), + str(self.current_id), + ) + + def __update_report__(self, first: bool = False) -> None: + """Update the reporting. + + Update the internal self.report_info variable with the + current object tracker status. + + :param first: If it is the first time reporting the status. + :return: None + """ + if first: + self.initial_time = time.time() + current_status = ( + time.time() - self.initial_time, + ( + len(self.file_names), + len(self.pending_to_synchronize), + len(self.written_objects), + len(self.obj_id_to_obj), + len(self.address_to_obj_id), + ), + ) + self.reporting_info.append(current_status) + + def generate_report(self, target_path: str) -> None: + """Generate a plot reporting the behaviour of the object tracker. + + Uses the self.report_info internal variable contents. + + :param target_path: Path where to store the report. + :return: None + """ + try: + import matplotlib # pylint: disable=import-outside-toplevel + + # Agg avoid issues in MN + matplotlib.use("Agg") + import matplotlib.pyplot as plt # pylint: disable=C0415 + + # disable=import-outside-toplevel + except ImportError: + print("WARNING: Could not generate the Object Tracker report") + print("REASON : matplotlib not available.") + return None + if __debug__: + LOGGER.debug("Generating object tracker report...") + x_axis = [status[0] for status in self.reporting_info] + y_axis = [status[1] for status in self.reporting_info] + plt.xlabel("Time (seconds)") + plt.ylabel("# Elements") + plt.title("Object tracker behaviour") + labels = [ + "File names", + "Pending to synchronize", + "Updated mappings", + "IDs", + "Addresses", + ] + for i in range(len(y_axis[0])): + plt.plot(x_axis, [pt[i] for pt in y_axis], label=f"{labels[i]}") + plt.legend() + target = os.path.join(target_path, "object_tracker.png") + plt.savefig(target) + if __debug__: + LOGGER.debug("Object tracker report stored in %s", target) + return None + + +# Instantiation of the Object tracker class to be shared among +# management modules +OT = ObjectTracker() diff --git a/examples/dds/pycompss/runtime/management/synchronization.py b/examples/dds/pycompss/runtime/management/synchronization.py new file mode 100644 index 00000000..aea0386b --- /dev/null +++ b/examples/dds/pycompss/runtime/management/synchronization.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Management - Object Synchronization. + +This file contains the object synchronization core methods. +""" + +import logging +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.context import CONTEXT +from pycompss.runtime.management.classes import Future +from pycompss.runtime.management.direction import get_compss_direction +from pycompss.runtime.management.object_tracker import OT +from pycompss.runtime.task.shared_args import SHARED_ARGUMENTS +from pycompss.util.serialization.serializer import deserialize_from_file +from pycompss.util.storages.persistent import get_by_id +from pycompss.util.storages.persistent import get_id +from pycompss.util.storages.persistent import is_psco +from pycompss.util.typing_helper import typing + + +LOGGER = logging.getLogger(__name__) + + +def wait_on_object(obj: typing.Any, mode: str) -> typing.Any: + """Wait on an object (synchronize). + + :param obj: Object to wait on. + :param mode: Read or write mode + :return: An object of "file" type. + """ + compss_mode = get_compss_direction(mode) + if isinstance(obj, Future) or not isinstance(obj, (list, dict)): + return _synchronize(obj, compss_mode) + if len(obj) == 0: # FUTURE OBJECT + return _synchronize(obj, compss_mode) + # Otherwise, will be an iterable object + + return _wait_on_iterable(obj, compss_mode) + + +def _synchronize(obj: typing.Any, mode: int) -> typing.Any: + """Synchronize the given object. + + This method retrieves the value of a future object. + Calls the runtime in order to wait for the value and returns it when + received. + + :param obj: Object to synchronize. + :param mode: Direction of the object to synchronize. + :return: The value of the object requested. + """ + # TODO: Add a boolean to differentiate between files and object on the + # COMPSs.open_file call. This change pretends to obtain better traces. + # Must be implemented first in the Runtime, then in the bindings common + # C API and finally add the boolean here + app_id = 0 + obj_id = "" # noqa + + if is_psco(obj): + obj_id = str(get_id(obj)) + if not OT.is_pending_to_synchronize(obj_id): + return obj + # file_path is of the form storage://pscoId or + # file://sys_path_to_file + file_path = COMPSs.open_file( + app_id, "".join(("storage://", str(obj_id))), mode + ) + # TODO: Add switch on protocol + # (first parameter returned currently ignored) + _, file_name = file_path.split("://") + new_obj = get_by_id(file_name) + OT.stop_tracking(obj) + return new_obj + + obj_id = OT.is_tracked(obj) + obj_name = OT.get_obj_name(obj_id) + if obj_id is None: # Not being tracked + return obj + if not OT.is_pending_to_synchronize(obj_id): + return obj + + if __debug__: + LOGGER.debug("Synchronizing object %s with mode %s", obj_id, mode) + file_name = OT.get_file_name(obj_id) + compss_file = COMPSs.open_file(app_id, file_name, mode) + # Runtime can return a path or a PSCOId + if compss_file.startswith("/"): + # If the real filename is null, then return None. The task that + # produces the output file may have been ignored or cancelled, so its + # result does not exist. + real_file_name = compss_file.split("/")[-1] + if real_file_name == "null": + print( + "WARNING: Could not retrieve the object " + + str(file_name) + + " since the task that produces it may have been IGNORED or" + + " CANCELLED. Please, check the logs. Returning None." + ) + return None + new_obj = deserialize_from_file(compss_file, LOGGER) + COMPSs.close_file(app_id, file_name, mode) + else: + new_obj = get_by_id(compss_file) + + if CONTEXT.is_nesting_enabled() and CONTEXT.in_worker(): + # If nesting and in worker, the user wants to synchronize an object. + # So it is necessary to update the parameter with the new object. + SHARED_ARGUMENTS.update_worker_argument_parameter_content( + obj_name, new_obj + ) + + if mode == "r": + OT.update_mapping(obj_id, new_obj) + + if mode != "r": + COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False) + OT.stop_tracking(obj) + + return new_obj + + +def _wait_on_iterable(iter_obj: typing.Any, compss_mode: int) -> typing.Any: + """Wait on an iterable object (Recursive). + + Currently, supports lists and dictionaries (syncs the values). + + :param iter_obj: Iterable object. + :return: Synchronized object. + """ + # check if the object is in our pending_to_synchronize dictionary + obj_id = OT.is_tracked(iter_obj) + if OT.is_pending_to_synchronize(obj_id): + return _synchronize(iter_obj, compss_mode) + if isinstance(iter_obj, list): + return [_wait_on_iterable(x, compss_mode) for x in iter_obj] + if isinstance(iter_obj, dict): + return { + _wait_on_iterable(k, compss_mode): _wait_on_iterable( + v, compss_mode + ) + for k, v in iter_obj.items() + } + return _synchronize(iter_obj, compss_mode) diff --git a/examples/dds/pycompss/runtime/mpi/__init__.py b/examples/dds/pycompss/runtime/mpi/__init__.py new file mode 100644 index 00000000..6e21d9de --- /dev/null +++ b/examples/dds/pycompss/runtime/mpi/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime mpi helpers.""" diff --git a/examples/dds/pycompss/runtime/mpi/keys.py b/examples/dds/pycompss/runtime/mpi/keys.py new file mode 100644 index 00000000..759d32ec --- /dev/null +++ b/examples/dds/pycompss/runtime/mpi/keys.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - MPI - Keys. + +This file contains the MPI Collection layout keys. +""" + + +class MPILayoutKeys: + """Strings used in MPI layout for collections.""" + + block_count = "block_count" + block_length = "block_length" + stride = "stride" + + +MPI_LAYOUT_KEYS = MPILayoutKeys() diff --git a/examples/dds/pycompss/runtime/start/__init__.py b/examples/dds/pycompss/runtime/start/__init__.py new file mode 100644 index 00000000..5a5e3f9c --- /dev/null +++ b/examples/dds/pycompss/runtime/start/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime.""" diff --git a/examples/dds/pycompss/runtime/start/initialization.py b/examples/dds/pycompss/runtime/start/initialization.py new file mode 100644 index 00000000..7833b2fa --- /dev/null +++ b/examples/dds/pycompss/runtime/start/initialization.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Initialization. + +This file contains the initialization functionalities. +""" + +import logging + + +class LaunchStatus: + """This class contains the global status variables used all around.""" + + __slots__ = ["app_path", "streaming", "persistent_storage", "logger"] + + def __init__(self) -> None: + """Create a new state object.""" + self.app_path = "undefined" + self.streaming = False + self.persistent_storage = False + self.logger = logging.getLogger(__name__) # type: logging.Logger + + def get_app_path(self) -> str: + """Get APP_PATH value. + + :return: App path value. + """ + return self.app_path + + def get_streaming(self) -> bool: + """Get STREAMING state value. + + :return: Streaming status value. + """ + return self.streaming + + def get_persistent_storage(self) -> bool: + """Get PERSISTENT_STORAGE state value. + + :return: Persistent storage status value. + """ + return self.persistent_storage + + def get_logger(self) -> logging.Logger: + """Get LOGGER value. + + :return: Logger value. + """ + return self.logger + + def set_app_path(self, app_path: str) -> None: + """Set APP_PATH value. + + :param: app_path: New app path value. + :return: None + """ + self.app_path = app_path + + def set_streaming(self, streaming: bool) -> None: + """Set STREAMING value. + + :param: streaming: New streaming value. + :return: None + """ + self.streaming = streaming + + def set_persistent_storage(self, persistent_storage: bool) -> None: + """Set PERSISTENT STORAGE value. + + :param: persistent_storage: New persistent storage value. + :return: None + """ + self.persistent_storage = persistent_storage + + def set_logger(self, logger: logging.Logger) -> None: + """Set LOGGER value. + + :param: logger: New logger value. + :return: None + """ + self.logger = logger + + +LAUNCH_STATUS = LaunchStatus() diff --git a/examples/dds/pycompss/runtime/start/interactive_initialization.py b/examples/dds/pycompss/runtime/start/interactive_initialization.py new file mode 100644 index 00000000..7142948d --- /dev/null +++ b/examples/dds/pycompss/runtime/start/interactive_initialization.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Interactive Initialization. + +This file contains the extra interactive initialization functionalities. +""" + + +class ExtraLaunchStatus: + """This class contains the global status variables used in interactive.""" + + __slots__ = ["log_path", "graphing", "line_separator", "disable_external"] + + def __init__(self) -> None: + """Create a new state object.""" + self.log_path = "undefined" + self.graphing = False + self.line_separator = 56 * "*" + self.disable_external = False + + def get_graphing(self) -> bool: + """Get GRAPHING state value. + + :return: Graphing status value. + """ + return self.graphing + + def get_line_separator(self) -> str: + """Get LINE_SEPARATOR. + + :return: LINE SEPARATOR string. + """ + return self.line_separator + + def get_disable_external(self) -> bool: + """Get DISABLE_EXTERNAL value. + + :return: Disable external value. + """ + return self.disable_external + + def set_graphing(self, graphing: bool) -> None: + """Set GRAPHING value. + + :param: graphing: New graphing value. + :return: None + """ + self.graphing = graphing + + def set_line_separator(self, line_separator: str) -> None: + """Set LINE SEPARATOR value. + + :param: line_separator: New line separator value. + :return: None + """ + self.line_separator = line_separator + + def set_disable_external(self, disable_external: bool) -> None: + """Set DISABLE_EXTERNAL value. + + :param: disable_external: New disable_external value. + :return: None + """ + self.disable_external = disable_external + + +EXTRA_LAUNCH_STATUS = ExtraLaunchStatus() diff --git a/examples/dds/pycompss/runtime/task/__init__.py b/examples/dds/pycompss/runtime/task/__init__.py new file mode 100644 index 00000000..f029e979 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime task management helpers.""" diff --git a/examples/dds/pycompss/runtime/task/arguments.py b/examples/dds/pycompss/runtime/task/arguments.py new file mode 100644 index 00000000..7261c654 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/arguments.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Arguments. + +This file contains the classes needed for the argument identification. +""" + + +def is_vararg(param_name: str) -> bool: + """Determine if a parameter is named as a (internal) vararg. + + :param param_name: String with a parameter name. + :returns: True if the name has the form of an internal vararg name. + """ + return param_name.startswith("*") + + +def get_name_from_vararg(full_name: str) -> str: + """Extract the vararg name from the name given with full_name. + + Part before "*". + + :param full_name: Complete vararg name. + :return: The vararg name. + """ + return full_name.split("*")[1] + + +def is_kwarg(param_name: str) -> bool: + """Determine if a parameter is named as a (internal) kwargs. + + :param param_name: String with a parameter name. + :return: True if the name has the form of an internal kwarg name. + """ + return param_name.startswith("#kwarg") + + +def is_return(param_name: str) -> bool: + """Determine if a parameter is named as a (internal) return. + + :param param_name: String with a parameter name. + :returns: True if the name has the form of an internal return name. + """ + return param_name.startswith("$return") + + +def get_vararg_name(varargs_name: str, i: int) -> str: + """Given some integer i, return the name of the ith vararg. + + Note that the given internal names to these parameters are + impossible to be assigned by the user because they are invalid + Python variable names, as they start with a star + + :param varargs_name: Vararg names. + :param i: A non negative integer. + :return: The name of the ith vararg according to our internal naming + convention. + """ + return f"*{varargs_name}*_{i}" + + +def get_kwarg_name(var: str) -> str: + """Given some variable name, get the kwarg identifier. + + :param var: A string with a variable name. + :return: The name of the kwarg according to our internal naming convention. + """ + return f"#kwarg_{var}" + + +def get_name_from_kwarg(var: str) -> str: + """Given some kwarg name, return the original variable name. + + :param var: A string with a (internal) kwarg name. + :return: The original variable name. + """ + return var.replace("#kwarg_", "") + + +def get_return_name(i: int) -> str: + """Given some integer i, return the name of the ith return. + + :param i: A non-negative integer. + :return: The name of the return identifier according to our internal + naming. + """ + return f"$return_{i}" diff --git a/examples/dds/pycompss/runtime/task/commons.py b/examples/dds/pycompss/runtime/task/commons.py new file mode 100644 index 00000000..e2146a73 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/commons.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Arguments commons. + +This file contains the task arguments commons. +""" + +from pycompss.api import parameter +from pycompss.util.typing_helper import typing +from pycompss.runtime.task.definitions.arguments import TaskArguments + + +def get_default_direction( + var_name: str, + decorator_arguments: TaskArguments, + param_args: typing.List[typing.Any], +) -> str: + """Return the default direction for a given parameter. + + :param var_name: Variable name. + :param decorator_arguments: Decorator arguments. + :param param_args: Parameter args. + :return: An identifier of the direction. + """ + # We are the "self" or "cls" in an instance or class method that + # modifies the given class, so we are an INOUT, CONCURRENT or + # COMMUTATIVE + self_dirs = [ + parameter.INOUT.key, + parameter.CONCURRENT.key, + parameter.COMMUTATIVE.key, + ] + if ( + decorator_arguments.target_direction in self_dirs + and var_name in ["self", "cls"] + and param_args + and param_args[0] == var_name + ): + return decorator_arguments.target_direction + return parameter.IN.key diff --git a/examples/dds/pycompss/runtime/task/definitions/__init__.py b/examples/dds/pycompss/runtime/task/definitions/__init__.py new file mode 100644 index 00000000..f029e979 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/definitions/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime task management helpers.""" diff --git a/examples/dds/pycompss/runtime/task/definitions/arguments.py b/examples/dds/pycompss/runtime/task/definitions/arguments.py new file mode 100644 index 00000000..630a0c10 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/definitions/arguments.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Definitions - Arguments. + +This file contains the task decorator arguments definition. +""" + +from pycompss.api import parameter +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.util.typing_helper import typing +from pycompss.runtime.task.parameter import Parameter +from pycompss.runtime.task.parameter import get_new_parameter +from pycompss.runtime.task.parameter import get_parameter_from_dictionary +from pycompss.runtime.task.parameter import is_param +from pycompss.util.exceptions import PyCOMPSsException + +if __debug__: + import logging + + LOGGER = logging.getLogger(__name__) + + +class TaskArguments: + """Task decorator arguments class definition. + + This class represents the supported task decorator arguments. + """ + + __slots__ = [ + # Supported keywords + "target_direction", + "returns", + "cache_returns", + "priority", + "defaults", + "time_out", + "is_replicated", + "is_distributed", + "computing_nodes", + "processes_per_node", + "is_reduce", + "chunk_size", + "on_failure", + "tracing_hook", + "numba", + "numba_flags", + "numba_signature", + "numba_declaration", + # varargs_type kept for legacy purposes: + "varargs_type", + # Function parameters: + "parameters", + ] + + def __init__(self) -> None: + """Set the default task decorator arguments values.""" + # Supported arguments in the @task decorator + self.target_direction = parameter.INOUT.key + self.returns = False # type: typing.Any + self.cache_returns = True + self.priority = False + self.defaults = {} # type: typing.Dict[str, typing.Any] + self.time_out = 0 + self.is_replicated = False + self.is_distributed = False + self.computing_nodes = 1 + self.processes_per_node = 1 + self.is_reduce = False + self.chunk_size = 0 + self.on_failure = "RETRY" + self.tracing_hook = False + # numba mode (jit, vectorize, guvectorize) + self.numba = ( + False + ) # type: typing.Union[bool, str, typing.Dict[str, bool]] + # user defined extra numba flags + self.numba_flags = ( + {} + ) # type: typing.Dict[str, typing.Union[str, bool]] + # vectorize and guvectorize signature + self.numba_signature = ( + None + ) # type: typing.Optional[typing.Union[str, typing.List[str]]] + # guvectorize declaration + self.numba_declaration = None # type: typing.Optional[str] + # varargs_type kept for legacy purposes: + self.varargs_type = parameter.IN.key + # Function parameters: + self.parameters = ( + {} + ) # type: typing.Dict[str, typing.Union[Parameter, typing.Any]] + + @staticmethod + def __deprecation_warning__(old_keyword: str, new_keyword: str) -> None: + """Log the deprecation warning message for the given keyword. + + :param old_keyword: Deprecated keyword. + :param new_keyword: Keyword that should be used. + :return: None + """ + LOGGER.warning( + "Detected deprecated %s. Please, change it to %s", + old_keyword, + new_keyword, + ) + + def update_arguments(self, kwargs: typing.Dict[str, typing.Any]) -> None: + """Update the task arguments with the given kwargs. + + :param kwargs: Task keyword arguments. + :returns: None + """ + # Argument: target_direction + if LABELS.target_direction in kwargs: + target_direction = kwargs.pop(LABELS.target_direction) + if is_param(target_direction): + self.target_direction = target_direction.key + else: + raise PyCOMPSsException( + f"Unexpected {LABELS.target_direction} type. " + f"Must be a direction." + ) + elif LEGACY_LABELS.target_direction in kwargs: + target_direction = kwargs.pop(LEGACY_LABELS.target_direction) + if is_param(target_direction): + self.target_direction = target_direction.key + self.__deprecation_warning__( + LEGACY_LABELS.target_direction, LABELS.target_direction + ) + else: + raise PyCOMPSsException( + f"Unexpected {LEGACY_LABELS.target_direction} type. " + f"Must be a direction." + ) + # Argument: returns + if LABELS.returns in kwargs: + self.returns = kwargs.pop(LABELS.returns) + # Argument: cache_returns + if LABELS.cache_returns in kwargs: + self.cache_returns = kwargs.pop(LABELS.cache_returns) + # Argument: priority + if LABELS.priority in kwargs: + self.priority = kwargs.pop(LABELS.priority) + # Argument: defaults + if LABELS.defaults in kwargs: + self.defaults = kwargs.pop(LABELS.defaults) + # Argument: time_out + if LABELS.time_out in kwargs: + self.time_out = kwargs.pop(LABELS.time_out) + elif LEGACY_LABELS.time_out in kwargs: + self.time_out = kwargs.pop(LEGACY_LABELS.time_out) + self.__deprecation_warning__( + LEGACY_LABELS.time_out, LABELS.time_out + ) + # Argument: is_replicated + if LABELS.is_replicated in kwargs: + self.is_replicated = kwargs.pop(LABELS.is_replicated) + elif LEGACY_LABELS.is_replicated in kwargs: + self.is_replicated = kwargs.pop(LEGACY_LABELS.is_replicated) + self.__deprecation_warning__( + LEGACY_LABELS.is_replicated, LABELS.is_replicated + ) + # Argument: is_distributed + if LABELS.is_distributed in kwargs: + self.is_distributed = kwargs.pop(LABELS.is_distributed) + elif LEGACY_LABELS.is_distributed in kwargs: + self.is_distributed = kwargs.pop(LEGACY_LABELS.is_distributed) + self.__deprecation_warning__( + LEGACY_LABELS.is_distributed, LABELS.is_distributed + ) + # Argument: computing_nodes + if LABELS.computing_nodes in kwargs: + self.computing_nodes = kwargs.pop(LABELS.computing_nodes) + elif LEGACY_LABELS.computing_nodes in kwargs: + self.computing_nodes = kwargs.pop(LEGACY_LABELS.computing_nodes) + self.__deprecation_warning__( + LEGACY_LABELS.computing_nodes, LABELS.computing_nodes + ) + # Argument: processes_per_node + if LABELS.processes_per_node in kwargs: + self.processes_per_node = kwargs.pop(LABELS.processes_per_node) + # Argument: is_reduce + if LABELS.is_reduce in kwargs: + self.is_reduce = kwargs.pop(LABELS.is_reduce) + # Argument: chunk_size + if LABELS.chunk_size in kwargs: + self.chunk_size = kwargs.pop(LABELS.chunk_size) + # Argument: on_failure + if LABELS.on_failure in kwargs: + self.on_failure = kwargs.pop(LABELS.on_failure) + # Argument: tracing_hook + if LABELS.tracing_hook in kwargs: + self.tracing_hook = kwargs.pop(LABELS.tracing_hook) + # Argument: numba + if LABELS.numba in kwargs: + self.numba = kwargs.pop(LABELS.numba) + # Argument: numba_flags + if LABELS.numba_flags in kwargs: + self.numba_flags = kwargs.pop(LABELS.numba_flags) + # Argument: numba_signature + if LABELS.numba_signature in kwargs: + self.numba_signature = kwargs.pop(LABELS.numba_signature) + # Argument: numba_declaration + if LABELS.numba_declaration in kwargs: + self.numba_declaration = kwargs.pop(LABELS.numba_declaration) + # Argument: varargs_type + if LABELS.varargs_type in kwargs: + varargs_type = kwargs.pop(LABELS.varargs_type) + if is_param(varargs_type): + self.varargs_type = varargs_type.key + else: + raise PyCOMPSsException( + "Unexpected varargs_type type. Must be a direction." + ) + elif LEGACY_LABELS.varargs_type in kwargs: + varargs_type = kwargs.pop(LEGACY_LABELS.varargs_type) + if is_param(varargs_type): + self.varargs_type = varargs_type.key + self.__deprecation_warning__( + LEGACY_LABELS.varargs_type, LABELS.varargs_type + ) + else: + raise PyCOMPSsException( + "Unexpected varargsType type. Must be a direction." + ) + # Argument: the rest (named function parameters). + # The rest of the arguments are expected to be only the function + # parameter related information + for key, value in kwargs.items(): + if isinstance(value, dict): + # It is a dictionary for the given parameter + # (e.g. param={Direction: IN}) + self.parameters[key] = get_parameter_from_dictionary(value) + else: + # It is a keyword for the given parameter (e.g. param=IN) + try: + self.parameters[key] = get_new_parameter(value.key) + except AttributeError: # no .key in value. + # Non defined parameters: usually mistaken parameters. + self.parameters[key] = value + + def __repr__(self) -> str: + """Representation of the task arguments definition. + + :return: String representing the task arguments. + """ + task_arguments = "Task arguments:\n" + task_arguments += f"- Target direction: {self.target_direction}\n" + task_arguments += f"- Returns: {self.returns}\n" + task_arguments += f"- Cache returns: {self.cache_returns}\n" + task_arguments += f"- Priority: {self.priority}\n" + task_arguments += f"- Defaults: {self.defaults}\n" + task_arguments += f"- Time out: {self.time_out}\n" + task_arguments += f"- Is replicated: {self.is_replicated}\n" + task_arguments += f"- Is distributed: {self.is_distributed}\n" + task_arguments += f"- Computing nodes: {self.computing_nodes}\n" + task_arguments += f"- Processes per node: {self.processes_per_node}\n" + task_arguments += f"- Is reduce: {self.is_reduce}\n" + task_arguments += f"- Chunk size: {self.chunk_size}\n" + task_arguments += f"- On failure: {self.on_failure}\n" + task_arguments += f"- Tracing hook: {self.tracing_hook}\n" + task_arguments += f"- Numba: {self.numba}\n" + task_arguments += f"- Numba flags: {self.numba_flags}\n" + task_arguments += f"- Numba signature: {self.numba_signature}\n" + task_arguments += f"- Numba declaration: {self.numba_declaration}\n" + task_arguments += f"- Varargs type: {self.varargs_type}\n" + task_arguments += f"- Parameters: {self.parameters}" + return task_arguments + + def get_keys(self) -> typing.List[str]: # TODO: avoid this function. + """Return defined arguments. + + :return: List of strings + """ + arguments = [ + "target_direction", + "returns", + "cache_returns", + "priority", + "defaults", + "time_out", + "is_replicated", + "is_distributed", + "computing_nodes", + "processes_per_node", + "is_reduce", + "chunk_size", + "on_failure", + "tracing_hook", + "numba", + "numba_flags", + "numba_signature", + "numba_declaration", + "varargs_type", + ] + return arguments + + def get_parameter(self, name: str, default_direction: str) -> Parameter: + """Retrieve the Parameter object associated to name. + + If not found, retrieves a new Parameter object with the default + direction. + + :param name: Parameter name. + :param default_direction: Default direction if not found. + :return: The Parameter object + """ + if name in self.parameters: + return self.parameters[name] + # Not found: + default_parameter = get_new_parameter(default_direction) + return default_parameter + + def get_parameter_or_none(self, name: str) -> typing.Optional[Parameter]: + """Retrieve the Parameter object associated to name. + + If not found, retrieves None. + + :param name: Parameter name. + :return: The Parameter object or None + """ + if name in self.parameters: + return self.parameters[name] + return None diff --git a/examples/dds/pycompss/runtime/task/definitions/constraints.py b/examples/dds/pycompss/runtime/task/definitions/constraints.py new file mode 100644 index 00000000..3226c36b --- /dev/null +++ b/examples/dds/pycompss/runtime/task/definitions/constraints.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Definitions - Constraints. + +This file contains the Constraints class, needed for the task registration. +""" + +from pycompss.util.typing_helper import typing + + +class ConstraintDescription: + """Constraint Description class.""" + + __slots__ = [ + "param_name", + "is_static", + ] + + def __init__( + self, + param_name: typing.Union[str, int], + is_static: bool = True, + ) -> None: + """Constraint Description constructor. + + :param param_name: Parameter name of the constraint. + :param is_static: If the constraint is static. + """ + self.param_name = param_name + self.is_static = is_static + + ########### + # METHODS # + ########### + + def reset(self) -> None: + """Reset the constraint description. + + :returns: None. + """ + self.param_name = "" + self.is_static = True + + ########### + # GETTERS # + ########### + + def get_param_name(self) -> typing.Union[str, int]: + """Get the parameter name. + + :return: The parameter name. + """ + return self.param_name + + def get_is_static(self) -> bool: + """Get if the constraint is static. + + :return: The state of the constraint. + """ + return self.is_static + + ########### + # SETTERS # + ########### + + def set_param_name(self, param_name: typing.Union[str, int]) -> None: + """Set the param name. + + :param param_name: The name of the parameter. + :returns: None. + """ + self.param_name = param_name + + def set_is_static(self, is_static: bool) -> None: + """Set if is static. + + :param is_static: The bool if its static. + :return: None. + """ + self.is_static = is_static + + ################## + # REPRESENTATION # + ################## + + def __repr__(self) -> str: + """Build the element representation as string. + + :return: The constraint description representation. + """ + _repr = "Constraint Description: \n" + _repr += f"\t - Parameter name : {str(self.param_name)}\n" + _repr += f"\t - Static : {str(self.is_static)}\n" + + return _repr diff --git a/examples/dds/pycompss/runtime/task/definitions/core_element.py b/examples/dds/pycompss/runtime/task/definitions/core_element.py new file mode 100644 index 00000000..8d22a2c5 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/definitions/core_element.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Definitions - Core element. + +This file contains the Core Element class, needed for the task registration. +""" + +from pycompss.api.commons.constants import INTERNAL_LABELS +from pycompss.util.typing_helper import typing + + +class CE: # pylint: disable=too-many-instance-attributes + """Core Element class.""" + + __slots__ = [ + "ce_signature", + "impl_signature", + "impl_constraints", + "impl_type", + "impl_local", + "impl_io", + "impl_prolog", + "impl_epilog", + "impl_container", + "impl_type_args", + ] + + def __init__( # pylint: disable=too-many-arguments + self, + ce_signature: str = "", + impl_signature: str = "", + impl_constraints: typing.Optional[ + typing.Dict[str, typing.Union[str, int]] + ] = None, + impl_type: typing.Optional[str] = None, + impl_local: bool = False, + impl_io: bool = False, + impl_prolog: typing.Optional[typing.List[str]] = None, + impl_epilog: typing.Optional[typing.List[str]] = None, + impl_container: typing.Optional[typing.List[str]] = None, + impl_type_args: typing.Optional[typing.List[str]] = None, + ) -> None: + """Core Element constructor. + + :param ce_signature: Core element signature. + :param impl_signature: Implementation signature. + :param impl_constraints: Implementation constraints. + :param impl_type: Implementation type. + :param impl_local: If the implementation has to be executed locally. + :param impl_io: If the implementation has IO requirements. + :param impl_prolog: Implementation prolog. + :param impl_epilog: Implementation epilog. + :param impl_container: Implementation container. + :param impl_type_args: Implementation type arguments. + """ + self.ce_signature = ce_signature + self.impl_signature = impl_signature + if impl_constraints is None: + self.impl_constraints = {} + else: + self.impl_constraints = impl_constraints + self.impl_type = impl_type + self.impl_local = impl_local + self.impl_io = impl_io + if impl_type_args is None: + self.impl_type_args = [] + else: + self.impl_type_args = impl_type_args + + self.impl_prolog = ( + [INTERNAL_LABELS.unassigned] * 3 + if impl_prolog is None + else impl_prolog + ) + self.impl_epilog = ( + [INTERNAL_LABELS.unassigned] * 3 + if impl_epilog is None + else impl_epilog + ) + # engine, image, options + self.impl_container = ( + [INTERNAL_LABELS.unassigned] * 3 + if impl_container is None + else impl_container + ) + + ########### + # METHODS # + ########### + + def reset(self) -> None: + """Reset the core element. + + :returns: None. + """ + self.ce_signature = "" + self.impl_signature = "" + self.impl_constraints = {} + self.impl_type = "" + self.impl_io = False + self.impl_local = False + self.impl_type_args = [] + self.impl_prolog = [] + self.impl_epilog = [] + self.impl_container = [] + + ########### + # GETTERS # + ########### + + def get_ce_signature(self) -> str: + """Get the core element signature. + + :return: The core element signature. + """ + return self.ce_signature + + def get_impl_signature(self) -> str: + """Get the core element implementation signature. + + :return: The core element implementation signature. + """ + return self.impl_signature + + def get_impl_constraints(self) -> typing.Dict[str, typing.Union[str, int]]: + """Get the core element implementation constraints. + + :return: The core element implementation constraints. + """ + return self.impl_constraints + + def get_impl_type(self) -> typing.Optional[str]: + """Get the core element implementation type. + + :return: The core element implementation type. + """ + return self.impl_type + + def get_impl_local(self) -> bool: + """Get the core element implementation local. + + :return: The core element implementation local. + """ + return self.impl_local + + def get_impl_io(self) -> bool: + """Get the core element implementation IO. + + :return: The core element implementation IO. + """ + return self.impl_io + + def get_impl_type_args(self) -> typing.List[str]: + """Get the core element implementation type arguments. + + :return: The core element implementation type arguments. + """ + return self.impl_type_args + + def get_impl_prolog(self) -> typing.List[str]: + """Get the core element implementation prolog. + + :return: The core element implementation prolog. + """ + return self.impl_prolog + + def get_impl_epilog(self) -> typing.List[str]: + """Get the core element implementation epilog. + + :return: The core element implementation epilog. + """ + return self.impl_epilog + + def get_impl_container(self) -> typing.List[str]: + """Get the core element implementation container. + + :return: The core element implementation container. + """ + return self.impl_container + + ########### + # SETTERS # + ########### + + def set_ce_signature(self, ce_signature: str) -> None: + """Set the core element signature. + + :param ce_signature: The core element signature. + :returns: None. + """ + self.ce_signature = ce_signature + + def set_impl_signature(self, impl_signature: str) -> None: + """Set the core element implementation signature. + + :param impl_signature: The implementation signature. + :return: None. + """ + self.impl_signature = impl_signature + + def set_impl_constraints( + self, impl_constraints: typing.Dict[str, typing.Union[str, int]] + ) -> None: + """Set the core element implementation constraints. + + :param impl_constraints: The implementation constraints. + :return: None. + """ + self.impl_constraints = impl_constraints + + def set_impl_type(self, impl_type: str) -> None: + """Set the core element implementation type. + + :param impl_type: The implementation type. + :return: None. + """ + self.impl_type = impl_type + + def set_impl_local(self, impl_local: bool) -> None: + """Set the core element implementation local. + + :param impl_local: The implementation local. + :return: None. + """ + self.impl_local = impl_local + + def set_impl_io(self, impl_io: bool) -> None: + """Set the core element implementation IO. + + :param impl_io: The implementation IO. + :return: None. + """ + self.impl_io = impl_io + + def set_impl_type_args(self, impl_type_args: list) -> None: + """Set the core element implementation type arguments. + + :param impl_type_args: The implementation type arguments. + :return: None. + """ + self.impl_type_args = impl_type_args + + def set_impl_prolog(self, impl_prolog: list) -> None: + """Set the core element implementation prolog. + + :param impl_prolog: The implementation prolog. + :return: None. + """ + self.impl_prolog = impl_prolog + + def set_impl_epilog(self, impl_epilog: list) -> None: + """Set the core element implementation epilog. + + :param impl_epilog: The implementation epilog. + :return: None. + """ + self.impl_epilog = impl_epilog + + def set_impl_container(self, impl_container: list) -> None: + """Set the core element implementation container. + + :param impl_container: The implementation container. + :return: None. + """ + self.impl_container = impl_container + + ################## + # REPRESENTATION # + ################## + + def __repr__(self) -> str: + """Build the element representation as string. + + :return: The core element representation. + """ + _repr = "CORE ELEMENT: \n" + _repr += f"\t - CE signature : {str(self.ce_signature)}\n" + _repr += f"\t - Impl. signature : {str(self.impl_signature)}\n" + if self.impl_constraints: + impl_constraints = "" + for key, value in self.impl_constraints.items(): + impl_constraints += f"{key}:{str(value)};" + else: + impl_constraints = str(self.impl_constraints) + _repr += f"\t - Impl. constraints: {impl_constraints}\n" + _repr += f"\t - Impl. type : {str(self.impl_type)}\n" + _repr += f"\t - Impl. local : {str(self.impl_local)}\n" + _repr += f"\t - Impl. io : {str(self.impl_io)}\n" + _repr += f"\t - Impl. prolog : {str(self.impl_prolog)}\n" + _repr += f"\t - Impl. epilog : {str(self.impl_epilog)}\n" + _repr += f"\t - Impl. container : {str(self.impl_container)}\n" + _repr += f"\t - Impl. type args : {str(self.impl_type_args)}\n" + return _repr diff --git a/examples/dds/pycompss/runtime/task/definitions/function.py b/examples/dds/pycompss/runtime/task/definitions/function.py new file mode 100644 index 00000000..9e240e20 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/definitions/function.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Definitions - Function. + +This file contains the task function definition. +""" +import types + +from pycompss.util.typing_helper import typing # noqa: F401 +from pycompss.util.typing_helper import dummy_function + + +class FunctionDefinition: + """Task decorated function definition class. + + This class represents the decorated function definition attributes. + It contains the function related attributes. + """ + + __slots__ = [ + "function", + "registered", + "signature", + "interactive", + "module", + "function_name", + "module_name", + "function_type", + "class_name", + "code_strings", + ] + + def __init__(self) -> None: + """Set the default function definition values.""" + self.function = dummy_function # type: typing.Callable + # Global variables common for all tasks of this kind + self.registered = False + self.signature = "" + # Saved from the initial task + self.interactive = False + self.module = types.ModuleType("None") # type: types.ModuleType + self.function_name = "" + self.module_name = "" + self.function_type = -1 + self.class_name = "" + self.code_strings = False diff --git a/examples/dds/pycompss/runtime/task/features.py b/examples/dds/pycompss/runtime/task/features.py new file mode 100644 index 00000000..5847b739 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/features.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Features. + +This file contains the task features. +""" + + +class TaskFeatures: # pylint: disable=too-few-public-methods + """Features that any task may have.""" + + __slots__ = ["prepend_strings", "register_only", "object_conversion"] + + def __init__(self) -> None: + """Instantiate task features class.""" + # Determine if strings should have a sharp symbol prepended or not + self.prepend_strings = True + # Only register the task + self.register_only = False + # Convert small objects to string if enabled + self.object_conversion = False + + def get_prepend_strings(self) -> bool: + """Get PREPEND_STRINGS value. + + :return: Prepend strings value. + """ + return self.prepend_strings + + def get_register_only(self) -> bool: + """Get REGISTER_ONLY value. + + :return: Register only value. + """ + return self.register_only + + def get_object_conversion(self) -> bool: + """Get OBJECT_CONVERSION value. + + :return: Object conversion value. + """ + return self.object_conversion + + def set_prepend_strings(self, prepend_strings: bool) -> None: + """Set PREPEND_STRINGS value. + + :param: prepend_strings: New prepend_strings value. + :return: None + """ + self.prepend_strings = prepend_strings + + def set_register_only(self, register_only: bool) -> None: + """Set REGISTER_ONLY value. + + :param: register_only: New register only value. + :return: None + """ + self.register_only = register_only + + def set_object_conversion(self, object_conversion: bool) -> None: + """Set OBJECT_CONVERSION value. + + :param: object_conversion: New object conversion value. + :return: None + """ + self.object_conversion = object_conversion + + +TASK_FEATURES = TaskFeatures() diff --git a/examples/dds/pycompss/runtime/task/keys.py b/examples/dds/pycompss/runtime/task/keys.py new file mode 100644 index 00000000..6d2070b6 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/keys.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Keys. + +This file contains the task keys. +""" + + +class ParamAliasKeys: # pylint: disable=too-few-public-methods + """Strings used in Tasks definition.""" + + IN = "IN" + OUT = "OUT" + INOUT = "INOUT" + CONCURRENT = "CONCURRENT" + COMMUTATIVE = "COMMUTATIVE" + IN_DELETE = "IN_DELETE" + + FILE = "FILE" + FILE_IN = "FILE_IN" + FILE_OUT = "FILE_OUT" + FILE_INOUT = "FILE_INOUT" + FILE_CONCURRENT = "FILE_CONCURRENT" + FILE_COMMUTATIVE = "FILE_COMMUTATIVE" + + FILE_STDIN = "FILE_STDIN" + FILE_STDERR = "FILE_STDERR" + FILE_STDOUT = "FILE_STDOUT" + + FILE_IN_STDIN = "FILE_IN_STDIN" + FILE_IN_STDERR = "FILE_IN_STDERR" + FILE_IN_STDOUT = "FILE_IN_STDOUT" + FILE_OUT_STDIN = "FILE_OUT_STDIN" + FILE_OUT_STDERR = "FILE_OUT_STDERR" + FILE_OUT_STDOUT = "FILE_OUT_STDOUT" + FILE_INOUT_STDIN = "FILE_INOUT_STDIN" + FILE_INOUT_STDERR = "FILE_INOUT_STDERR" + FILE_INOUT_STDOUT = "FILE_INOUT_STDOUT" + FILE_CONCURRENT_STDIN = "FILE_CONCURRENT_STDIN" + FILE_CONCURRENT_STDERR = "FILE_CONCURRENT_STDERR" + FILE_CONCURRENT_STDOUT = "FILE_CONCURRENT_STDOUT" + FILE_COMMUTATIVE_STDIN = "FILE_COMMUTATIVE_STDIN" + FILE_COMMUTATIVE_STDERR = "FILE_COMMUTATIVE_STDERR" + FILE_COMMUTATIVE_STDOUT = "FILE_COMMUTATIVE_STDOUT" + + DIRECTORY = "DIRECTORY" + DIRECTORY_IN = "DIRECTORY_IN" + DIRECTORY_OUT = "DIRECTORY_OUT" + DIRECTORY_INOUT = "DIRECTORY_INOUT" + + COLLECTION = "COLLECTION" + COLLECTION_IN = "COLLECTION_IN" + COLLECTION_INOUT = "COLLECTION_INOUT" + COLLECTION_OUT = "COLLECTION_OUT" + COLLECTION_IN_DELETE = "COLLECTION_IN_DELETE" + COLLECTION_FILE = "COLLECTION_FILE" + COLLECTION_FILE_IN = "COLLECTION_FILE_IN" + COLLECTION_FILE_INOUT = "COLLECTION_FILE_INOUT" + COLLECTION_FILE_OUT = "COLLECTION_FILE_OUT" + + DICT_COLLECTION = "DICT_COLLECTION" + DICT_COLLECTION_IN = "DICT_COLLECTION_IN" + DICT_COLLECTION_INOUT = "DICT_COLLECTION_INOUT" + DICT_COLLECTION_OUT = "DICT_COLLECTION_OUT" + DICT_COLLECTION_IN_DELETE = "DICT_COLLECTION_IN_DELETE" + + STREAM_IN = "STREAM_IN" + STREAM_OUT = "STREAM_OUT" + + +class ParamDictKeys: # pylint: disable=too-few-public-methods + """Strings used in Parameter definition as dictionary.""" + + # Exposed to the user (see api/parameter.py) + TYPE = "type" + DIRECTION = "direction" + STDIOSTREAM = "stream" + PREFIX = "prefix" + DEPTH = "depth" + WEIGHT = "weight" + KEEP_RENAME = "keep_rename" + # Private (see task/parameter.py) + CONTENT_TYPE = "content_type" + IS_FILE_COLLECTION = "is_file_collection" + CACHE = "cache" + + +PARAM_ALIAS_KEYS = ParamAliasKeys() +PARAM_DICT_KEYS = ParamDictKeys() diff --git a/examples/dds/pycompss/runtime/task/master.py b/examples/dds/pycompss/runtime/task/master.py new file mode 100644 index 00000000..0fd428db --- /dev/null +++ b/examples/dds/pycompss/runtime/task/master.py @@ -0,0 +1,2815 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Master. + +This file contains the task core functions when acting as master. +""" + +import ast +import inspect +import logging +import pickle +import os +import re +import sys +import types +from base64 import b64encode +from collections import OrderedDict +from threading import Lock +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait +from typing import get_type_hints + +from pycompss.api import parameter +from pycompss.runtime import binding +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.commons.constants import LEGACY_LABELS +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.commons.error_msgs import cast_env_to_int_error +from pycompss.api.commons.implementation_types import IMPLEMENTATION_TYPES +from pycompss.api.parameter import DIRECTION +from pycompss.api.parameter import TYPE +from pycompss.runtime.binding import wait_on +from pycompss.runtime.commons import CONSTANTS +from pycompss.runtime.commons import GLOBALS +from pycompss.runtime.start.initialization import LAUNCH_STATUS +from pycompss.runtime.management.classes import FunctionType +from pycompss.runtime.management.classes import Future +from pycompss.runtime.management.direction import get_compss_direction +from pycompss.runtime.management.object_tracker import OT +from pycompss.runtime.task.arguments import get_kwarg_name +from pycompss.runtime.task.arguments import get_name_from_kwarg +from pycompss.runtime.task.arguments import get_return_name +from pycompss.runtime.task.arguments import get_vararg_name +from pycompss.runtime.task.arguments import is_kwarg +from pycompss.runtime.task.arguments import is_vararg +from pycompss.runtime.task.commons import get_default_direction +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.runtime.task.definitions.constraints import ConstraintDescription +from pycompss.runtime.task.definitions.arguments import TaskArguments +from pycompss.runtime.task.definitions.function import FunctionDefinition +from pycompss.runtime.task.features import TASK_FEATURES +from pycompss.runtime.task.keys import PARAM_ALIAS_KEYS +from pycompss.runtime.task.parameter import COMPSsFile +from pycompss.runtime.task.parameter import JAVA_MAX_INT +from pycompss.runtime.task.parameter import JAVA_MAX_LONG +from pycompss.runtime.task.parameter import JAVA_MIN_INT +from pycompss.runtime.task.parameter import JAVA_MIN_LONG +from pycompss.runtime.task.parameter import Parameter +from pycompss.runtime.task.parameter import UNDEFINED_CONTENT_TYPE +from pycompss.runtime.task.parameter import get_compss_type +from pycompss.runtime.task.parameter import get_new_parameter +from pycompss.runtime.task.wrappers.psco_stream import PscoStreamWrapper +from pycompss.util.arguments import check_arguments +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.exceptions import SerializerException +from pycompss.util.interactive.helpers import update_tasks_code_file +from pycompss.util.objects.properties import get_module_name +from pycompss.util.objects.properties import get_wrapped_source +from pycompss.util.objects.properties import is_basic_iterable +from pycompss.util.objects.properties import is_dict +from pycompss.util.objects.sizer import total_sizeof +from pycompss.util.serialization import serializer +from pycompss.util.serialization.serializer import serialize_to_file +from pycompss.util.storages.persistent import get_id +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.helpers import EventMaster +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.util.storages.persistent import is_psco + +logger = logging.getLogger(__name__) + +# Types conversion dictionary from python to COMPSs +_PYTHON_TO_COMPSS = { + int: TYPE.INT, # int # long + float: TYPE.DOUBLE, # float + bool: TYPE.BOOLEAN, # bool + str: TYPE.STRING, # str + # The type of instances of user-defined classes + # types.InstanceType: TYPE.OBJECT, + # The type of methods of user-defined class instances + # types.MethodType: TYPE.OBJECT, + # The type of user-defined old-style classes + # types.ClassType: TYPE.OBJECT, + # The type of modules + # types.ModuleType: TYPE.OBJECT, + # The type of tuples (e.g. (1, 2, 3, "Spam")) + tuple: TYPE.OBJECT, + # The type of lists (e.g. [0, 1, 2, 3]) + list: TYPE.OBJECT, + # The type of dictionaries (e.g. {"Bacon":1,"Ham":0}) + dict: TYPE.OBJECT, + # The type of generic objects + object: TYPE.OBJECT, +} # type: typing.Dict[typing.Type, int] + +MANDATORY_ARGUMENTS = set() # type: typing.Set[str] +# List since the parameter names are included before checking for unexpected +# arguments (the user can define a=INOUT in the task decorator and this is not +# an unexpected argument) +SUPPORTED_ARGUMENTS = { + LABELS.target_direction, + LABELS.returns, + LABELS.cache_returns, + LABELS.priority, + LABELS.defaults, + LABELS.time_out, + LABELS.is_replicated, + LABELS.is_distributed, + LABELS.computing_nodes, + LABELS.processes_per_node, + LABELS.is_reduce, + LABELS.chunk_size, + LABELS.on_failure, + LABELS.tracing_hook, + LABELS.numba, + LABELS.numba_flags, + LABELS.numba_signature, + LABELS.numba_declaration, + LABELS.varargs_type, + LABELS.config_file, +} # type: typing.Set[str] +# Deprecated arguments. Still supported but shows a message when used. +DEPRECATED_ARGUMENTS = { + LEGACY_LABELS.is_replicated, + LEGACY_LABELS.is_distributed, + LEGACY_LABELS.varargs_type, + LEGACY_LABELS.target_direction, + LEGACY_LABELS.time_out, +} # type: typing.Set[str] +# All supported arguments +ALL_SUPPORTED_ARGUMENTS = SUPPORTED_ARGUMENTS.union(DEPRECATED_ARGUMENTS) +# Some attributes cause memory leaks, we must delete them from memory after +# master call +ATTRIBUTES_TO_BE_REMOVED = { + "decorator_arguments", + "param_args", + "param_varargs", + "param_defaults", + "first_arg_name", + "parameters", + "returns", + "multi_return", +} + +# This lock allows tasks to be launched with the Threading module while +# ensuring that no attribute is overwritten +MASTER_LOCK = Lock() +VALUE_OF = "value_of" +RETURN_OPEN_SYMBOL = "{{" +RETURN_CLOSE_SYMBOL = "}}" + + +class TaskMaster: + """Task class representation for the Master. + + Process the task decorator and prepare all information to call binding + runtime. + """ + + __slots__ = [ + "user_function", + "core_element", + "decorator_arguments", + "decorated_function", + "param_args", + "param_varargs", + "param_defaults", + "first_arg_name", + "interactive_task_file", + "parameters", + "returns", + "constraint_args", + "registered_signatures", + ] + + def __init__( + self, + user_function: typing.Callable, + core_element: CE, + decorator_arguments: TaskArguments, + decorated_function: FunctionDefinition, + registered_signatures: typing.Dict[ + str, typing.Dict[str, typing.List[str]] + ] = {}, + constraint_args: typing.Dict[str, ConstraintDescription] = {}, + ) -> None: + """Task at master constructor. + + :param core_element: Core Element. + :param decorator_arguments: Decorator arguments. + :param decorated_function: Decorated function. + """ + self.user_function = user_function + # Task won't be registered until called from the master for the first + # time or have a different signature + self.core_element = core_element + # Initialize TaskCommons + self.decorator_arguments = decorator_arguments + self.decorated_function = decorated_function + # Initialize Master specific attributes + self.param_args = [] # type: typing.List[str] + self.param_varargs = "" # type: str + self.param_defaults = None # type: typing.Optional[tuple] + self.first_arg_name = "" + self.interactive_task_file = "" + self.parameters = ( + OrderedDict() + ) # type: typing.OrderedDict[str, Parameter] + self.returns = ( + OrderedDict() + ) # type: typing.OrderedDict[str, Parameter] + self.registered_signatures = registered_signatures + self.constraint_args = constraint_args + + def call( + self, args: tuple, kwargs: dict + ) -> typing.Tuple[typing.Any, CE, FunctionDefinition, dict, dict]: + """Run the task as master. + + This part deals with task calls in the master's side + Also, this function must return an appropriate number of + future objects that point to the appropriate objects/files. + + :return: A function that does "nothing" and returns futures if needed. + """ + # This lock makes this decorator able to handle various threads + # calling the same task concurrently + with MASTER_LOCK: + # Inspect the user function, get information about the arguments + # and their names. This defines self.param_args, + # self.param_varargs, and self.param_defaults. + # And gives non-None default values to them if necessary + with EventMaster(TRACING_MASTER.inspect_function_arguments): + self.inspect_user_function_arguments() + + # It will be easier to deal with functions if we pretend that + # all have the signature f(positionals, *variadic, **named). + # This is why we are substituting Nones with default stuff. + # As long as we remember what was the users original intention + # with the parameters we can internally mess with his signature + # as much as we want. There is no need to add self-imposed + # constraints here. Also, the very nature of decorators are a + # huge hint about how we should treat user functions, as most + # wrappers return a function f(*a, **k) + if self.param_varargs == "": + self.param_varargs = LABELS.varargs_type + if self.param_defaults is None: + self.param_defaults = () + + # Inspect the constraints to see if there are any dynamic + # constraints, in order to get the value from the given + # parameter name. + with EventMaster(TRACING_MASTER.inspect_constraints): + self.inspect_constraints(args, kwargs) + + # Compute the function path, class (if any), and name + with EventMaster(TRACING_MASTER.get_function_information): + self.compute_user_function_information(args) + + # Extract the core element (has to be extracted before processing + # the kwargs to avoid issues processing the parameters) + with EventMaster(TRACING_MASTER.extract_core_element): + ce = kwargs.pop(CORE_ELEMENT_KEY, None) + pre_defined_ce = self.extract_core_element(ce) + + with EventMaster(TRACING_MASTER.get_function_signature): + impl_signature, impl_type_args = self.get_signature() + + if __debug__: + logger.debug( + "TASK: %s of type %s, in module %s, in class %s", + self.decorated_function.function_name, + self.decorated_function.function_type, + self.decorated_function.module_name, + self.decorated_function.class_name, + ) + + if not GLOBALS.in_tracing_task_name_to_id(impl_signature): + GLOBALS.set_tracing_task_name_to_id( + impl_signature, GLOBALS.len_tracing_task_name_to_id() + 1 + ) + + emit_manual_event_explicit( + TRACING_WORKER.binding_tasks_func_type, + GLOBALS.get_tracing_task_name_id(impl_signature), + ) + + # Check if we are in interactive mode and update if needed + with EventMaster(TRACING_MASTER.check_interactive): + if self.decorated_function.interactive: + self.update_if_interactive() + else: + inter, inter_mod = self.check_if_interactive() + if inter: + self.update_if_interactive() + self.decorated_function.module = inter_mod + self.decorated_function.interactive = inter + + # Prepare the core element registration information + with EventMaster(TRACING_MASTER.prepare_core_element): + self.get_code_strings() + + # It is necessary to decide whether to register or not (the task + # may be inherited, and in this case it has to be registered again + # with the new implementation signature). + if ( + not self.decorated_function.registered + or self.decorated_function.signature != impl_signature + ): + with EventMaster(TRACING_MASTER.update_core_element): + self.update_core_element( + impl_signature, impl_type_args, pre_defined_ce + ) + if CONTEXT.is_loading(): + # This case will only happen with @implements since it + # calls explicitly to this call from his call. + CONTEXT.add_to_register_later((self, impl_signature)) + else: + if impl_signature not in self.registered_signatures: + self.register_task() + self.decorated_function.registered = True + self.decorated_function.signature = impl_signature + self.register_constraints() + elif __debug__: + logger.debug("Task already registered") + # _________________________________________________________________________________________________________________________________________ + # Did we call this function to only register the associated core + # element? (This can happen with @implements) + # Do not move this import: + if TASK_FEATURES.get_register_only(): + # Only register, no launch + future_object = None + else: + # Launch task to the runtime + # Extract task related parameters from upper decorators + # (e.g. returns, computing_nodes, etc.) + with EventMaster(TRACING_MASTER.get_upper_decorators_kwargs): + self.get_upper_decorators_kwargs(kwargs) + # Process any other decorator argument + with EventMaster(TRACING_MASTER.process_other_arguments): + # Check if the function is an instance method or + # a class method. + has_target = ( + self.decorated_function.function_type + == FunctionType.INSTANCE_METHOD + ) + is_http = ( + self.core_element.get_impl_type() + == IMPLEMENTATION_TYPES.http + ) + + # Process the parameters, give them a proper direction + with EventMaster(TRACING_MASTER.process_parameters): + code_strings = self.decorated_function.code_strings + self.process_parameters( + args, + kwargs, + code_strings=code_strings, + ) + + # Deal with the return part. + with EventMaster(TRACING_MASTER.process_return): + num_returns = self.add_return_parameters( + self.decorator_arguments.returns, + code_strings=code_strings, + ) + if not self.returns: + num_returns = self.update_return_if_no_returns( + self.decorated_function.function + ) + + # Build return objects + with EventMaster(TRACING_MASTER.build_return_objects): + future_object = None + if self.returns: + future_object = self._build_return_objects(num_returns) + + # Infer COMPSs types from real types, except for files + self._serialize_objects() + # Reset serializer forcing: + # May have been set by @http and the next task may not need + # it anymore. + self._reset_forced_serializer() + + # Build values and COMPSs types and directions + with EventMaster(TRACING_MASTER.build_compss_types_directions): + vtdsc = self._build_values_types_directions() + ( + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + ) = vtdsc # noqa + + if __debug__: + logger.debug( + "TASK: %s of type %s, in module %s, in class %s", + self.decorated_function.function_name, + self.decorated_function.function_type, + self.decorated_function.module_name, + self.decorated_function.class_name, + ) + + # Process the task + with EventMaster(TRACING_MASTER.process_task_binding): + binding.process_task( + impl_signature, + has_target, + names, + values, + num_returns, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + content_types, + weights, + keep_renames, + self.decorator_arguments, + is_http, + ) + + # Remove unused attributes from the memory + with EventMaster(TRACING_MASTER.attributes_cleanup): + for attribute in ATTRIBUTES_TO_BE_REMOVED: + if hasattr(self, attribute): + try: + delattr(self, attribute) + except AttributeError: + # Only happens when compiled + pass + + emit_manual_event_explicit( + TRACING_WORKER.binding_tasks_func_type, 0 + ) + + # The with statement automatically released the lock. + + # Return the future object/s corresponding to the task + # This object will substitute the user expected return from the task + # and will be used later for synchronization or as a task parameter + # (then the runtime will take care of the dependency). + # Also return if the task has been registered and its signature, + # so that future tasks of the same function register if necessary. + return ( + future_object, + self.core_element, + self.decorated_function, + self.registered_signatures, + self.constraint_args, + ) + + def register_constraints(self): + """Register constraints signature. + + Stores the signature, so it will not be registered + again. + + :return: the signatures are updated. + """ + if __debug__: + logger.debug("Registering signature") + signature = self.decorated_function.signature + self.registered_signatures[signature] = {} + constraints = self.core_element.get_impl_constraints() + for a in constraints: + self.registered_signatures[signature][a] = [] + self.registered_signatures[signature][a].append(constraints[a]) + + def inspect_constraints(self, args: tuple, kwargs: dict) -> None: + """Get constraint arguments. + + Inspect the arguments in the constraints and store them + with the updated parameter in the core element. + + Store the names of the arguments in the constraints + and if they are static, in order to be accessed to + check if the values have changed. + + :return: the attributes to be reused. + """ + if CORE_ELEMENT_KEY in kwargs: + if __debug__: + logger.debug("Inspecting constraint arguments first time") + constraints = kwargs[CORE_ELEMENT_KEY].get_impl_constraints() + for key, value in constraints.items(): + self.constraint_args[key] = ConstraintDescription(value) + if ( + isinstance(value, int) + or (isinstance(value, str) and value.isdigit()) + or (isinstance(value, str) and value.startswith("$")) + or ( + key != "computing_units" + and key != "computingUnits" + and key != "memory_size" + and key != "memorySize" + and key != "storage_size" + and key != "storageSize" + ) + ): + if __debug__: + logger.debug( + "Detected constraint as static value or env var" + ) + elif value in kwargs: + if __debug__: + logger.debug( + "Detected dynamic constraint passed as a dict" + ) + constraints[key] = kwargs[value] + self.constraint_args[key].set_is_static(False) + elif value in self.param_args: + if __debug__: + logger.debug( + "Detected dynamic constraint passed as a value" + ) + index = self.param_args.index(value) + constraints[key] = args[index] + self.constraint_args[key].set_is_static(False) + elif value in self.user_function.__globals__: + constraints[key] = int( + self.user_function.__globals__[value] + ) + self.constraint_args[key].set_is_static(False) + else: + try: + args_dict = { + self.param_args[i]: args[i] + for i in range(len(self.param_args)) + } + except IndexError: + args_dict = kwargs + else: + args_dict.update(kwargs) + try: + constraints[key] = int( + eval(value, {"__builtins__": {}}, args_dict) + ) + self.constraint_args[key].set_is_static(False) + if __debug__: + logger.debug( + "Detected dynamic constraint as an expression" + ) + except NameError: + if __debug__: + logger.debug( + "Parameter not found, treating value %s as " + "static string" % value + ) + kwargs[CORE_ELEMENT_KEY].set_impl_constraints(constraints) + elif self.core_element is not None: + constraints = self.core_element.get_impl_constraints() + for key, value in constraints.items(): + if not self.constraint_args[key].get_is_static(): + param_name = self.constraint_args[key].get_param_name() + if param_name in kwargs: + if value != kwargs[param_name]: + constraints[key] = kwargs[param_name] + elif param_name in self.param_args: + index = self.param_args.index(str(param_name)) + if value != args[index]: + constraints[key] = args[index] + elif param_name in self.user_function.__globals__: + constraints[key] = int( + self.user_function.__globals__[str(param_name)] + ) + else: + try: + args_dict = { + self.param_args[i]: args[i] + for i in range(len(self.param_args)) + } + except IndexError: + args_dict = kwargs + else: + args_dict.update(kwargs) + try: + constraints[key] = int( + eval( + str(param_name), + {"__builtins__": {}}, + args_dict, + ) + ) + except NameError: + if __debug__: + logger.debug( + "Parameter not found, treating value %s as" + " static string" % value + ) + + self.core_element.set_impl_constraints(constraints) + + def check_if_interactive(self) -> typing.Tuple[bool, types.ModuleType]: + """Check if running in interactive mode. + + :return: True if interactive. False otherwise. + """ + mod = inspect.getmodule(self.decorated_function.function) + if mod is None: + raise PyCOMPSsException( + f"Module of the {self.decorated_function.function} is None" + ) + else: + module_name = mod.__name__ + if CONTEXT.in_pycompss() and module_name in ( + "__main__", + "pycompss.runtime.launch", + ): + # 1.- The runtime is running. + # 2.- The module where the function is defined was run as __main__. + return True, mod + return False, types.ModuleType("None") + + def update_if_interactive(self) -> None: + """Update the code for jupyter notebook. + + Update the user code if in interactive mode and the session has + been started. + + :return: None. + """ + # We need to find out the real module name from launched + path = LAUNCH_STATUS.get_app_path() + # Get the file name + file_path, full_file_name = os.path.split(path) + file_name, file_name_ext = os.path.splitext(full_file_name) + # Do any necessary pre-processing action before executing any code + if file_name.startswith(CONSTANTS.interactive_file_name): + self.interactive_task_file = path + if not self.decorated_function.registered: + # If the file_name starts with "InteractiveMode" means that + # the user is using PyCOMPSs from jupyter-notebook. + # Convention between this file and interactive.py + # In this case it is necessary to do a pre-processing step + # that consists of putting all user code that may be executed + # in the worker on a file. + # This file will be sent to all workers as first parameter. + # Update the code also calls compss_delete_object if the file + # "path" is updated. + update_tasks_code_file(self.decorated_function.function, path) + # # It is possible to create a specific file per task + # file_name_fields = file_name.split("_") + # file_name_fields.insert(1, + # self.decorated_function.function_name) + # file_name_fields.insert(2, + # str(time.time_ns())) + # updated_file_name = "_".join(file_name_fields) + # updated_path = os.path.join( + # file_path, updated_file_name + file_name_ext + # ) + # # Copy file to accompany the task + # shutil.copyfile(path, updated_path) + + print(f"Found task: {self.decorated_function.function_name}") + # print(f"Written in file: {self.interactive_task_file}") + + def extract_core_element( + self, ce: typing.Optional[CE] + ) -> typing.Tuple[bool, bool]: + """Get or instantiate the Task's core element. + + Extract the core element if created in a higher level decorator, + uses an existing or creates a new one if does not. + + IMPORTANT! extract the core element from kwargs if pre-defined + in decorators defined on top of @task. + + :param ce: Core Element. + :return: If previously created and if created in higher level + decorator. + """ + pre_defined_core_element = False + upper_decorator = False + if ce: + # Core element has already been created in a higher level decorator + self.core_element = ce + pre_defined_core_element = True + upper_decorator = True + elif self.core_element: + # A core element from previous task calls was saved. + pre_defined_core_element = True + else: + # No decorators over @task: instantiate an empty core element. + self.core_element = CE() + return pre_defined_core_element, upper_decorator + + def inspect_user_function_arguments(self) -> None: + """Get user function arguments. + + Inspect the arguments of the user function and store them. + Read the names of the arguments and remember their order. + We will also learn things like if the user function contained + variadic arguments, named arguments and so on. + This will be useful when pairing arguments with the direction + the user has specified for them in the decorator. + + The third return value was self.param_kwargs - not used (removed). + + :return: the attributes to be reused. + """ + try: + arguments = self._getargspec(self.decorated_function.function) + except TypeError: + func_attrs = dir(self.decorated_function.function) + if "py_func" in func_attrs: + # This is a numba jit declared task + py_func = self.get_user_function_py_func() + arguments = self._getargspec(py_func) + else: + # This is a compiled function + wrapped_func = self.get_user_function_wrapped() + arguments = self._getargspec(wrapped_func) + param_args, param_varargs, param_defaults = arguments + self.param_args = param_args + if param_varargs is None: + self.param_varargs = "" + else: + self.param_varargs = param_varargs + self.param_defaults = param_defaults + + def is_numba_function(self) -> bool: + """Check if decorated function is compiled with numba. + + Check if self.decorated_function.function is in reality a numba + compiled function. + + :return: True if self.decorated_function.function has py_func. + """ + return "py_func" in dir(self.decorated_function.function) + + def get_user_function_py_func(self) -> typing.Callable: + """Retrieve py_func from self.decorated_function.function. + + WARNING!!! Only available in numba wrapped functions. + + :return: The self.decorated_function.function py_func. + """ + return self.decorated_function.function.py_func # type: ignore + + def user_func_py_func_glob_getter(self, field: str) -> typing.Any: + """Retrieve function from numba wrapped __globals__. + + Retrieve a field from __globals__ from py_func of + self.decorated_function.function. + + WARNING!!! Only available in numba wrapped functions. + + :param field: Field in the globals to get. + :return: __globals__ getter for the given field. + """ + py_func = self.get_user_function_py_func() + return py_func.__globals__.get(field) + + def get_user_function_wrapped(self) -> typing.Callable: + """Retrieve __wrapped__ from self.decorated_function.function. + + WARNING!!! Only available in compiled functions. + + :return: The user function from __wrapped__. + """ + return self.decorated_function.function.__wrapped__ # type: ignore + + def user_func_wrapped_glob_getter(self, field: str) -> typing.Any: + """Retrieve the user function from a numba wrapped function. + + Retrieve a field from __globals__ from __wrapped__ of + self.decorated_function.function. + + WARNING!!! Only available in compiled functions. + + :param field: Field in the wrapped globals to get. + :return: __globals__ getter for the given field result. + """ + wrapped_func = self.get_user_function_wrapped() + return wrapped_func.__globals__.get(field) + + def user_func_glob_getter(self, field: str) -> typing.Any: + """Retrieve the user function from py_func of numba compiled function. + + Retrieve a field from __globals__ from py_func of + self.decorated_function.function. + + WARNING!!! Only available in numba wrapped functions. + + :param field: Field in the function globals to get. + :return: __globals__ getter for the given field result. + """ + return self.decorated_function.function.__globals__.get(field) + + @staticmethod + def _getargspec( + function: typing.Callable, + ) -> typing.Tuple[ + typing.List[str], typing.Optional[str], typing.Optional[tuple] + ]: + """Private method that retrieves the function argspec. + + :param function: Function to analyse. + :return: args, varargs and defaults dictionaries. + """ + full_argspec = inspect.getfullargspec(function) + as_args = full_argspec.args + as_varargs = full_argspec.varargs + as_defaults = full_argspec.defaults + return as_args, as_varargs, as_defaults + + def get_upper_decorators_kwargs(self, kwargs: dict) -> None: + """Extract all @task related parameters placed by upper decorators. + + The kwargs are received from higher level decorators!!! + + Updates: + - self.decorator_arguments: + - self.decorator_arguments.returns + - self.decorator_arguments.computing_nodes + - self.decorator_arguments.processes_per_node + - self.decorator_arguments.on_failure + - self.decorator_arguments.is_reduce + - self.decorator_arguments.chunk_size + + :param kwargs: @task decorator keyword arguments. + :return: Number of computing nodes. + """ + # Pop returns from kwargs + if LABELS.returns in kwargs: + self.decorator_arguments.returns = kwargs.pop(LABELS.returns, None) + + # Deal with dynamic computing nodes + # If we have an MPI, COMPSs or MultiNode decorator above us we should + # have computing_nodes as a kwarg, we should detect it and remove it. + # Otherwise we set it to 1 + if LABELS.computing_nodes in kwargs: + cns = kwargs.pop(LABELS.computing_nodes, 1) + elif LEGACY_LABELS.computing_nodes in kwargs: + cns = kwargs.pop(LEGACY_LABELS.computing_nodes, 1) + else: + cns = self.decorator_arguments.computing_nodes + if cns != 1: + # Non default => parse + computing_nodes = self.parse_computing_nodes(cns) + else: + computing_nodes = 1 + self.decorator_arguments.computing_nodes = computing_nodes + + # Deal with processes per node + if LABELS.processes_per_node in kwargs: + processes_per_node = kwargs.pop(LABELS.processes_per_node, 1) + else: + processes_per_node = self.decorator_arguments.processes_per_node + if processes_per_node != 1: + # Non default => parse + processes_per_node = self.parse_processes_per_node( + processes_per_node + ) + else: + processes_per_node = 1 + self.decorator_arguments.processes_per_node = processes_per_node + if processes_per_node > 1: + if self.core_element.impl_type != IMPLEMENTATION_TYPES.multi_node: + # Check processes per node + self.validate_processes_per_node( + computing_nodes, processes_per_node + ) + computing_nodes = int(computing_nodes / processes_per_node) + self.decorator_arguments.computing_nodes = computing_nodes + + # Deal with on_failure + if LABELS.on_failure in kwargs: + self.decorator_arguments.on_failure = kwargs.pop( + LABELS.on_failure, "RETRY" + ) + + # Deal with defaults + if LABELS.defaults in kwargs: + self.decorator_arguments.defaults = kwargs.pop(LABELS.defaults, {}) + + # Deal with reductions + if LABELS.is_reduce in kwargs: + is_reduce = kwargs.pop(LABELS.is_reduce, False) + else: + is_reduce = self.decorator_arguments.is_reduce + if is_reduce is not False: + updated_is_reduce = self.parse_is_reduce(is_reduce) + else: + updated_is_reduce = False + self.decorator_arguments.is_reduce = updated_is_reduce + + # Deal with chunk size + if LABELS.chunk_size in kwargs: + chunk_size = kwargs.pop(LABELS.chunk_size, 0) + else: + chunk_size = self.decorator_arguments.chunk_size + if chunk_size != 0: + updated_chunk_size = self.parse_chunk_size(chunk_size) + else: + updated_chunk_size = 0 + self.decorator_arguments.chunk_size = updated_chunk_size + + def process_parameters( + self, args: tuple, kwargs: dict, code_strings: bool = True + ) -> None: + """Process all the input parameters. + + Basically, processing means "build a dictionary of , + where each parameter has an associated Parameter object". + This function also assigns default directions to parameters. + + :param args: Arguments. + :param kwargs: Keyword arguments. + :param code_strings: Code strings. + :return: None, it only modifies self.parameters. + """ + # It is important to know the name of the first argument to determine + # if we are dealing with a class or instance method (i.e: first + # argument is named self) + # Process the positional arguments and fill self.parameters with + # their corresponding Parameter object + # Some of these positional arguments may have been not + # explicitly defined + num_positionals = min(len(self.param_args), len(args)) + arg_names = self.param_args[:num_positionals] + arg_objects = args[:num_positionals] + for arg_name, arg_object in zip(arg_names, arg_objects): + self.parameters[arg_name] = self.build_parameter_object( + arg_name, arg_object, code_strings=code_strings + ) + + # Check defaults + if self.param_defaults: + num_defaults = len(self.param_defaults) + if num_defaults > 0: + # Give default values to all the parameters that have a + # default value and are not already set + # As an important observation, defaults are matched as follows: + # defaults[-1] goes with positionals[-1] + # defaults[-2] goes with positionals[-2] + # ... + # Also, |defaults| <= |positionals| + for arg_name, default_value in reversed( + list( + zip( + list(reversed(self.param_args))[:num_defaults], + list(reversed(self.param_defaults)), + ) + ) + ): + if arg_name not in self.parameters: + real_arg_name = get_kwarg_name(arg_name) + self.parameters[real_arg_name] = ( + self.build_parameter_object( + real_arg_name, + default_value, + code_strings=code_strings, + ) + ) + + # Process variadic and keyword arguments + # Note that they are stored with custom names + # This will allow us to determine the class of each parameter + # and their order in the case of the variadic ones + # Process the variadic arguments + supported_varargs = [] + for i, var_arg in enumerate(args[num_positionals:]): + arg_name = get_vararg_name(self.param_varargs, i) + self.parameters[arg_name] = self.build_parameter_object( + arg_name, var_arg, code_strings=code_strings + ) + if self.param_varargs not in supported_varargs: + supported_varargs.append(self.param_varargs) + # Process keyword arguments + supported_kwargs = [] + for name, value in kwargs.items(): + arg_name = get_kwarg_name(name) + self.parameters[arg_name] = self.build_parameter_object( + arg_name, value, code_strings=code_strings + ) + if name not in supported_kwargs: + supported_kwargs.append(name) + # Check the arguments - Look for mandatory and unexpected arguments + supported_arguments = ALL_SUPPORTED_ARGUMENTS.union(self.param_args) + supported_arguments = supported_arguments.union(supported_varargs) + supported_arguments = supported_arguments.union(supported_kwargs) + check_arguments( + MANDATORY_ARGUMENTS, + DEPRECATED_ARGUMENTS, + supported_arguments, + self.decorator_arguments.get_keys() + + list(self.decorator_arguments.parameters.keys()), + "@task", + ) + # Add interactive FILE_IN parameter if necessary + if ( + self.interactive_task_file != "" + and self.core_element.get_impl_type() + == IMPLEMENTATION_TYPES.method + ): + self.add_synthetic_parameter_interactive_file() + + def add_synthetic_parameter_interactive_file(self) -> None: + """Include a synthetic Parameter object for interactive source file. + + This function includes a special FILE_IN parameter with the interactive + source file. This parameter will be removed in the worker side. + However, it will be used to transfer the interactive source file + automatically by the runtime. + + :return: None + """ + direction = PARAM_ALIAS_KEYS.FILE_IN + param = get_new_parameter(direction) + param.file_name = COMPSsFile(str(self.interactive_task_file)) + param.extra_content_type = "FILE" + self.parameters[CONSTANTS.compss_interactive_source_file] = param + + def build_parameter_object( + self, arg_name: str, arg_object: typing.Any, code_strings=True + ) -> Parameter: + """Create the Parameter object from an argument name and object. + + WARNING: Any modification in the param object will modify the + original Parameter set in the task.py __init__ constructor + for the rest of the task calls. + + :param arg_name: Argument name. + :param arg_object: Argument object. + :param code_strings: Code strings. + :return: Parameter object. + """ + # Is the argument a vararg? or a kwarg? Then check the direction + # for varargs or kwargs + if is_vararg(arg_name): + varargs_direction = self.decorator_arguments.varargs_type + param = get_new_parameter(varargs_direction) + elif is_kwarg(arg_name): + real_name = get_name_from_kwarg(arg_name) + default_parameter = get_default_direction( + real_name, self.decorator_arguments, self.param_args + ) + param = self.decorator_arguments.get_parameter( + real_name, default_parameter + ) + else: + # The argument is named, check its direction + # Default value = IN if not class or instance method and + # isModifier, INOUT otherwise + # see self.get_default_direction + # Note that if we have something like @task(self = IN) it + # will have priority over the default + # direction resolution, even if this implies a contradiction + # with the target_direction flag + default_parameter = get_default_direction( + arg_name, self.decorator_arguments, self.param_args + ) + param = self.decorator_arguments.get_parameter( + arg_name, default_parameter + ) + + # If the parameter is a FILE then its type will already be defined, + # and get_compss_type will misslabel it as a parameter.TYPE.STRING + if param.is_object(): + param.content_type = get_compss_type( + arg_object, code_strings=code_strings + ) + + # Set if the object is really a future. + if isinstance(arg_object, Future): + param.is_future = True + + # If the parameter is a DIRECTORY or FILE update the file_name + # or content type depending if object. Otherwise, update content. + if param.is_file() or param.is_directory(): + if isinstance(arg_object, COMPSsFile): + param.file_name = arg_object + else: + param.file_name = COMPSsFile(str(arg_object)) + # todo: beautify this + param.extra_content_type = "FILE" + else: + param.extra_content_type = str(type(arg_object)) + param.content = arg_object + return param + + def compute_user_function_information(self, args: tuple) -> None: + """Get the user function path and name. + + Compute the function path p and the name n such that + "from p import n" imports self.decorated_function.function. + + :return: None, it just sets self.user_function_path and + self.user_function_name. + """ + self.decorated_function.function_name = ( + self.decorated_function.function.__name__ + ) + # Detect if self is present + num_positionals = min(len(self.param_args), len(args)) + arg_names = self.param_args[:num_positionals] + first_object = None + if arg_names and self.first_arg_name == "": + self.first_arg_name = arg_names[0] + first_object = args[0] + # Get the module name (the x part "from x import y"), except for the + # class name + self.compute_module_name(first_object) + # Get the function type (function, instance method, class method) + self.compute_function_type(first_object) + + def compute_module_name(self, first_object: typing.Any) -> None: + """Compute the user's function module name. + + There are various cases: + 1) The user function is defined in some file. This is easy, just + get the module returned by inspect.getmodule. + 2) The user function is in the main module. Retrieve the file and + build the import name from it. + 3) We are in interactive mode. + + :return: None, it just modifies self.decorated_function.module_name. + """ + mod = inspect.getmodule(self.decorated_function.function) + if mod is None: + raise PyCOMPSsException( + f"Module of the {self.decorated_function.function} is None" + ) + else: + mod_name = mod.__name__ + self.decorated_function.module_name = mod_name + # If it is a task within a class, the module it will be where the one + # where the class is defined, instead of the one where the task is + # defined. + # This avoids conflicts with task inheritance. + if self.first_arg_name == "self": + mod = inspect.getmodule(type(first_object)) + self.decorated_function.module_name = mod_name + elif self.first_arg_name == "cls": + self.decorated_function.module_name = first_object.__module__ + if self.decorated_function.module_name in ( + "__main__", + "pycompss.runtime.launch", + ): + # The module where the function is defined was run as __main__, + # We need to find out the real module name + path = LAUNCH_STATUS.get_app_path() + # Get the file name + file_name = os.path.splitext(os.path.basename(path))[0] + # Get the module + self.decorated_function.module_name = get_module_name( + path, file_name + ) + + def compute_function_type(self, first_object: typing.Any) -> None: + """Compute user function type. + + Compute some properties of the user function, as its name, + its import path, and its type (module function, instance method, + class method), etc. + + :return: None, just updates self.decorated_function.class_name and + self.decorated_function.function_type. + """ + # Check the type of the function called. + # inspect.ismethod(f) does not work here, + # for methods python hasn't wrapped the function as a method yet + # Everything is still a function here, can't distinguish yet + # with inspect.ismethod or isfunction + self.decorated_function.function_type = FunctionType.FUNCTION + self.decorated_function.class_name = "" + if self.first_arg_name == "self": + self.decorated_function.function_type = ( + FunctionType.INSTANCE_METHOD + ) + self.decorated_function.class_name = type(first_object).__name__ + elif self.first_arg_name == "cls": + self.decorated_function.function_type = FunctionType.CLASS_METHOD + self.decorated_function.class_name = first_object.__name__ + # Finally, check if the function type is really a module function or + # a static method. + # Static methods are ONLY supported with Python 3 due to __qualname__ + # feature, which enables to know to which class they belong. + # The class name is needed in order to define properly the class_name + # for the correct registration and later invoke. + # Since these methods don't have self, nor cls, they are considered as + # FUNCTIONS to the runtime + name = str(self.decorated_function.function_name) + qualified_name = str(self.decorated_function.function.__qualname__) + if name != qualified_name: + # Then there is a class definition before the name in the + # qualified name + self.decorated_function.class_name = qualified_name[ + : -len(name) - 1 + ] + # -1 to remove the last point + + def get_code_strings(self) -> None: + """Get if the strings must be coded or not. + + IMPORTANT! modify f adding __code_strings__ which is used in binding. + + :return: None. + """ + ce_type = self.core_element.get_impl_type() + default = IMPLEMENTATION_TYPES.method + if ce_type is None or (isinstance(ce_type, str) and ce_type == ""): + ce_type = default + # code_strings = True if default, python_mpi or multinode + # code_strings = False if mpi, binary and container + code_strings = bool( + ce_type + in ( + default, + IMPLEMENTATION_TYPES.python_mpi, + IMPLEMENTATION_TYPES.multi_node, + ) + ) + self.decorated_function.code_strings = code_strings + + if __debug__: + logger.debug( + "[@TASK] Task type of function %s in module %s: %s", + self.decorated_function.function_name, + self.decorated_function.module_name, + str(ce_type), + ) + + def get_signature(self) -> typing.Tuple[str, list]: + """Find out the function signature. + + The information is needed in order to compare the implementation + signature, so that if it has been registered with a different + signature, it can be re-registered with the new one (enable + inheritance). + + :return: Implementation signature and implementation type arguments. + """ + module_name = self.decorated_function.module_name + class_name = self.decorated_function.class_name + function_name = self.decorated_function.function_name + if self.decorated_function.class_name != "": + # Within class or subclass + impl_signature = ".".join([module_name, class_name, function_name]) + impl_type_args = [ + ".".join([module_name, class_name]), + function_name, + ] + else: + # The task is defined within the main app file. + # Not in a class or subclass + # This case can be reached in Python 3, where particular + # frames are included, but not class names found. + impl_signature = ".".join([module_name, function_name]) + impl_type_args = [module_name, function_name] + constraints = self.core_element.get_impl_constraints() + for key, value in constraints.items(): + if not self.constraint_args[key].get_is_static(): + impl_signature += "." + if key.__contains__("_"): + impl_signature += key.split("_", 1)[0][:1] + impl_signature += key.split("_", 1)[1][:1] + else: + upper_letter = re.findall("[A-Z]+", key) + impl_signature += key[:1] + impl_signature += upper_letter[0][:1] + impl_signature += str(value) + return impl_signature, impl_type_args + + def update_core_element( + self, + impl_signature: str, + impl_type_args: list, + pre_defined_ce: typing.Tuple[bool, bool], + ) -> None: + """Add the @task decorator information to the core element. + + CAUTION: Modifies the core_element parameter. + + :param impl_signature: Implementation signature. + :param impl_type_args: Implementation type arguments. + :param pre_defined_ce: Two boolean (if core element contains + predefined fields and if they have been + predefined by upper decorators). + :return: None. + """ + pre_defined_core_element = pre_defined_ce[0] + upper_decorator = pre_defined_ce[1] + + # Include the registering info related to @task + impl_type = IMPLEMENTATION_TYPES.method + impl_constraints = {} # type: dict + impl_local = False + impl_io = False + + if __debug__: + logger.debug("Configuring core element.") + + set_ce_signature = self.core_element.set_ce_signature + set_impl_signature = self.core_element.set_impl_signature + set_impl_type_args = self.core_element.set_impl_type_args + set_impl_constraints = self.core_element.set_impl_constraints + set_impl_type = self.core_element.set_impl_type + set_impl_local = self.core_element.set_impl_local + set_impl_io = self.core_element.set_impl_io + if pre_defined_core_element: + # Core element has already been created in an upper decorator + # (e.g. @implements and @compss) + if __debug__: + logger.debug("Predefined core element.") + _ce_signature = self.core_element.get_ce_signature() + _impl_constraints = self.core_element.get_impl_constraints() + _impl_type = self.core_element.get_impl_type() + _impl_type_args = self.core_element.get_impl_type_args() + _impl_local = self.core_element.get_impl_local() + _impl_io = self.core_element.get_impl_io() + if _ce_signature == "": + set_ce_signature(impl_signature) + set_impl_signature(impl_signature) + elif _ce_signature != impl_signature and not upper_decorator: + # Specific for inheritance - not for @implements. + set_ce_signature(impl_signature) + set_impl_signature(impl_signature) + set_impl_type_args(impl_type_args) + else: + # If we are here that means that we come from an implements + # decorator, which means that this core element has already + # a signature + set_impl_signature(impl_signature) + set_impl_type_args(impl_type_args) + if not _impl_constraints: + set_impl_constraints(impl_constraints) + if not _impl_type: + set_impl_type(impl_type) + if not _impl_type_args: + set_impl_type_args(impl_type_args) + # Need to update impl_type_args if task is PYTHON_MPI and + # if the parameter with layout exists. + if _impl_type == IMPLEMENTATION_TYPES.python_mpi: + self.check_layout_params(_impl_type_args) + set_impl_signature( + ".".join([IMPLEMENTATION_TYPES.mpi, impl_signature]) + ) + if _impl_type_args: + set_impl_type_args(impl_type_args + _impl_type_args[1:]) + else: + set_impl_type_args(impl_type_args) + elif _impl_type == IMPLEMENTATION_TYPES.multi_node: + if _impl_type_args: + set_impl_type_args(impl_type_args + _impl_type_args) + if not _impl_local: + set_impl_local(impl_local) + if not _impl_io: + set_impl_io(impl_io) + else: + # @task is in the top of the decorators stack. + # Update the empty core_element + self.core_element = CE( + impl_signature, + impl_signature, + impl_constraints, + impl_type, + impl_local, + impl_io, + impl_type_args=impl_type_args, + ) + + def check_layout_params(self, impl_type_args: typing.List[str]) -> None: + """Check the layout parameter format. + + :param impl_type_args: Parameter arguments. + :return: None. + """ + # todo: replace these INDEXES with CONSTANTS + num_layouts = int(impl_type_args[8]) + if num_layouts > 0: + for i in range(num_layouts): + param_name = impl_type_args[(9 + (i * 4))].strip() + if param_name: + if param_name in self.decorator_arguments.parameters: + if ( + self.decorator_arguments.parameters[ + param_name + ].content_type + != parameter.TYPE.COLLECTION + ): + raise PyCOMPSsException( + f"Parameter {param_name} is not a collection!" + ) + else: + raise PyCOMPSsException( + f"Parameter {param_name} does not exist!" + ) + + def register_task(self) -> None: + """Register the task in the runtime. + + This registration must be done only once on the task decorator + initialization, unless there is a signature change (this will mean + that the user has changed the implementation interactively). + + :return: None. + """ + if __debug__: + logger.debug( + "[@TASK] Registering the function %s in module %s", + self.decorated_function.function_name, + self.decorated_function.module_name, + ) + binding.register_ce(self.core_element) + + @staticmethod + def validate_processes_per_node( + computing_nodes: int, processes_per_node: int + ) -> None: + """Check the processes per node property. + + :return: None. + """ + if computing_nodes < processes_per_node: + raise PyCOMPSsException( + "Processes is smaller than processes_per_node." + ) + if (computing_nodes % processes_per_node) > 0: + raise PyCOMPSsException( + "Processes is not a multiple of processes_per_node." + ) + + def parse_processes_per_node( + self, processes_per_node: typing.Union[int, str] + ) -> int: + """Retrieve the number of processes per node. + + This value can be defined by upper decorators and can also be defined + dynamically defined with a global or environment variable. + + :return: The number of processes per node. + """ + parsed_processes_per_node = 1 + if isinstance(processes_per_node, int): + # Nothing to do + parsed_processes_per_node = processes_per_node + elif isinstance(processes_per_node, str): + # Check if processes_per_node can be casted to string + # Check if processes_per_node is an environment variable + # Check if processes_per_node is a dynamic global variable + try: + # Cast string to int + parsed_processes_per_node = int(processes_per_node) + except ValueError: + # Environment variable + if processes_per_node.strip().startswith("$"): + # Computing nodes is an ENV variable, load it + env_var = processes_per_node.strip()[1:] # Remove $ + if env_var.startswith("{"): + env_var = env_var[1:-1] # remove brackets + try: + parsed_processes_per_node = int(os.environ[env_var]) + except ValueError as value_error: + raise PyCOMPSsException( + cast_env_to_int_error("ComputingNodes") + ) from value_error + else: + # Dynamic global variable + try: + # Load from global variables + parsed_processes_per_node = self.user_func_glob_getter( + processes_per_node + ) + except AttributeError: + # This is a numba jit declared task + try: + if self.is_numba_function(): + parsed_processes_per_node = ( + self.user_func_py_func_glob_getter( + processes_per_node + ) + ) + else: + parsed_processes_per_node = ( + self.user_func_wrapped_glob_getter( + processes_per_node + ) + ) + except AttributeError as attribute_error: + # No more chances + # Ignore error and parsed_processes_per_node will + # raise the exception + raise PyCOMPSsException( + "ERROR: Wrong Computing Nodes value." + ) from attribute_error + else: + raise PyCOMPSsException( + "Unexpected processes_per_node value. Must be str or int." + ) + + if parsed_processes_per_node <= 0: + logger.warning( + "Registered processes_per_node is less than 1 (%s <= 0). " + "Automatically set it to 1", + str(parsed_processes_per_node), + ) + parsed_processes_per_node = 1 + + return parsed_processes_per_node + + def parse_computing_nodes( + self, computing_nodes: typing.Union[int, str] + ) -> int: + """Retrieve the number of computing nodes. + + This value can be defined by upper decorators and can also be defined + dynamically defined with a global or environment variable. + + :return: The number of computing nodes. + """ + parsed_computing_nodes = 1 + if isinstance(computing_nodes, int): + # Nothing to do + parsed_computing_nodes = computing_nodes + elif isinstance(computing_nodes, str): + # Check if computing_nodes can be casted to string + # Check if computing_nodes is an environment variable + # Check if computing_nodes is a dynamic global variable + try: + # Cast string to int + parsed_computing_nodes = int(computing_nodes) + except ValueError: + # Environment variable + if computing_nodes.strip().startswith("$"): + # Computing nodes is an ENV variable, load it + env_var = computing_nodes.strip()[1:] # Remove $ + if env_var.startswith("{"): + env_var = env_var[1:-1] # remove brackets + try: + parsed_computing_nodes = int(os.environ[env_var]) + except ValueError as value_error: + raise PyCOMPSsException( + cast_env_to_int_error("ComputingNodes") + ) from value_error + else: + # Dynamic global variable + try: + # Load from global variables + parsed_computing_nodes = self.user_func_glob_getter( + computing_nodes + ) + except AttributeError: + # This is a numba jit declared task + try: + if self.is_numba_function(): + parsed_computing_nodes = ( + self.user_func_py_func_glob_getter( + computing_nodes + ) + ) + else: + parsed_computing_nodes = ( + self.user_func_wrapped_glob_getter( + computing_nodes + ) + ) + except AttributeError as attribute_error: + # No more chances + # Ignore error and parsed_computing_nodes will + # raise the exception + raise PyCOMPSsException( + "ERROR: Wrong Computing Nodes value." + ) from attribute_error + else: + raise PyCOMPSsException( + f"Unexpected computing_nodes value {computing_nodes}. " + f"Must be str or int." + ) + + if parsed_computing_nodes <= 0: + logger.warning( + "Registered computing_nodes is less than 1 (%s <= 0). " + "Automatically set it to 1", + str(parsed_computing_nodes), + ) + parsed_computing_nodes = 1 + + return parsed_computing_nodes + + def parse_chunk_size(self, chunk_size: typing.Union[str, int]) -> int: + """Parse the chunk size value. + + :param chunk_size: Chunk size defined in the @task decorator + :return: Chunk size as integer. + """ + if isinstance(chunk_size, int): + return chunk_size + if isinstance(chunk_size, str): + # Check if chunk_size can be cast to string + # Check if chunk_size is an environment variable + # Check if chunk_size is a dynamic global variable + try: + # Cast string to int + return int(chunk_size) + except ValueError: + # Environment variable + if chunk_size.strip().startswith("$"): + # Chunk size is an ENV variable, load it + env_var = chunk_size.strip()[1:] # Remove $ + if env_var.startswith("{"): + env_var = env_var[1:-1] # remove brackets + try: + return int(os.environ[env_var]) + except ValueError as value_error: + raise PyCOMPSsException( + cast_env_to_int_error("ChunkSize") + ) from value_error + else: + # Dynamic global variable + try: + # Load from global variables + return self.user_func_glob_getter(chunk_size) + except AttributeError: + # This is a numba jit declared task + try: + if self.is_numba_function(): + return self.user_func_py_func_glob_getter( + chunk_size + ) + return self.user_func_wrapped_glob_getter( + chunk_size + ) + except AttributeError as attribute_error: + # No more chances + # Ignore error and parsed_chunk_size will + # raise the exception + raise PyCOMPSsException( + "ERROR: Wrong chunk_size value." + ) from attribute_error + raise PyCOMPSsException( + "Unexpected chunk_size value. Must be str or int." + ) + + @staticmethod + def parse_is_reduce(is_reduce: typing.Union[bool, str]) -> bool: + """Parse the is_reduce parameter. + + :return: If it is a reduction or not. + """ + if isinstance(is_reduce, bool): + # Nothing to do + return is_reduce + if isinstance(is_reduce, str): + # Check if is_reduce can be cast to string + try: + # Cast string to int + return bool(is_reduce) + except ValueError: + return False + raise PyCOMPSsException( + "Unexpected is_reduce value. Must be bool or str." + ) + + def add_return_parameters( + self, returns: typing.Any, code_strings: bool = True + ) -> int: + """Modify the return parameters accordingly to the return statement. + + :return: Creates and modifies self.returns and returns the number of + returns. + """ + if returns: + _returns = returns # type: typing.Any + else: + _returns = self.decorator_arguments.returns + + # Note that RETURNS is by default False + if not _returns: + return 0 + + # A return statement can be the following: + # 1) A type. This means "this task returns an object of this type" + # 2) An integer N. This means "this task returns N objects" + # 3) A basic iterable (tuple, list...). This means "this task + # returns an iterable with the indicated elements inside + # We are returning multiple objects until otherwise proven + # It is important to know because this will determine if we will + # return a single object or [a single object] in some cases + defined_type = False + to_return = None # type: typing.Any + if isinstance(_returns, str): + # Check if the returns statement contains a string with an + # integer or a global variable. + # In such case, build a list of objects of value length and + # set it in ret_type. + # Global variable, value_of(Parameter) or string wrapping integer + # value (Evaluated in reverse order) + num_rets = self.get_num_returns_from_string(_returns) + # Construct hidden multi-return + if num_rets > 1: + to_return = num_rets + else: + to_return = 1 + elif is_basic_iterable(_returns): + # The task returns a basic iterable with some types + # already defined + to_return = _returns + defined_type = True + elif isinstance(_returns, int): + # The task returns a list of N objects, defined by the int N + to_return = _returns + else: + # The task returns a single object of a single type + # This is also the only case when no multiple objects are + # returned but only one + to_return = 1 + defined_type = True + + # At this point we have a list of returns + ret_dir = DIRECTION.OUT + if defined_type: + if to_return == 1: + ret_type = get_compss_type(_returns, code_strings=code_strings) + self.returns[get_return_name(0)] = Parameter( + content=_returns, content_type=ret_type, direction=ret_dir + ) + else: + for i, elem in enumerate(to_return): # noqa + ret_type = get_compss_type(elem, code_strings=code_strings) + self.returns[get_return_name(i)] = Parameter( + content=elem, content_type=ret_type, direction=ret_dir + ) + else: + ret_type = TYPE.OBJECT + for i in range(to_return): + self.returns[get_return_name(i)] = Parameter( + content=_returns, content_type=ret_type, direction=ret_dir + ) + + # Hopefully, an exception have been thrown if some invalid + # stuff has been put in the returns field + if defined_type: + if to_return == 1: + return to_return + return len(to_return) # noqa + return to_return + + def get_num_returns_from_string(self, returns: str) -> int: + """Convert the number of returns from string to integer. + + :param returns: Returns as string. + :return: Number of returned parameters. + """ + try: + # Return is hidden by an int as a string. + # i.e., returns="var_int" + return int(returns) + except ValueError as value_error: + if returns.startswith(VALUE_OF): + # from "value_of ( xxx.yyy )" to [xxx, yyy] + param_ref = ( + returns.replace(VALUE_OF, "") + .replace("(", "") + .replace(")", "") + .strip() + .split(".") + ) # noqa: E501 + if len(param_ref) > 0: + obj = self.parameters[param_ref[0]].content + return int(_get_object_property(param_ref, obj)) + raise PyCOMPSsException( + f"Incorrect value_of format in {returns}" + ) from value_error + + # for cases like returns = "{{a}}" there can be only + # 1 return value + elif self._is_return_param_name(returns): + return 1 + + # Else: return is hidden by a global variable. i.e., LT_ARGS + try: + num_rets = self.user_func_glob_getter(returns) + except AttributeError: + if self.is_numba_function(): + # This is a numba jit declared task + num_rets = self.user_func_py_func_glob_getter(returns) + else: + # This is a compiled task + num_rets = self.user_func_wrapped_glob_getter(returns) + return int(num_rets) + + @staticmethod + def _is_return_param_name(returns_str): + return ( + isinstance(returns_str, str) + and returns_str.startswith(RETURN_OPEN_SYMBOL) + and returns_str.endswith(RETURN_CLOSE_SYMBOL) + ) + + def update_return_if_no_returns(self, function: typing.Callable) -> int: + """Look for returns if no returns is specified. + + Checks the code looking for return statements if no returns is + specified in @task decorator. + + WARNING: Updates self.returns if returns are found. + + :param function: Function to check. + :return: The number of return elements if found. + """ + type_hints = get_type_hints(function) + if "return" in type_hints: + # There is a return defined as type-hint + ret = type_hints["return"] + try: + if hasattr(ret, "__len__"): + num_returns = len(ret) + else: + num_returns = 1 + except TypeError: + # Is not iterable, so consider just 1 + num_returns = 1 + if num_returns > 1: + for i in range(num_returns): + param = Parameter( + content_type=parameter.TYPE.FILE, + direction=parameter.DIRECTION.OUT, + ) + param.content = object() + self.returns[get_return_name(i)] = param + else: + param = Parameter( + content_type=parameter.TYPE.FILE, + direction=parameter.DIRECTION.OUT, + ) + param.content = object() + self.returns[get_return_name(0)] = param + # Found return defined as type-hint + return num_returns + # else: + # # The user has not defined return as type-hint + # # So, continue searching as usual + # pass + + # Could not find type-hinting + source_code = get_wrapped_source(function).strip() + + code = [] # type: list + if self.first_arg_name == "self" or source_code.startswith( + "@classmethod" + ): + # It is a task defined within a class (can not parse the code + # with ast since the class does not exist yet). + # Alternatively, the only way I see is to parse it manually + # line by line. + ret_mask = [] + code = source_code.split("\n") + for line in code: + if "return " in line: + ret_mask.append(True) + else: + ret_mask.append(False) + else: + code = list(ast.walk(ast.parse(source_code))) + ret_mask = [isinstance(node, ast.Return) for node in code] + + if any(ret_mask): + has_multireturn = False + lines = [i for i, li in enumerate(ret_mask) if li] + max_num_returns = 0 + if self.first_arg_name == "self" or source_code.startswith( + "@classmethod" + ): + # Parse code as string (it is a task defined within a class) + def _has_multireturn(statement: str) -> bool: + parsed_ret = ast.parse( + statement.strip() + ) # type: typing.Any + try: + if len(parsed_ret.body[0].value.elts) > 1: + return True + return False + except (KeyError, AttributeError): + # KeyError: "elts" means that it is a multiple return. + # "Ask forgiveness not permission" + return False + + def _get_return_elements(statement: str) -> int: + parsed_ret = ast.parse( + statement.strip() + ) # type: typing.Any + return len(parsed_ret.body[0].value.elts) + + for i in lines: + if _has_multireturn(code[i]): + has_multireturn = True + num_returns = _get_return_elements(code[i]) + if num_returns > max_num_returns: + max_num_returns = num_returns + else: + # Parse code AST (it is not a task defined within a class) + for i in lines: + try: + if "elts" in code[i].value.__dict__: # noqa + has_multireturn = True + num_returns = len( + code[i].value.__dict__["elts"] + ) # noqa + if num_returns > max_num_returns: + max_num_returns = num_returns + except (KeyError, AttributeError): + # KeyError: "elts" means that it is a multiple return. + # "Ask forgiveness not permission" + pass + if has_multireturn: + for i in range(max_num_returns): + param = Parameter( + content_type=parameter.TYPE.FILE, + direction=parameter.DIRECTION.OUT, + ) + param.content = object() + self.returns[get_return_name(i)] = param + else: + param = Parameter( + content_type=parameter.TYPE.FILE, + direction=parameter.DIRECTION.OUT, + ) + param.content = object() + self.returns[get_return_name(0)] = param + else: + # Return not found + pass + return len(self.returns) + + def _build_return_objects(self, num_returns: int) -> typing.Any: + """Build the return objects. + + Build the return object from the self.returns dictionary and include + their filename in self.returns. + Normally they are future objects, unless the user has defined a user + defined class where an empty instance (needs an empty constructor) + will be returned. This case will enable users to call tasks within + user defined classes from future objects. + + WARNING: Updates self.returns dictionary. + + :param num_returns: Number of returned elements. + :return: Future object/s. + """ + future_object = None # type: typing.Any + if num_returns == 0: + # No return - Return always None (as Python does) + return future_object + if num_returns == 1: + # Simple return + if __debug__: + logger.debug("Simple object return found.") + # Build the appropriate future object + ret_value = self.returns[get_return_name(0)].content + + if self._is_return_param_name(ret_value): + # for the cases like 'returns = {{param_name}}' we replace the + # return value with the parameter itself + tmp = ret_value[ + len(RETURN_OPEN_SYMBOL) : -len( # noqa: E203 + RETURN_CLOSE_SYMBOL + ) + ] + if not self.parameters.get(tmp): + raise PyCOMPSsException( + "Invalid parameter name in 'returns'" + ) + future_object = self.parameters[tmp].content + elif ( + type(ret_value) in _PYTHON_TO_COMPSS + or ret_value in _PYTHON_TO_COMPSS + ): + future_object = Future() # primitives,string,dic,list,tuple + elif inspect.isclass(ret_value): + # For objects: + # type of future has to be specified to allow o = func; o.func + try: + future_object = ret_value() + except TypeError: + logger.warning( + "Type %s does not have an empty constructor, " + "building generic future object", + str(ret_value), + ) + future_object = Future() + else: + future_object = Future() # modules, functions, methods + + _, ret_filename = OT.track(future_object) + # when the return value is an IN param, after tracking on OT, + # we should also serialize to a file. Otherwise, already-tracked + # IN param will not be serialized when "processing params". + if self._is_return_param_name(ret_value): + serialize_to_file(future_object, ret_filename, logger) + OT.set_pending_to_synchronize(_) + + single_return = self.returns[get_return_name(0)] + single_return.content_type = TYPE.FILE + single_return.extra_content_type = "FILE" + single_return.prefix = "#" + single_return.file_name = COMPSsFile(ret_filename) + else: + # Multiple returns (the future object is a list of future objects) + future_object = [] + if __debug__: + logger.debug("Multiple objects return found.") + for _, ret_v in self.returns.items(): + # Build the appropriate future object + if ret_v.content in _PYTHON_TO_COMPSS: + # Primitives, string, dic, list, tuple + future_object_element = Future() + elif inspect.isclass(ret_v.content): + # For objects: type of future has to be specified to allow: + # o = func; o.func + try: + future_object_element = ret_v.content() + except TypeError: + logger.warning( + "Type %s does not have an empty constructor, " + "building generic future object", + str(type(ret_v.content())), + ) + future_object_element = Future() + else: + # Modules, functions, methods + future_object_element = Future() + future_object.append(future_object_element) + _, ret_filename = OT.track(future_object_element) + # Once determined the filename where the returns are going to + # be stored, create a new Parameter object for each return + # object + return_k = ret_v + return_k.content_type = TYPE.FILE + return_k.extra_content_type = "FILE" + return_k.prefix = "#" + return_k.file_name = COMPSsFile(ret_filename) + return future_object + + def _serialize_objects(self) -> None: + """Infer COMPSs types for the task parameters and serialize them. + + :return: None. + """ + # # Old school: + # for k in self.parameters: + # self._serialize_object(k) + # Allow concurrent serialization if python 3 and env. var: + if "COMPSS_THREADED_SERIALIZATION" in os.environ: + # Concurrent: + with ThreadPoolExecutor() as executor: + futures = [] + for k in self.parameters: + futures.append(executor.submit(self._serialize_object, k)) + wait(futures) + # Threaded: (somehow takes more time than sequential?) + # threads = [] + # # Serialize each object in a different thread (non blocking IO) + # for k in self.parameters: + # io_thread = threading.Thread(target=self._serialize_object, + # args=(k,)) + # threads.append(io_thread) + # io_thread.start() + # # Wait for all threads to finish + # for thread in threads: + # thread.join() + else: + # Sequential: + for k in self.parameters: + self._serialize_object(k) + + def _serialize_object(self, name: str) -> None: + """Infer COMPSs types for a single task parameter and serializes it. + + WARNING: Updates self.parameters dictionary. + + :param name: Name of the element in self.parameters + :return: None. + """ + # 320k is usually the maximum size of all objects + max_obj_arg_size = 320000 / 32 + with EventMaster(TRACING_MASTER.serialize_object): + # Check user annotations concerning this argument + param = self.parameters[name] + if TASK_FEATURES.get_object_conversion(): + # Convert small objects to string if enabled + # Check if the object is small in order not to serialize it. + param = self._convert_parameter_obj_to_string( + name, param, max_obj_arg_size + ) + else: + # Serialize objects into files + param = _serialize_object_into_file(name, param) + # Update k parameter's Parameter object + self.parameters[name] = param + + if __debug__: + logger.debug( + "Final type for parameter %s: %d", name, param.content_type + ) + + @staticmethod + def _reset_forced_serializer() -> None: + """Reset serializer forcing. + + May be forced by the @http decorator. + + :return: None + """ + serializer.FORCED_SERIALIZER = -1 + + def _build_values_types_directions( + self, + ) -> typing.Tuple[ + typing.List[typing.Union[COMPSsFile, str]], + typing.List[str], + typing.List[int], + typing.List[int], + typing.List[int], + typing.List[str], + typing.List[str], + typing.List[str], + typing.List[bool], + ]: + """Build the values, the values types and the values directions lists. + + Uses: + - self.decorated_function.function_type: task function type. + If it is an instance + method, the first + parameter will be put at + the end. + - self.parameters: Function parameters. + - self.returns: - Function returns. + - self.decorated_function.function.__code_strings__: + Code strings + (or not). + :return: List of values, their types, their directions, their streams + and their prefixes. + """ + values = [] # type: typing.List[typing.Union[COMPSsFile, str]] + names = [] + arg_names = list(self.parameters.keys()) + result_names = list(self.returns.keys()) + compss_types = [] + compss_directions = [] + compss_streams = [] + compss_prefixes = [] + extra_content_types = [] + slf_name = "" + weights = [] + keep_renames = [] + code_strings = self.decorated_function.code_strings + + # Build the range of elements + if self.decorated_function.function_type in ( + FunctionType.INSTANCE_METHOD, + FunctionType.CLASS_METHOD, + ): + slf_name = arg_names.pop(0) + # Put the CONSTANTS.compss_interactive_source_file in first position + # if exists + if CONSTANTS.compss_interactive_source_file in arg_names: + arg_names.remove(CONSTANTS.compss_interactive_source_file) + arg_names.insert(0, CONSTANTS.compss_interactive_source_file) + # Fill the values, compss_types, compss_directions, compss_streams and + # compss_prefixes from function parameters + for name in arg_names: + ( + value, + typ, + direction, + stream, + prefix, + con_type, + weight, + keep_rename, + ) = _extract_parameter(self.parameters[name], code_strings) + if isinstance(value, COMPSsFile): + values.append(value.original_path) + else: + values.append(value) + compss_types.append(typ) + compss_directions.append(direction) + compss_streams.append(stream) + compss_prefixes.append(prefix) + names.append(name) + extra_content_types.append(con_type) + weights.append(weight) + keep_renames.append(keep_rename) + # Fill the values, compss_types, compss_directions, compss_streams and + # compss_prefixes from self (if exist) + if ( + self.decorated_function.function_type + == FunctionType.INSTANCE_METHOD + ): + # self is always an object + ( + value, + typ, + direction, + stream, + prefix, + con_type, + weight, + keep_rename, + ) = _extract_parameter(self.parameters[slf_name], code_strings) + if isinstance(value, COMPSsFile): + values.append(value.original_path) + else: + values.append(value) + compss_types.append(typ) + compss_directions.append(direction) + compss_streams.append(stream) + compss_prefixes.append(prefix) + names.append(slf_name) + extra_content_types.append(con_type) + weights.append(weight) + keep_renames.append(keep_rename) + + # Fill the values, compss_types, compss_directions, compss_streams and + # compss_prefixes from function returns + for return_key in self.returns: + return_param = self.returns[return_key] + if isinstance(return_param.file_name, COMPSsFile): + values.append(return_param.file_name.original_path) + else: + values.append(return_param.file_name) + compss_types.append(return_param.content_type) + compss_directions.append(return_param.direction) + compss_streams.append(return_param.stream) + compss_prefixes.append(return_param.prefix) + names.append(result_names.pop(0)) + extra_content_types.append(return_param.extra_content_type) + weights.append(return_param.weight) + keep_renames.append(return_param.keep_rename) + + return ( + values, + names, + compss_types, + compss_directions, + compss_streams, + compss_prefixes, + extra_content_types, + weights, + keep_renames, + ) + + @staticmethod + def _convert_parameter_obj_to_string( + name: str, param: Parameter, max_obj_arg_size: float + ) -> Parameter: + """Convert object to string. + + Convert small objects into strings that can fit into the task + parameters call. + + :param name: Parameter name. + :param param: Parameter. + :param max_obj_arg_size: Max size of the object to be converted. + :return: The object possibly converted to string. + """ + is_future = param.is_future + base_string = str + num_bytes = 0 + # Check if the object is small in order to serialize it. + # Evaluates the size of the object before serializing the object. + # Warning: calculate the size of a python object can be difficult + # in terms of time and precision. + if ( + param.content_type == TYPE.OBJECT + and not is_future + and param.direction == DIRECTION.IN + and not isinstance(param.content, base_string) + ): + # check object size - The following line does not work + # properly with recursive objects + # bytes = sys.getsizeof(param.content) + num_bytes = total_sizeof(param.content) + if __debug__: + megabytes = num_bytes / 1000000 # truncate + logger.debug( + "Object size %d bytes (%d Mb).", num_bytes, megabytes + ) + if num_bytes < max_obj_arg_size: + # be careful... more than this value produces: + # Cannot run program "/bin/bash"...: error=7, \ + # The arguments list is too long + if __debug__: + logger.debug("The object size is less than 320 kb.") + try: + value = pickle.dumps(param.content) + if TASK_FEATURES.get_prepend_strings(): + # new_content = value.decode(CONSTANTS.str_escape) + param.content = f"#HiddenObj#{value!r}" + param.content_type = TYPE.STRING_64 + param.extra_content_type = str(type("str")) + if __debug__: + logger.debug( + "Inferred type modified " + "(Object converted to String)." + ) + except SerializerException as serializer_exception: + raise PyCOMPSsException( + "The object cannot be converted due to: " + "not serializable." + ) from serializer_exception + else: + if __debug__: + logger.warning( + "Could not serialize object to string conversion" + ) + param = _serialize_object_into_file(name, param) + + return param + + +def _get_object_property(param_ref: list, obj: typing.Any) -> typing.Any: + """Get the object property from the given parameter references. + + :param param_ref: List of parameter references. + :param obj: Target object. + :returns: The object property from the given parameters. + """ + if len(param_ref) == 1: + return obj + return _get_object_property(param_ref[1:], getattr(obj, param_ref[1])) + + +def _manage_persistent_object(param: Parameter) -> None: + """Manage a persistent object within a Parameter. + + Does the necessary actions over a persistent object used as task parameter. + In particular, saves the object id provided by the persistent storage + (getID()) into the pending_to_synchronize dictionary. + + :param param: Parameter. + :return: None. + """ + param.content_type = TYPE.EXTERNAL_PSCO + obj_id = str(get_id(param.content)) + OT.set_pending_to_synchronize(obj_id) + param.content = obj_id + if __debug__: + logger.debug("Managed persistent object: %s", obj_id) + + +def _serialize_object_into_file( + name: str, param: Parameter, code_strings=True, force_file=False +) -> Parameter: + """Serialize an object into a file if necessary. + + :param name: Name of the object. + :param param: Parameter. + :param code_strings: If strings will be encoded. + :param force_file: If the default value is file (collections of files). + :return: Parameter (whose type and value might be modified). + """ + # ######################################################################### + # ## THIS IS TEMPORAL UNTIL THE EXTERNAL PSCO STREAM TYPE IS IMPLEMENTED ## + # ######################################################################### + is_a_psco = is_psco(param.content) + if is_a_psco and param.content_type == TYPE.EXTERNAL_STREAM: + # If is a persisted object annotated as STREAM, create a wrapper + # and manage it as a normal STREAM object: + # - Negative: requires a serialization of the wrapper. + # - Positive: reuses the stream implementation and saves a lot of + # implementation. + psco_id = get_id(param.content) + wrapped_psco_id = PscoStreamWrapper(psco_id) + param.content = wrapped_psco_id + # ######################################################################### + + if ( + param.content_type == TYPE.OBJECT + or param.content_type == TYPE.EXTERNAL_STREAM + or param.is_future + ): + # 2nd condition: real type can be primitive, but now it's acting as a + # future (object) + try: + val_type = type(param.content) + if isinstance(val_type, list) and any( + isinstance(value, Future) for value in param.content + ): + # Is there a future object within the list? + if __debug__: + logger.debug( + "Found a list that contains future objects " + "- synchronizing..." + ) + mode = get_compss_direction("in") + param.content = list( + map(wait_on, param.content, [mode] * len(param.content)) + ) + _skip_file_creation = ( + param.direction == DIRECTION.OUT + and param.content_type != TYPE.EXTERNAL_STREAM + ) + if __debug__ and _skip_file_creation: + logger.debug( + "Skipping object (%s) serialization " + "(it is OUT and not EXTERNAL_STREAM)." % name + ) + _turn_into_file(param, name, skip_creation=_skip_file_creation) + except SerializerException: + import traceback + + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception( + exc_type, exc_value, exc_traceback + ) + logger.exception( + "Pickling error exception: " + "non-serializable object found as a parameter." + ) + logger.exception("".join(line for line in lines)) + print( + "[ ERROR ]: Non serializable objects can not " + "be used as parameters (e.g. methods)." + ) + print(f"[ ERROR ]: Object: {param.content}") + # Raise the exception up tu launch.py in order to point where the + # error is in the user code. + raise + elif param.content_type == TYPE.EXTERNAL_PSCO: + _manage_persistent_object(param) + elif param.content_type == TYPE.INT: + if param.content > JAVA_MAX_INT or param.content < JAVA_MIN_INT: + # This must go through Java as a long to prevent overflow with + # Java integer + param.content_type = TYPE.LONG + elif param.content_type == TYPE.LONG: + if param.content > JAVA_MAX_LONG or param.content < JAVA_MIN_LONG: + # This must be serialized to prevent overflow with Java long + param.content_type = TYPE.OBJECT + _skip_file_creation = param.direction == DIRECTION.OUT + _turn_into_file(param, name, _skip_file_creation) + elif param.content_type in (TYPE.STRING, TYPE.STRING_64): + # Do not move this import to the top + if TASK_FEATURES.get_prepend_strings(): + # Strings can be empty. If a string is empty their base64 encoding + # will be empty. + # So we add a leading character to it to make it non empty + param.content = f"#{param.content}" + elif param.content_type == TYPE.COLLECTION: + # Just make contents available as serialized files (or objects) + # We will build the value field later + # (which will be used to reconstruct the collection in the worker) + if param.is_file_collection: + new_object_col = [ + _serialize_object_into_file( + name, + Parameter( + content=x, + content_type=get_compss_type( + x, param.depth - 1, force_file=True + ), + is_file_collection=True, + direction=param.direction, + file_name=COMPSsFile(x), + depth=param.depth - 1, + ), + force_file=True, + ) + for x in param.content + ] + else: + if force_file: + # Leaf of a file collection will enter here. + new_object_col = [ + _serialize_object_into_file( + name, + Parameter( + content=x, + content_type=get_compss_type( + x, + param.depth - 1, + code_strings=code_strings, + force_file=force_file, + ), + direction=param.direction, + file_name=COMPSsFile(x), + depth=param.depth - 1, + ), + ) + for x in param.content + ] + else: + if param.content is None: + # to avoid None is not iterable error, assign an empty list + new_object_col = [] + else: + new_object_col = [ + _serialize_object_into_file( + name, + Parameter( + content=x, + content_type=get_compss_type( + x, + param.depth - 1, + code_strings=code_strings, + ), + direction=param.direction, + depth=param.depth - 1, + extra_content_type=str(type(x).__name__), + ), + ) + for x in param.content + ] + param.content = new_object_col + # Give this object an identifier inside the binding + if param.direction != DIRECTION.IN_DELETE: + _, _ = OT.track(param.content, obj_name=name, collection=True) + elif param.content_type == TYPE.DICT_COLLECTION: + # Just make contents available as serialized files (or objects) + # We will build the value field later + # (which will be used to reconstruct the collection in the worker) + new_object_dict = {} + for p_key, p_value in param.content.items(): + key = _serialize_object_into_file( + name, + Parameter( + content=p_key, + content_type=get_compss_type( + p_key, param.depth - 1, code_strings=code_strings + ), + direction=param.direction, + depth=param.depth - 1, + extra_content_type=str(type(param).__name__), + ), + ) + value = _serialize_object_into_file( + name, + Parameter( + content=p_value, + content_type=get_compss_type( + p_value, param.depth - 1, code_strings=code_strings + ), + direction=param.direction, + depth=param.depth - 1, + extra_content_type=str(type(p_value).__name__), + ), + ) + new_object_dict[key] = value + param.content = new_object_dict + # Give this object an identifier inside the binding + if param.direction != DIRECTION.IN_DELETE: + _, _ = OT.track(param.content, obj_name=name, collection=True) + return param + + +def _turn_into_file( + param: Parameter, name: str, skip_creation: bool = False +) -> None: + """Write a object into a file if the object has not been already written. + + Consults the obj_id_to_filename to check if it has already been written + (reuses it if exists). If not, the object is serialized to file and + registered in the obj_id_to_filename dictionary. + This functions stores the object into pending_to_synchronize. + + :param p: Wrapper of the object to turn into file. + :param name: Name of the object. + :param skip_creation: Skips the serialization to file. + :return: None. + """ + obj_id = OT.is_tracked(param.content) + if obj_id == "": + # This is the first time a task accesses this object + if param.direction == DIRECTION.IN_DELETE: + obj_id, file_name = OT.not_track() + else: + obj_id, file_name = OT.track(param.content, obj_name=name) + if not skip_creation: + serialize_to_file(param.content, file_name, logger) + else: + file_name = OT.get_file_name(obj_id) + if OT.has_been_written(obj_id): + if param.direction in (DIRECTION.INOUT, DIRECTION.COMMUTATIVE): + OT.set_pending_to_synchronize(obj_id) + # Main program generated the last version + compss_file = OT.pop_written_obj(obj_id) + if __debug__: + logger.debug( + "Serializing object %s to file %s", obj_id, compss_file + ) + if not skip_creation: + serialize_to_file(param.content, compss_file, logger) + + # Set file name in Parameter object + param.file_name = COMPSsFile(file_name) + + +def _extract_parameter( + param: Parameter, code_strings: bool, collection_depth: int = 0 +) -> typing.Tuple[typing.Any, int, int, int, str, str, str, bool]: + """Extract the information of a single parameter. + + :param param: Parameter object. + :param code_strings: If the strings have to be encoded. + :param collection_depth: Collection depth. + :return: value, typ, direction, stream, prefix, extra_content_type, weight, + keep_rename of the given parameter. + """ + con_type = UNDEFINED_CONTENT_TYPE + + if param.content_type == TYPE.STRING_64 and not param.is_future: + # Encode the string in order to preserve the source + # Checks that it is not a future (which is indicated with a path) + # Considers multiple spaces between words + param.content = b64encode(param.content.encode()).decode() + if len(param.content) == 0: + # Empty string - use escape string to avoid padding + # Checked and substituted by empty string in the worker.py and + # piper_worker.py + param.content = b64encode( + CONSTANTS.empty_string_key.encode() + ).decode() # noqa: E501 + con_type = CONSTANTS.extra_content_type_format.format( + "builtins", str(param.content.__class__.__name__) + ) + elif ( + param.content_type == TYPE.STRING and not param.is_future + ): # noqa: E501 + if len(param.content) == 0: + param.content = CONSTANTS.empty_string_key + con_type = CONSTANTS.extra_content_type_format.format( + "builtins", str(param.content.__class__.__name__) + ) + + typ = -1 # type: int + value = None # type: typing.Any + if param.content_type == TYPE.FILE or param.is_future: + # If the parameter is a file or is future, the content is in a file + # and we register it as file + value = param.file_name + con_type = str(Future.__name__) if param.is_future else "FILE" + value_str = str(value) + if isinstance(value, str): + value_str = value + if isinstance(value, COMPSsFile): + value_str = value.original_path + if value_str != "None": + typ = TYPE.FILE + else: + typ = TYPE.NULL + elif param.content_type == TYPE.DIRECTORY: + value = param.file_name + value_str = str(value) + if isinstance(value, str): + value_str = value + if isinstance(value, COMPSsFile): + value_str = value.original_path + if value_str != "None": + typ = TYPE.DIRECTORY + else: + typ = TYPE.NULL + elif param.content_type == TYPE.OBJECT: + # If the parameter is an object, its value is stored in a file and + # we register it as file + value = param.file_name + typ = TYPE.FILE + try: + _mf = sys.modules[param.content.__class__.__module__].__file__ + except AttributeError: + # "builtin" modules do not have __file__ attribute! + _mf = "builtins" + _class_name = str(param.content.__class__.__name__) + con_type = CONSTANTS.extra_content_type_format.format(_mf, _class_name) + elif param.content_type == TYPE.EXTERNAL_STREAM: + # If the parameter type is stream, its value is stored in a file, but + # we keep the type + value = param.file_name + typ = TYPE.EXTERNAL_STREAM + elif param.content_type == TYPE.COLLECTION or ( + collection_depth > 0 and is_basic_iterable(param.content) + ): + # An object will be considered a collection if at least one of the + # following is true: + # 1) We said it is a collection in the task decorator + # 2) It is part of some collections object, it is iterable, and we + # are inside the specified depth radius + # + # The content of a collection is sent via JNI to the master, and the + # format is: + # collectionId numberOfElements collectionPyContentType + # type1 Id1 pyType1 + # type2 Id2 pyType2 + # ... + # typeN IdN pyTypeN + _class_name = str(param.content.__class__.__name__) + con_type = CONSTANTS.extra_content_type_format.format( + "collection", _class_name + ) + value = ( + f"{OT.is_tracked(param.content)} {len(param.content)} {con_type}" + ) + OT.stop_tracking(param.content, collection=True) + typ = TYPE.COLLECTION + for _, x_param in enumerate(param.content): + x_value, x_type, _, _, _, x_con_type, _, _ = _extract_parameter( + x_param, code_strings, param.depth - 1 + ) + if isinstance(x_value, COMPSsFile): + value += f" {x_type} {x_value.original_path} {x_con_type}" + else: + value += f" {x_type} {x_value} {x_con_type}" + elif param.content_type == TYPE.DICT_COLLECTION or ( + collection_depth > 0 and is_dict(param.content) + ): + # An object will be considered a dictionary collection if at least one + # of the following is true: + # 1) We said it is a dictionary collection in the task decorator + # 2) It is part of some collection object, it is dict and we + # are inside the specified depth radius + # + # The content of a dictionary collection is sent via JNI to the master, + # and the format is: + # dictCollectionId numberOfEntries dictCollectionPyContentType + # type1(key) Id1(key) pyType1(key) + # type1(value) Id1(value) pyType1(value) + # type2(key) Id2(key) pyType2(key) + # type2(value) Id2(value) pyType2(value) + # ... + # typeN(value) IdN(value) pyTypeN(value) + _class_name = str(param.content.__class__.__name__) + con_type = CONSTANTS.extra_content_type_format.format( + "dict_collection", _class_name + ) + value = ( + f"{OT.is_tracked(param.content)} {len(param.content)} {con_type}" + ) + OT.stop_tracking(param.content, collection=True) + typ = TYPE.DICT_COLLECTION + for k_param, v_param in param.content.items(): # noqa + k_value, k_type, _, _, _, k_con_type, _, _ = _extract_parameter( + k_param, code_strings, param.depth - 1 + ) + real_k_type = k_type + if isinstance(k_type, COMPSsFile): + real_k_type = k_type.original_path + real_k_value = k_value + if isinstance(k_value, COMPSsFile): + real_k_value = k_value.original_path + if k_con_type != con_type: + value = f"{value} {real_k_type} {real_k_value} {k_con_type}" + else: + # remove last dict_collection._classname if key is + # a dict_collection + value = f"{value} {real_k_type} {real_k_value}" + v_value, v_type, _, _, _, v_con_type, _, _ = _extract_parameter( + v_param, code_strings, param.depth - 1 + ) + real_v_type = v_type + if isinstance(v_type, COMPSsFile): + real_v_type = v_type.original_path + real_v_value = v_value + if isinstance(v_value, COMPSsFile): + real_v_value = v_value.original_path + if v_con_type != con_type: + value = f"{value} {real_v_type} {real_v_value} {v_con_type}" + else: + # remove last dict_collection._classname if value is + # a dict_collection + value = f"{value} {real_v_type} {real_v_value}" + else: + # Keep the original value and type + value = param.content + typ = param.content_type + + # Get direction, stream and prefix + direction = param.direction + # Get stream and prefix + stream = param.stream + prefix = param.prefix + # Get weights and keep rename + weight = param.weight + keep_rename = param.keep_rename + + return value, typ, direction, stream, prefix, con_type, weight, keep_rename diff --git a/examples/dds/pycompss/runtime/task/parameter.py b/examples/dds/pycompss/runtime/task/parameter.py new file mode 100644 index 00000000..6f754764 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/parameter.py @@ -0,0 +1,702 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Parameter. + +This file contains the classes needed for the task parameter definition. +""" + +import sys + +from pycompss.api.parameter import Cache +from pycompss.api.parameter import DIRECTION +from pycompss.api.parameter import Depth +from pycompss.api.parameter import Direction +from pycompss.api.parameter import IOSTREAM +from pycompss.api.parameter import Keep_rename +from pycompss.api.parameter import PREFIX +from pycompss.api.parameter import Prefix +from pycompss.api.parameter import StdIOStream +from pycompss.api.parameter import TYPE +from pycompss.api.parameter import Type +from pycompss.api.parameter import Weight +from pycompss.api.parameter import _Param as Param # noqa +from pycompss.runtime.task.keys import PARAM_ALIAS_KEYS +from pycompss.runtime.task.keys import PARAM_DICT_KEYS +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.objects.properties import is_basic_iterable +from pycompss.util.objects.properties import is_dict +from pycompss.util.storages.persistent import get_id +from pycompss.util.storages.persistent import has_id +from pycompss.util.typing_helper import typing + +# Try to import numpy +NP = None # type: typing.Union[None, typing.Any] +try: + import numpy + + NP = numpy +except ImportError: + pass + +# Python max and min integer values +PYTHON_MAX_INT = sys.maxsize +PYTHON_MIN_INT = -sys.maxsize - 1 +# Java max and min integer and long values +JAVA_MAX_INT = 2147483647 +JAVA_MIN_INT = -2147483648 +JAVA_MAX_LONG = PYTHON_MAX_INT +JAVA_MIN_LONG = PYTHON_MIN_INT + +# Content type format is :, separated by colon (":") +UNDEFINED_CONTENT_TYPE = "#UNDEFINED#:#UNDEFINED#" + + +class COMPSsFile: + """Class that represents a file in the worker.""" + + __slots__ = [ + "source_path", + "destination_name", + "keep_source", + "is_write_final", + "original_path", + ] + + def __init__(self, file_name: typing.Any = "None") -> None: + """Compss File constructor. + + :param file_name: File name. + """ + self.source_path = "None" # type: str + self.destination_name = "None" # type: str + self.keep_source = False # type: bool + self.is_write_final = False # type: bool + self.original_path = file_name # type: typing.Any + if ( + file_name is not None + and isinstance(file_name, str) + and ":" in file_name + ): + fields = file_name.split(":") + self.source_path = fields[0] + self.destination_name = fields[1] + self.keep_source = fields[2] == "true" + self.is_write_final = fields[3] == "true" + self.original_path = fields[4] + else: + # Can be a collection wrapper or a stream + pass + + def __repr__(self) -> str: + """Compss File object representation. + + :returns: The COMPSsFile representation as string. + """ + return ( + f"Source: {self.source_path}, " + f"Destination: {self.destination_name}, " + f"Keep source: {self.keep_source}, " + f"Is write final: {self.is_write_final}, " + f"Original path: {self.original_path}" + ) + + +class Parameter: + """Class that represents a file in the worker. + + Internal Parameter class + Used to group all parameter variables. + """ + + __slots__ = [ + "name", + "content", + "content_type", + "direction", + "stream", + "prefix", + "file_name", + "is_future", + "is_file_collection", + "collection_content", + "dict_collection_content", + "depth", + "extra_content_type", + "weight", + "keep_rename", + "cache", + ] + + def __init__( + self, + name: str = "None", + content: typing.Any = "", + content_type: int = -1, + direction: int = DIRECTION.IN, + stream: int = IOSTREAM.UNSPECIFIED, + prefix: str = PREFIX.PREFIX, + file_name: COMPSsFile = COMPSsFile(), + is_future: bool = False, + is_file_collection: bool = False, + collection_content: typing.Any = "", + dict_collection_content: typing.Optional[dict] = None, + depth: int = PYTHON_MAX_INT, + extra_content_type: str = UNDEFINED_CONTENT_TYPE, + weight: str = "1.0", + keep_rename: bool = True, + cache: bool = True, + ) -> None: + """Parameter constructor. + + :param name: Parameter name. + :param content: Object represented by the parameter. + :param content_type: Object type. + :param direction: Parameter direction. + :param stream: Parameter stream type. + :param prefix: Parameter prefix. + :param file_name: Parameter associated file name. + :param is_future: If is future or not. + :param is_file_collection: If is a collection of files or not. + :param collection_content: Collection contents (list of Parameters). + :param dict_collection_content: Dictionary collection contents + (dictionary of parameters). + :param depth: Collection or dictionary collection depth. + :param extra_content_type: Extra content types. + :param weight: Parameter weight. + :param keep_rename: Keep rename. + :param cache: If cache the parameter. + """ + if dict_collection_content is None: + dict_collection_content = {} + self.name = name + self.content = content # placeholder for parameter content + self.content_type = content_type + self.direction = direction + self.stream = stream + self.prefix = prefix + self.file_name = file_name # placeholder for object's serialized file + self.is_future = is_future + self.is_file_collection = is_file_collection + self.collection_content = collection_content + self.dict_collection_content = dict_collection_content + self.depth = depth # Recursive depth for collections + self.extra_content_type = extra_content_type + self.weight = weight + self.keep_rename = keep_rename + self.cache = cache + + def __repr__(self) -> str: + """Parameter object representation. + + :returns: The parameter representation as string. + """ + return ( + f"Parameter(name={str(self.name)}\n" + f" type={str(self.content_type)}, " + f"direction={str(self.direction)}, " + f"stream={str(self.stream)}, " + f"prefix={str(self.prefix)}\n" + f" extra_content_type={str(self.extra_content_type)}\n" + f" file_name={str(self.file_name)}\n" + f" is_future={str(self.is_future)}\n" + f" is_file_collection={str(self.is_file_collection)}, " + f"depth={str(self.depth)}\n" + f" weight={str(self.weight)}\n" + f" keep_rename={str(self.keep_rename)}\n" + f" cache={str(self.cache)}\n" + f" content={str(self.content)})" + ) + + def is_object(self) -> bool: + """Determine if parameter is an object (not a FILE). + + :return: True if param represents an object (IN, INOUT, OUT). + """ + return self.content_type == -1 + + def is_file(self) -> bool: + """Determine if parameter is a FILE. + + :return: True if param represents an FILE (IN, INOUT, OUT). + """ + return self.content_type is TYPE.FILE + + def is_directory(self) -> bool: + """Determine if parameter is a DIRECTORY. + + :return: True if param represents an DIRECTORY. + """ + return self.content_type is TYPE.DIRECTORY + + def is_collection(self) -> bool: + """Determine if parameter is a COLLECTION. + + :return: True if param represents an COLLECTION. + """ + return self.content_type is TYPE.COLLECTION + + def is_dict_collection(self) -> bool: + """Determine if parameter is a DICT_COLLECTION. + + :return: True if param represents an DICT_COLLECTION. + """ + return self.content_type is TYPE.DICT_COLLECTION + + +# Parameter conversion dictionary. +_param_conversion_dict_ = { + PARAM_ALIAS_KEYS.IN: {}, + PARAM_ALIAS_KEYS.OUT: { + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + }, + PARAM_ALIAS_KEYS.INOUT: { + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + }, + PARAM_ALIAS_KEYS.CONCURRENT: { + PARAM_DICT_KEYS.DIRECTION: DIRECTION.CONCURRENT, + }, + PARAM_ALIAS_KEYS.COMMUTATIVE: { + PARAM_DICT_KEYS.DIRECTION: DIRECTION.COMMUTATIVE, + }, + PARAM_ALIAS_KEYS.IN_DELETE: { + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN_DELETE, + }, + PARAM_ALIAS_KEYS.FILE: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_INOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.DIRECTORY: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DIRECTORY, + PARAM_DICT_KEYS.KEEP_RENAME: False, + }, + PARAM_ALIAS_KEYS.DIRECTORY_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DIRECTORY, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.DIRECTORY_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DIRECTORY, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.DIRECTORY_INOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DIRECTORY, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_CONCURRENT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.CONCURRENT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_COMMUTATIVE: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.COMMUTATIVE, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_IN_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_IN_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_IN_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_OUT_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_OUT_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_OUT_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_INOUT_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_INOUT_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_INOUT_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.CONCURRENT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.CONCURRENT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_CONCURRENT_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.CONCURRENT, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDIN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.COMMUTATIVE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDIN, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDERR: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.COMMUTATIVE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDERR, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.FILE_COMMUTATIVE_STDOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.FILE, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.COMMUTATIVE, + PARAM_DICT_KEYS.STDIOSTREAM: IOSTREAM.STDOUT, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.COLLECTION: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + }, + PARAM_ALIAS_KEYS.COLLECTION_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + }, + PARAM_ALIAS_KEYS.COLLECTION_INOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + }, + PARAM_ALIAS_KEYS.COLLECTION_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + }, + PARAM_ALIAS_KEYS.DICT_COLLECTION: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DICT_COLLECTION, + }, + PARAM_ALIAS_KEYS.DICT_COLLECTION_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DICT_COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + }, + PARAM_ALIAS_KEYS.DICT_COLLECTION_INOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DICT_COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + }, + PARAM_ALIAS_KEYS.DICT_COLLECTION_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DICT_COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + }, + PARAM_ALIAS_KEYS.COLLECTION_IN_DELETE: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN_DELETE, + }, + PARAM_ALIAS_KEYS.DICT_COLLECTION_IN_DELETE: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.DICT_COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN_DELETE, + }, + PARAM_ALIAS_KEYS.STREAM_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.EXTERNAL_STREAM, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.STREAM_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.EXTERNAL_STREAM, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.COLLECTION_FILE: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.IS_FILE_COLLECTION: True, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.COLLECTION_FILE_IN: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.IN, + PARAM_DICT_KEYS.IS_FILE_COLLECTION: True, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.COLLECTION_FILE_INOUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.INOUT, + PARAM_DICT_KEYS.IS_FILE_COLLECTION: True, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, + PARAM_ALIAS_KEYS.COLLECTION_FILE_OUT: { + PARAM_DICT_KEYS.CONTENT_TYPE: TYPE.COLLECTION, + PARAM_DICT_KEYS.DIRECTION: DIRECTION.OUT, + PARAM_DICT_KEYS.IS_FILE_COLLECTION: True, + PARAM_DICT_KEYS.KEEP_RENAME: False, + PARAM_DICT_KEYS.CACHE: False, + }, +} # type: dict + + +def is_param(obj: typing.Any) -> bool: + """Check if given object is a parameter. + + Avoids internal _param_ import. + + :param obj: Object to check. + :return: True if obj is instance of _Param. + """ + return isinstance(obj, Param) + + +def is_parameter(obj: typing.Any) -> bool: + """Check if given object is a parameter. + + Avoids internal Parameter import. + + :param obj: Object to check. + :return: True if obj is instance of Parameter. + """ + return isinstance(obj, Parameter) + + +def get_new_parameter(key: str) -> Parameter: + """Return a new parameter (no copies!). + + :param key: A string that is a key of a valid Parameter template. + :return: A new Parameter from the given key. + """ + return Parameter(**_param_conversion_dict_[key]) + + +def is_dict_specifier(value: typing.Any) -> bool: + """Check if value is a supported dictionary. + + Check if a parameter of the task decorator is a dictionary that specifies + at least Type (and therefore can include things like Prefix, see binary + decorator test for some examples). + + :param value: Decorator value to check. + :return: True if value is a dictionary that specifies at least the Type of + the key. + """ + return isinstance(value, dict) and Type in value + + +def get_parameter_from_dictionary(dictionary: dict) -> Parameter: + """Convert a dictionary to Parameter. + + Given a dictionary with fields like Type, Direction, etc. + returns an actual Parameter object. + + :param dictionary: Parameter description as dictionary. + :return: an actual Parameter object. + """ + if not isinstance(dictionary, dict): + raise PyCOMPSsException("Unexpected type for parameter.") + if Type not in dictionary: # If no Type specified => IN + parameter = Parameter() + else: + parameter = get_new_parameter(dictionary[Type].key) + # Add other modifiers + if Direction in dictionary: + parameter.direction = dictionary[Direction] + if StdIOStream in dictionary: + parameter.stream = dictionary[StdIOStream] + if Prefix in dictionary: + parameter.prefix = dictionary[Prefix] + if Depth in dictionary: + parameter.depth = dictionary[Depth] + if Weight in dictionary: + parameter.weight = dictionary[Weight] + if Keep_rename in dictionary: + parameter.keep_rename = dictionary[Keep_rename] + if Cache in dictionary: + parameter.cache = dictionary[Cache] + return parameter + + +def get_direction_from_key(key: str) -> int: + """Return the direction (as integer) for the given key. + + :param key: A direction key string (e.g. keys defined in Parameter). + :return: The associated integer value for the given key. + """ + if key == PARAM_ALIAS_KEYS.IN: + return DIRECTION.IN + elif key == PARAM_ALIAS_KEYS.OUT: + return DIRECTION.OUT + elif key == PARAM_ALIAS_KEYS.INOUT: + return DIRECTION.INOUT + elif key == PARAM_ALIAS_KEYS.CONCURRENT: + return DIRECTION.CONCURRENT + elif key == PARAM_ALIAS_KEYS.COMMUTATIVE: + return DIRECTION.COMMUTATIVE + elif key == PARAM_ALIAS_KEYS.IN_DELETE: + return DIRECTION.IN_DELETE + else: + raise PyCOMPSsException(f"Wrong direction key: {key}") + + +def get_compss_type( + value: typing.Any, depth: int = 0, code_strings=True, force_file=False +) -> int: + """Retrieve the value type mapped to COMPSs types. + + :param value: Value to analyse. + :param depth: Collections depth. + :param code_strings: If strings will be encoded. + :param force_file: If the default value is file (collections of files). + :return: The Type of the value. + """ + # First check if it is a PSCO since a StorageNumpy can be detected + # as a numpy object. + if has_id(value): + # If has method getID maybe is a PSCO + try: + if get_id(value) not in [None, "None"]: + # the "getID" + id == criteria for persistent object + return TYPE.EXTERNAL_PSCO + return TYPE.OBJECT + except TypeError: + # A PSCO class has been used to check its type (when checking + # the return). Since we still don't know if it is going to be + # persistent inside, we assume that it is not. It will be checked + # later on the worker side when the task finishes. + return TYPE.OBJECT + + if force_file: + if depth > 0 and is_basic_iterable(value): + return TYPE.COLLECTION + elif depth > 0 and is_dict(value): + return TYPE.DICT_COLLECTION + else: + return TYPE.FILE + + # If it is a numpy scalar, we manage it as all objects to avoid to + # infer its type wrong. For instance isinstance(NP.float64 object, float) + # returns true + if NP and isinstance(value, NP.generic): + return TYPE.OBJECT + + if isinstance(value, (bool, str, int, float)): + value_type = type(value) + if value_type is bool: + return TYPE.BOOLEAN + if value_type is str: + # Char does not exist as char, only strings. + # Files will be detected as string, since it is a path. + # The difference among them is defined by the parameter + # decoration as FILE. + return TYPE.STRING if not code_strings else TYPE.STRING_64 + if value_type is int: + if int(value) < PYTHON_MAX_INT: # noqa + return TYPE.INT + return TYPE.LONG + if value_type is float: + return TYPE.DOUBLE + elif depth > 0 and is_basic_iterable(value): + return TYPE.COLLECTION + elif depth > 0 and is_dict(value): + return TYPE.DICT_COLLECTION + else: + # Default type + return TYPE.OBJECT + return -1 diff --git a/examples/dds/pycompss/runtime/task/shared_args.py b/examples/dds/pycompss/runtime/task/shared_args.py new file mode 100644 index 00000000..e4444cf1 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/shared_args.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Worker shared arguments. + +This file contains the global variables used in the Python binding. +""" + +from pycompss.util.typing_helper import typing + + +class SharedArguments: + """Class that contains shared arguments between the worker and master. + + It is used to know things that are known in the worker side from the + master side (e.g. synchronization). + + IMPORTANT: Required for nesting purposes. + """ + + __slots__ = ("worker_args",) + + def __init__(self) -> None: + """Instantiate Shared Arguments class.""" + # Worker arguments received on the task call + self.worker_args = tuple() # type: tuple + + def set_worker_args(self, worker_args: tuple) -> None: + """Worker arguments to save. + + :param worker_args: Worker arguments. + :return: None. + """ + self.worker_args = worker_args + + def get_worker_args(self) -> tuple: + """Retrieve the worker arguments. + + :return: Worker arguments. + """ + return self.worker_args + + def update_worker_argument_parameter_content( + self, name: typing.Optional[str], content: typing.Any + ) -> None: + """Update the Parameter's content for the given name. + + :param name: Parameter name. + :param content: New content. + :return: None. + """ + if name: + for param in self.worker_args: + if ( + not param.is_collection() + and not param.is_dict_collection() + and param.name == name + ): + param.content = content + param.is_future = False + return + + def delete_worker_args(self) -> None: + """Remove the worker args variable contents. + + :return: None. + """ + self.worker_args = tuple() + + +# ############################################################# # +# ##################### SHARED ARGUMENTS ###################### # +# ############################################################# # + +SHARED_ARGUMENTS = SharedArguments() diff --git a/examples/dds/pycompss/runtime/task/worker.py b/examples/dds/pycompss/runtime/task/worker.py new file mode 100644 index 00000000..54ccf759 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/worker.py @@ -0,0 +1,1961 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Worker. + +This file contains the task core functions when acting as worker. +""" + +import gc +import os +import re +import sys +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait +from shutil import copyfile + +from pycompss.api import parameter +from pycompss.util.context import CONTEXT +from pycompss.api.commons.constants import LABELS +from pycompss.api.exceptions import COMPSsException +from pycompss.runtime.commons import CONSTANTS +from pycompss.runtime.management.classes import Future +from pycompss.runtime.management.object_tracker import OT +from pycompss.runtime.task.arguments import get_name_from_kwarg +from pycompss.runtime.task.arguments import get_name_from_vararg +from pycompss.runtime.task.arguments import is_kwarg +from pycompss.runtime.task.arguments import is_return +from pycompss.runtime.task.arguments import is_vararg +from pycompss.runtime.task.commons import get_default_direction +from pycompss.runtime.task.definitions.arguments import TaskArguments +from pycompss.runtime.task.definitions.function import FunctionDefinition +from pycompss.runtime.task.parameter import Parameter +from pycompss.runtime.task.parameter import get_compss_type +from pycompss.runtime.task.parameter import get_new_parameter +from pycompss.runtime.task.parameter import get_direction_from_key +from pycompss.runtime.task.shared_args import SHARED_ARGUMENTS +from pycompss.runtime.task.wrappers.psco_stream import PscoStreamWrapper +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.logger.helpers import swap_logger_name +from pycompss.util.objects.properties import create_object_by_con_type +from pycompss.util.objects.util import group_iterable +from pycompss.util.serialization.serializer import deserialize_from_file +from pycompss.util.serialization.serializer import serialize_to_file +from pycompss.util.serialization.serializer import serialize_to_file_mpienv +from pycompss.util.std.redirects import not_std_redirector +from pycompss.util.std.redirects import std_redirector +from pycompss.util.storages.persistent import is_psco +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.worker.commons.worker import build_task_parameter + +# The cache is only available currently for piper_worker.py and python >= 3.8 +# If supported in the future by another worker, add a common interface +# with these two functions and import the appropriate. +from pycompss.worker.piper.cache.tracker import CACHE_TRACKER +from pycompss.worker.piper.cache.classes import TaskWorkerCache + +NP = None # type: typing.Any +try: + import numpy + + NP = numpy +except ImportError: + NP = None + +if __debug__: + import logging + + LOGGER = logging.getLogger(__name__) + + +class TaskWorker: + """Task class representation for the Worker. + + Process the task decorator and prepare call the user function. + """ + + __slots__ = [ + "user_function", + "decorator_arguments", + "param_args", + "param_varargs", + "on_failure", + "defaults", + "cache", + ] + + def __init__( + self, + decorator_arguments: TaskArguments, + decorated_function: FunctionDefinition, + ) -> None: + """Task at worker constructor. + + :param decorator_arguments: Decorator arguments. + :param decorated_function: Decorated function. + """ + # Initialize TaskCommons + self.decorator_arguments = decorator_arguments + self.user_function = decorated_function.function + self.param_args = [] # type: typing.List[typing.Any] + self.param_varargs = None # type: typing.Any + self.on_failure = "" + self.defaults = {} # type: dict + self.cache = TaskWorkerCache() + + def call( + self, *args: typing.Any, **kwargs: typing.Any + ) -> typing.Tuple[list, list, str, tuple]: + """Run the task as worker. + + This function deals with task calls in the worker's side + Note that the call to the user function is made by the worker, + not by the user code. + + :param args: Arguments. + :param kwargs: Keyword arguments. + :return: A function that calls the user function with the given + parameters and does the proper serializations and updates + the affected objects. + """ + global LOGGER + # Save the args in a global place + # (needed from synchronize when using nesting) + SHARED_ARGUMENTS.set_worker_args(args) + # Grab LOGGER from kwargs + # (shadows outer LOGGER since it is set by the worker) + LOGGER = kwargs["compss_logger"] # noqa + with swap_logger_name(LOGGER, __name__): + if __debug__: + LOGGER.debug("Starting @task decorator worker call") + + # Redirect stdout/stderr if necessary to show the prints/exceptions + # in the job out/err files + redirect_std = True + if kwargs["compss_log_files"]: + # Redirect all stdout and stderr during the user code execution + # to job out and err files. + job_out, job_err = kwargs["compss_log_files"] + else: + job_out, job_err = None, None + redirect_std = False + if __debug__: + LOGGER.debug("Redirecting stdout to: %s", str(job_out)) + LOGGER.debug("Redirecting stderr to: %s", str(job_err)) + with ( + std_redirector(job_out, job_err) + if redirect_std + else not_std_redirector() + ): + # Update the on_failure attribute + # (could be defined by @on_failure) + if LABELS.on_failure in kwargs: + self.on_failure = kwargs.pop(LABELS.on_failure, "RETRY") + self.decorator_arguments.on_failure = self.on_failure + else: + self.on_failure = self.decorator_arguments.on_failure + self.defaults = kwargs.pop(LABELS.defaults, {}) + self.decorator_arguments.defaults = self.defaults + + # Pop cache if available + cache_on = False + cache = kwargs.pop("compss_cache", None) + if cache: + ( + self.cache.in_queue, + self.cache.out_queue, + self.cache.ids, + self.cache.profiler, + ) = cache + cache_on = True + + if __debug__: + LOGGER.debug("Revealing objects") + # All parameters are in the same args list. At the moment we + # only know the type, the name and the "value" of the + # parameter. This value may be treated to get the actual + # object (e.g: deserialize it, query the database in case of + # persistent objects, etc.) + self.reveal_objects( + args, + kwargs["compss_collections_layouts"], + kwargs["compss_python_MPI"], + ) + if __debug__: + LOGGER.debug("Finished revealing objects") + LOGGER.debug("Building task parameters structures") + + # After this line all the objects in arg have a "content" + # field, now we will segregate them in User positional and + # variadic args + user_args, user_kwargs, ret_params = self.segregate_objects( + args + ) + num_returns = len(ret_params) + + if __debug__: + LOGGER.debug("Finished building parameters structures.") + + # Self definition (only used when defined in the task) + # Save the self object type and value before executing the task + # (it could be persisted inside if its a persistent object) + self_type = -1 + self_value = None + has_self = False + if args and not isinstance(args[0], Parameter): + if __debug__: + LOGGER.debug("Detected self parameter") + # Then the first arg is self + has_self = True + self_type = get_compss_type(args[0]) + if self_type == parameter.TYPE.EXTERNAL_PSCO: + if __debug__: + LOGGER.debug("\t - Self is a PSCO") + self_value = args[0].getID() + else: + # Since we are checking the type of the deserialized + # self parameter, get_compss_type will return that its + # type is parameter.TYPE.OBJECT, which, although it is + # an object, self is always a file for the runtime. + # So we must force its type to avoid that the return + # message notifies that it has a new type "object" + # which is not supported for python objects in the + # runtime. + self_type = parameter.TYPE.FILE + self_value = "null" + + # Call the user function with all the reconstructed parameters + # and get the return values. + if __debug__: + LOGGER.debug("Invoking user code") + # Now execute the user code + result = self.execute_user_code( + user_args, user_kwargs, kwargs["compss_tracing"], cache_on + ) + user_returns, compss_exception, default_values = result + + # It is not necessary to wait for all nested tasks since we + # delegate the serialization of their parameters or returns + # to the inner tasks. + + if __debug__: + LOGGER.debug("Finished user code") + + python_mpi = False + if kwargs["compss_python_MPI"]: + python_mpi = True + + # Deal with defaults if any + if default_values: + self.manage_defaults(args, default_values) + + # Deal with INOUTs and COL_OUTs + self.manage_inouts(args, python_mpi) + + # Deal with COMPSsExceptions + if compss_exception is not None: + if __debug__: + LOGGER.warning("Detected COMPSs Exception. Raising.") + raise compss_exception + + # Deal with returns (if any) + user_returns = self.manage_returns( + num_returns, user_returns, ret_params, python_mpi + ) + + # We must notify COMPSs when types are updated + new_types, new_values = self.manage_new_types_values( + num_returns, + user_returns, + args, + ret_params, + has_self, + self_type, + self_value, + ) + + # Clean cached references + if self.cache.references: + # Let the garbage collector act + self.cache.references = [] # loose all references + + # Release memory after task execution + self.__release_memory__(user_returns) + + if __debug__ and "COMPSS_WORKER_PROFILE_PATH" in os.environ: + self.__report_heap__() + + if __debug__: + LOGGER.debug("Finished @task decorator") + + return ( + new_types, + new_values, + self.decorator_arguments.target_direction, + args, + ) + + @staticmethod + def __release_memory__(user_returns) -> None: + """Release memory after task execution explicitly. + + :return: None. + """ + SHARED_ARGUMENTS.delete_worker_args() + # Call garbage collector: The memory may not be freed to the SO, + # although the objects are removed. + gc.collect() + # Then try to deallocate the empty memory. + try: + import ctypes + + libc = ctypes.CDLL("libc.so.6") + libc.malloc_trim(0) + except Exception: # pylint: disable=broad-except + if __debug__: + LOGGER.warning("Could NOT deallocate memory.") + + try: + import cupy + + if user_returns is not None: + for obj in user_returns: + if isinstance(obj, cupy.ndarray): + obj.data.mem.free() + except ImportError: + pass + + @staticmethod + def __report_heap__() -> None: + """Print the heap status. + + :return: None. + """ + if __debug__: + LOGGER.debug("Memory heap report:") + try: + import guppy # noqa + except ImportError: + LOGGER.warning("Could NOT import Guppy.") + else: + if __debug__: + LOGGER.debug(guppy.hpy().heap()) + + def reveal_objects( + self, + args: tuple, + collections_layouts: typing.Dict[str, typing.Tuple[int, int, int]], + python_mpi: bool = False, + ) -> None: + """Get the objects from the args message. + + This function takes the arguments passed from the persistent worker + and treats them to get the proper parameters for the user function. + + :param args: Arguments. + :param python_mpi: If the task is python MPI. + :param collections_layouts: Layouts of collections params for python + MPI tasks. + :return: None. + """ + if self.storage_supports_pipelining(): + if __debug__: + LOGGER.debug("The storage supports pipelining.") + # Perform the pipelined getByID operation + pscos = [ + x + for x in args + if x.content_type == parameter.TYPE.EXTERNAL_PSCO + ] + identifiers = [x.content for x in pscos] + from storage.api import getByID # noqa + + objects = getByID(*identifiers) + # Just update the Parameter object with its content + for obj, value in zip(objects, pscos): + obj.content = value + + # Deal with all the parameters that are NOT returns + if "COMPSS_THREADED_DESERIALIZATION" in os.environ: + # Concurrent: + if "COMPSS_NUM_CPUS" in os.environ: + max_workers = int(os.environ["COMPSS_NUM_CPUS"]) + else: + max_workers = 1 + if __debug__: + LOGGER.debug( + "Parallelized retrieve_content (max_workers=%s)" + % str(max_workers) + ) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for arg in [ + x + for x in args + if isinstance(x, Parameter) and not is_return(x.name) + ]: + futures.append( + executor.submit( + self.retrieve_content, + arg, + "", + python_mpi, + collections_layouts, + ) + ) + wait(futures) + else: + for arg in [ + x + for x in args + if isinstance(x, Parameter) and not is_return(x.name) + ]: + self.retrieve_content(arg, "", python_mpi, collections_layouts) + + @staticmethod + def storage_supports_pipelining() -> bool: + """Check if storage supports pipelining. + + Some storage implementations use pipelining + Pipelining means "accumulate the getByID queries and perform them + in a single megaquery". + If this feature is not available (storage does not support it) + getByID operations will be performed one after the other. + + :return: True if pipelining is supported. False otherwise. + """ + try: + import storage.api # noqa + + return storage.api.__pipelining__ + except (ImportError, AttributeError): + return False + + def retrieve_content( + self, + argument: Parameter, + name_prefix: str, + python_mpi: bool, + collections_layouts: typing.Dict[str, typing.Tuple[int, int, int]], + depth: int = 0, + force_file: bool = False, + ) -> None: + """Retrieve the content of a particular argument. + + :param argument: Argument. + :param name_prefix: Name prefix. + :param python_mpi: If the task is python MPI. + :param collections_layouts: Layouts of collections params for python + MPI tasks. + :param depth: Collection depth (0 if not a collection). + :param force_file: Force file type for collections or dict_collections + of files. + :return: None + """ + if __debug__: + LOGGER.debug("\t - Revealing: " + str(argument.name)) + LOGGER.debug( + "\t - checking: " + str(get_name_from_kwarg(argument.name)) + ) + # This case is special, as a FILE can actually mean a FILE or an + # object that is serialized in a file + if is_vararg(argument.name): + self.param_varargs = argument.name + if __debug__: + LOGGER.debug("\t\t - It is vararg") + + content_type = argument.content_type + type_file = parameter.TYPE.FILE + type_directory = parameter.TYPE.DIRECTORY + type_external_stream = parameter.TYPE.EXTERNAL_STREAM + type_collection = parameter.TYPE.COLLECTION + type_dict_collection = parameter.TYPE.DICT_COLLECTION + type_external_psco = parameter.TYPE.EXTERNAL_PSCO + + if content_type == type_file: + if ( + self.is_parameter_an_object(argument.name) + and not force_file + # and argument.content_type != parameter.TYPE.FILE + ): + # Get the direction from the decorator (could be taken from + # the runtime in compss_types if sorted) + _dec_arg = self.decorator_arguments.get_parameter_or_none( + argument.name + ) + if _dec_arg: + argument.direction = _dec_arg.direction + if argument.direction == parameter.DIRECTION.OUT: + # It is an OUT direction object: The file will not exist + # So, instantiate from info + if __debug__: + LOGGER.debug( + "\t\t - It is an OUT Object. " + "Instantiating type: %s", + str(argument.extra_content_type), + ) + argument.content = create_object_by_con_type( + argument.extra_content_type + ) + if __debug__: + LOGGER.debug("\t\t - Instantiation finished") + else: + # The object is stored in some file, load and deserialize + if __debug__: + LOGGER.debug( + "\t\t - It is an OBJECT. " + "Deserializing from file: %s", + str(argument.file_name.original_path), + ) + argument.content = self.recover_object(argument) + if __debug__: + LOGGER.debug("\t\t - Deserialization finished") + else: + # The object is a FILE, just forward the path of the file + # as a string parameter + argument.content = argument.file_name.original_path + if __debug__: + LOGGER.debug( + "\t\t - It is FILE: %s", str(argument.content) + ) + elif content_type == type_directory: + if __debug__: + LOGGER.debug("\t\t - It is a DIRECTORY") + argument.content = argument.file_name.original_path + elif content_type == type_external_stream: + if __debug__: + LOGGER.debug("\t\t - It is an EXTERNAL STREAM") + argument.content = self.recover_object(argument) + # ################################################################# + # ## THIS IS TEMPORAL UNTIL THE EXTERNAL PSCO STREAM TYPE IS ## + # ## IMPLEMENTED ## + # ################################################################# + if isinstance(argument.content, PscoStreamWrapper): + # It is a persisted object hidden within a stream object + psco_id = argument.content.get_psco_id() + if __debug__: + LOGGER.debug( + "\t\t - It is hiding a PSCO STREAM with id: %s", + str(psco_id), + ) + # The object is a PSCO, do a single getByID of the PSCO + from storage.api import getByID # noqa + + argument.content = getByID(psco_id) + # ################################################################# + elif content_type == type_collection: + argument.content = [] + # This field is exclusive for COLLECTION_T parameters, so make + # sure you have checked this parameter is a collection before + # consulting it + argument.collection_content = [] + col_f_name = str(argument.file_name.original_path) + + # maybe it is an inner-collection.. + _dec_arg = self.decorator_arguments.get_parameter_or_none( + argument.name + ) + _col_dir = _dec_arg.direction if _dec_arg else None + _col_dep = _dec_arg.depth if _dec_arg else depth + if __debug__: + LOGGER.debug("\t\t - It is a COLLECTION: %s", col_f_name) + LOGGER.debug("\t\t\t - Depth: %s", str(_col_dep)) + + # Check if this collection is in layout + # Three conditions: + # 1- this is a mpi task + # 2- it has a collection layout + # 3- the current argument is the layout target + in_mpi_collection_env = False + if ( + python_mpi + and collections_layouts + and argument.name in collections_layouts + ): + in_mpi_collection_env = True + from pycompss.util.mpi.helper import rank_distributor + + # Call rank_distributor if the current param is the target of + # the layout for each rank, return its offset(s) in the + # collection. + rank_distribution = rank_distributor( + collections_layouts[argument.name] + ) + rank_distr_len = len(rank_distribution) + if __debug__: + LOGGER.debug( + "Rank distribution is: %s", str(rank_distribution) + ) + + with open(col_f_name, "r") as col_f_name_fd: + for i, line in enumerate(col_f_name_fd): + if in_mpi_collection_env and i not in rank_distribution: + # Isn't this my offset? skip + continue + elems = line.strip().split() + data_type = elems[0] + content_file = elems[1] + content_type_elem = elems[2] + # Same naming convention as in COMPSsRuntimeImpl.java + sub_name = f"{argument.name}.{i}" + if name_prefix: + sub_name = f"{name_prefix}.{argument.name}" + else: + sub_name = f"@{sub_name}" + + if __debug__: + LOGGER.debug( + "\t\t\t - Revealing element: %s", str(sub_name) + ) + + is_file_collection = self.is_parameter_file_collection( + argument.name + ) + is_really_file = ( + is_file_collection or content_type_elem == "FILE" + ) + sub_arg, _ = build_task_parameter( + int(data_type), + parameter.IOSTREAM.UNSPECIFIED, + "", + sub_name, + content_file, + str(argument.content_type), + logger=LOGGER, + ) + + # if direction of the collection is "out", it means we + # haven't received serialized objects from the Master + # (even though parameters have "file_name", those files + # haven't been created yet). plus, inner collections of + # col_out params do NOT have "direction", we identify + # them by "depth".. + if _col_dir == parameter.DIRECTION.OUT or ( + (_col_dir is None) and _col_dep > 0 + ): + # if we are at the last level of COL_OUT param, + # create "empty" instances of elements + if ( + _col_dep == 1 + or content_type_elem != "collection:list" + ): + # Not a nested collection anymore + if is_really_file: + sub_arg.content = content_file + sub_arg.content_type = parameter.TYPE.FILE + else: + temp = create_object_by_con_type( + content_type_elem + ) + sub_arg.content = temp + # In case that only one element is used in this + # mpi rank, the collection list is removed + if in_mpi_collection_env and rank_distr_len == 1: + argument.content = sub_arg.content + argument.content_type = sub_arg.content_type + else: + argument.content.append(sub_arg.content) + argument.collection_content.append(sub_arg) + else: + # Is nested collection + self.retrieve_content( + sub_arg, + sub_name, + python_mpi, + collections_layouts, + depth=_col_dep - 1, + force_file=is_really_file, + ) + # In case that only one element is used in this mpi + # rank, the collection list is removed + if in_mpi_collection_env and rank_distr_len == 1: + argument.content = sub_arg.content + argument.content_type = sub_arg.content_type + else: + argument.content.append(sub_arg.content) + argument.collection_content.append(sub_arg) + else: + # Recursively call the retrieve method, fill the + # content field in our new taskParameter object + self.retrieve_content( + sub_arg, + sub_name, + python_mpi, + collections_layouts, + force_file=is_really_file, + ) + # In case only one element is used in this mpi rank, + # the collection list is removed + if in_mpi_collection_env and rank_distr_len == 1: + argument.content = sub_arg.content + argument.content_type = sub_arg.content_type + else: + argument.content.append(sub_arg.content) + argument.collection_content.append(sub_arg) + elif content_type == type_dict_collection: + argument.content = {} + # This field is exclusive for DICT_COLLECTION_T parameters, so + # make sure you have checked this parameter is a dictionary + # collection before consulting it + argument.dict_collection_content = {} + dict_col_f_name = argument.file_name.original_path + # Uncomment if you want to check its contents: + # print("Dictionary file name: " + str(dict_col_f_name)) + # print("Dictionary file contents:") + # with open(dict_col_f_name, "r") as f: + # print(f.read()) + + # Maybe it is an inner-dict-collection + _dec_arg = self.decorator_arguments.get_parameter_or_none( + argument.name + ) + _dict_col_dir = _dec_arg.direction if _dec_arg else None + _dict_col_dep = _dec_arg.depth if _dec_arg else depth + + with open(dict_col_f_name, "r") as dict_file: + lines = dict_file.readlines() + entries = group_iterable(lines, 2) + i = 0 + for entry in entries: + entry_k = entry[0] + entry_v = entry[1] + ( + data_type_key, + content_file_key, + content_type_key, + ) = entry_k.strip().split() + ( + data_type_value, + content_file_value, + content_type_value, + ) = entry_v.strip().split() + # Same naming convention as in COMPSsRuntimeImpl.java + sub_name_key = f"{argument.name}.{i}" + sub_name_value = f"{argument.name}.{i}" + if name_prefix: + sub_name_key = f"{name_prefix}.{argument.name}" + sub_name_value = f"{name_prefix}.{argument.name}" + else: + sub_name_key = f"@key{sub_name_key}" + sub_name_value = f"@value{sub_name_value}" + + sub_arg_key, _ = build_task_parameter( + int(data_type_key), + parameter.IOSTREAM.UNSPECIFIED, + "", + sub_name_key, + content_file_key, + str(argument.content_type), + logger=LOGGER, + ) + sub_arg_value, _ = build_task_parameter( + int(data_type_value), + parameter.IOSTREAM.UNSPECIFIED, + "", + sub_name_value, + content_file_value, + str(argument.content_type), + logger=LOGGER, + ) + + # if direction of the dictionary collection is "out", it + # means we haven't received serialized objects from the + # Master (even though parameters have "file_name", those + # files haven't been created yet). plus, inner dictionary + # collections of dict_col_out params do NOT have + # "direction", we identify them by "depth".. + if _dict_col_dir == parameter.DIRECTION.OUT or ( + (_dict_col_dir is None) and _dict_col_dep > 0 + ): + # if we are at the last level of DICT_COL_OUT param, + # create "empty" instances of elements + if ( + _dict_col_dep == 1 + or content_type_elem != "collection:dict" + ): + if content_type_elem == "FILE": + temp_k = content_file_key + temp_v = content_file_value + else: + temp_k = create_object_by_con_type( + content_type_key + ) + temp_v = create_object_by_con_type( + content_type_value + ) + sub_arg_key.content = temp_k + sub_arg_value.content = temp_v + argument.content[sub_arg_key.content] = ( + sub_arg_value.content + ) + argument.dict_collection_content[sub_arg_key] = ( + sub_arg_value + ) + else: + self.retrieve_content( + sub_arg_key, + sub_name_key, + python_mpi, + collections_layouts, + depth=_dict_col_dep - 1, + ) + self.retrieve_content( + sub_arg_value, + sub_name_value, + python_mpi, + collections_layouts, + depth=_dict_col_dep - 1, + ) + argument.content[sub_arg_key.content] = ( + sub_arg_value.content + ) + argument.dict_collection_content[sub_arg_key] = ( + sub_arg_value + ) + else: + # Recursively call the retrieve method, fill the + # content field in our new taskParameter object + self.retrieve_content( + sub_arg_key, + sub_name_key, + python_mpi, + collections_layouts, + ) + self.retrieve_content( + sub_arg_value, + sub_name_value, + python_mpi, + collections_layouts, + ) + argument.content[sub_arg_key.content] = ( + sub_arg_value.content + ) # noqa: E501 + argument.dict_collection_content[sub_arg_key] = ( + sub_arg_value # noqa: E501 + ) + elif ( + not self.storage_supports_pipelining() + and content_type == type_external_psco + ): + if __debug__: + LOGGER.debug("\t\t - It is a PSCO") + # The object is a PSCO and the storage does not support + # pipelining, do a single getByID of the PSCO + from storage.api import getByID # noqa + + argument.content = getByID(argument.content) + # If we have not entered in any of these cases we will assume + # that the object was a basic type and the content is already + # available and properly casted by the python worker + + def recover_object(self, argument: Parameter) -> typing.Any: + """Recover the object within a file. + + :param argument: Parameter object for the argument to recover. + :return: The object associated to the given argument Parameter. + """ + name = argument.name + original_path = argument.file_name.original_path + + if is_vararg(name): + param_name = name + else: + regex_param_name = r"(?:@|^)([a-zA-Z_]\w*)(?=\.\d+|$)" + match = re.match(regex_param_name, name) + if match: + param_name = match.group(1) + else: + if __debug__: + LOGGER.debug( + "\t\t - Couldn't extract param name '%s'", str(name) + ) + param_name = name + + # Check if cache is available + cache = ( + self.cache.in_queue is not None + and self.cache.out_queue is not None + ) + use_cache = False # default store object in cache + + if cache: + # Check if the user has defined that the parameter has or not to be + # stored in cache explicitly + if ( + not self.cache.profiler + and param_name in self.decorator_arguments.parameters + ): + use_cache = self.decorator_arguments.parameters[ + param_name + ].cache + else: + if is_vararg(name): + vararg_name = get_name_from_vararg(name) + if ( + not self.cache.profiler + and vararg_name in self.decorator_arguments.parameters + ): + use_cache = self.decorator_arguments.parameters[ + vararg_name + ].cache + elif self.cache.profiler: + use_cache = True + else: + # if not explicitly said, the object is candidate to be + # stored in cache + use_cache = False + argument.cache = use_cache + if __debug__ and cache: + LOGGER.debug( + "\t\t - Has to be saved in cache: %s", str(use_cache) + ) + + if NP and cache and use_cache: + # Check if the object is already in cache + if CACHE_TRACKER.in_cache(LOGGER, original_path, self.cache.ids): + # The object is cached + with EventInsideWorker(TRACING_WORKER.cache_hit_event): + if __debug__: + LOGGER.debug( + "\t\t - Found in cache (Cache hit) " + "- retrieving: %s", + str(original_path), + ) + ( + retrieved, + existing_shm, + ) = CACHE_TRACKER.retrieve_object_from_cache( + LOGGER, + self.cache.ids, + self.cache.in_queue, + self.cache.out_queue, + original_path, + name, + self.user_function, + self.cache.profiler, + ) + self.cache.references.append(existing_shm) + return retrieved + # Else not in cache. + # Retrieve from file and put in cache if possible + # :::: + # Where: + # = source name + # = destination name + # = keep source + # = is write final value + # = original name + # out or inout + is write final ==> do not put into cache? + # (currently only means if is + # different from read) + # out + keep source ==> impossible + # noqa inout + keep source ==> Lool for the destination name + + # Put into cache after with the + # destination name + # If keep source == False ==> Look for the source name instead of + # the destination mane + # Do not put into cache if IN and keep source == False + # If keep source == True ==> Put in cache if it is not already + with EventInsideWorker(TRACING_WORKER.cache_miss_event): + if __debug__: + LOGGER.debug( + "\t\t - Not found in cache (Cache miss) " + "- deserializing: %s", + str(original_path), + ) + obj = deserialize_from_file(original_path, LOGGER) + if ( + argument.file_name.keep_source + and argument.direction != parameter.DIRECTION.IN_DELETE + ): + # Try to insert in cache. + # May not be inserted if there is other process inserting the + # same file. + CACHE_TRACKER.insert_object_into_cache( + LOGGER, + self.cache.in_queue, + self.cache.out_queue, + obj, + original_path, + name, + self.user_function, + ) + return obj + + return deserialize_from_file(original_path, LOGGER) + + def segregate_objects(self, args: tuple) -> typing.Tuple[list, dict, list]: + """Split a list of arguments. + + Segregates a list of arguments in user positional, variadic and + return arguments. + + :return: list of user arguments, dictionary of user kwargs and a list + of return parameters. + """ + # User args + user_args = [] + # User named args (kwargs) + user_kwargs = {} + # Return parameters, save them apart to match the user returns with + # the internal parameters + ret_params = [] + + for arg in args: + # Just fill the three data structures declared above + # Deal with the self parameter (if any) + if not isinstance(arg, Parameter): + user_args.append(arg) + # All these other cases are all about regular parameters + elif is_return(arg.name): + ret_params.append(arg) + elif is_kwarg(arg.name): + user_kwargs[get_name_from_kwarg(arg.name)] = arg.content + else: + if is_vararg(arg.name): + self.param_varargs = get_name_from_vararg(arg.name) + # Apart from the names we preserve the original order, so it + # is guaranteed that named positional arguments will never be + # swapped with variadic ones or anything similar + user_args.append(arg.content) + + return user_args, user_kwargs, ret_params + + def execute_user_code( + self, + user_args: list, + user_kwargs: dict, + tracing: bool, + cache_on: bool, + ) -> typing.Tuple[ + typing.Any, typing.Optional[COMPSsException], typing.Optional[dict] + ]: + """Execute the user code. + + Disables the tracing hook if tracing is enabled. Restores it + at the end of the user code execution. + + :param user_args: Function args. + :param user_kwargs: Function kwargs. + :param tracing: If tracing enabled. + :return: The user function returns and the compss exception (if any). + """ + with EventInsideWorker(TRACING_WORKER.execute_user_code_event): + # Tracing hook is disabled by default during the user code of + # the task. + # The user can enable it with tracing_hook=True in @task decorator + # for specific tasks or globally with the COMPSS_TRACING_HOOK=true + # environment variable. + restore_hook = False + pro_f = None + if tracing: + global_tracing_hook = False + if CONSTANTS.tracing_hook_env_var in os.environ: + hook_enabled = ( + os.environ[CONSTANTS.tracing_hook_env_var] == "true" + ) + global_tracing_hook = hook_enabled + if ( + self.decorator_arguments.tracing_hook + or global_tracing_hook + ): + # The user wants to keep the tracing hook + pass + else: + # When Extrae library implements the function to disable, + # use it, as: + # import pyextrae + # pro_f = pyextrae.shutdown() + # Since it is not available yet, we manage the tracing hook + # by ourselves + pro_f = sys.getprofile() + sys.setprofile(None) + restore_hook = True + + user_returns = None # type: typing.Any + compss_exception = None # type: typing.Optional[COMPSsException] + default_values = None # type: typing.Optional[dict] + if self.decorator_arguments.numba: + # Import all supported functionalities + from numba import jit + from numba import njit + from numba import generated_jit + from numba import vectorize + from numba import guvectorize + from numba import stencil + from numba import cfunc + + numba_mode = self.decorator_arguments.numba + numba_flags = self.decorator_arguments.numba_flags + if ( + isinstance(numba_mode, dict) + or numba_mode is True + or numba_mode == "jit" + ): + # Use the flags defined by the user + numba_flags["cache"] = True # Always force cache + user_returns = jit(self.user_function, **numba_flags)( + *user_args, **user_kwargs + ) + # Alternative way of calling: + # user_returns = jit(cache=True)(self.user_function) \ + # (*user_args, **user_kwargs) + elif numba_mode == "generated_jit": + user_returns = generated_jit( + self.user_function, **numba_flags + )(*user_args, **user_kwargs) + elif numba_mode == "njit": + numba_flags["cache"] = True # Always force cache + user_returns = njit(self.user_function, **numba_flags)( + *user_args, **user_kwargs + ) + elif numba_mode == "vectorize": + numba_signature = self.decorator_arguments.numba_signature + user_returns = vectorize(numba_signature, **numba_flags)( + self.user_function + )(*user_args, **user_kwargs) + elif numba_mode == "guvectorize": + numba_signature = self.decorator_arguments.numba_signature + numba_decl = self.decorator_arguments.numba_declaration + user_returns = guvectorize( + numba_signature, numba_decl, **numba_flags + )(self.user_function)(*user_args, **user_kwargs) + elif numba_mode == "stencil": + user_returns = stencil(**numba_flags)(self.user_function)( + *user_args, **user_kwargs + ) + elif numba_mode == "cfunc": + numba_signature = self.decorator_arguments.numba_signature + user_returns = cfunc(numba_signature)( + self.user_function + ).ctypes(*user_args, **user_kwargs) + else: + raise PyCOMPSsException("Unsupported numba mode.") + else: + try: + # Normal task execution + user_returns = self.user_function( + *user_args, **user_kwargs + ) + except COMPSsException as compss_exc: + # Perform any required action on failure + user_returns, default_values = self.manage_exception() + compss_exception = compss_exc + # Set target direction in the exception + compss_exception.target_direction = get_direction_from_key( + self.decorator_arguments.target_direction + ) + except Exception as exc: # pylint: disable=broad-except + if self.on_failure == "IGNORE": + # Perform any required action on failure + user_returns, default_values = self.manage_exception() + else: + # Re-raise the exception + raise exc + + # Reestablish the hook if it was disabled + if restore_hook: + sys.setprofile(pro_f) + + if cache_on: + clean_cupy_env() + + return user_returns, compss_exception, default_values + + def manage_exception( + self, + ) -> typing.Tuple[typing.Optional[int], typing.Optional[dict]]: + """Deal with exceptions (on failure action). + + :return: The default return and values. + """ + user_returns = None + default_values = None + if self.on_failure == "IGNORE": + # Provide default return + user_returns = self.defaults.pop("returns", None) + # Provide defaults to the runtime + default_values = self.defaults + return user_returns, default_values + + def manage_defaults(self, args: tuple, default_values: dict) -> None: + """Deal with default values. + + WARNING! Updates args with the appropriate object or file. + + :param args: Argument list. + :param default_values: Dictionary containing the default values. + :return: None. + """ + if __debug__: + LOGGER.debug("Dealing with default values") + for arg in args: + # Skip non-task-parameters + if not isinstance(arg, Parameter): + continue + # Skip returns + if is_return(arg.name): + continue + if self.is_parameter_an_object(arg.name): + # Update object + arg.content = default_values[arg.name] + else: + # Update file + copyfile(str(default_values[arg.name]), str(arg.content)) + + def manage_inouts(self, args: tuple, python_mpi: bool) -> None: + """Deal with INOUTS. + + Serializes the result of INOUT parameters. + + :param args: Argument list. + :param python_mpi: Boolean if python mpi. + :return: None. + """ + if __debug__: + LOGGER.debug("Dealing with INOUTs and OUTS") + if python_mpi: + LOGGER.debug("\t - Managing with MPI policy") + + # Manage all the possible outputs of the task and build the return new + # types and values + for arg in args: + # Handle only task parameters that are objects + + # Skip files and non-task-parameters + if not isinstance( + arg, Parameter + ) or not self.is_parameter_an_object( + arg.name, + ): + continue + + original_name = get_name_from_kwarg(arg.name) + real_direction = get_default_direction( + original_name, self.decorator_arguments, self.param_args + ) + param = self.decorator_arguments.get_parameter( + original_name, real_direction + ) + # Update args + arg.direction = param.direction + + # File collections are objects, but must be skipped as well + if self.is_parameter_file_collection(arg.name): + continue + + # Skip psco: since param.content_type has the old type, we can + # not use: param.content_type != parameter.TYPE.EXTERNAL_PSCO + _is_psco_true = ( + arg.content_type == parameter.TYPE.EXTERNAL_PSCO + or is_psco(arg.content) + ) + if _is_psco_true: + continue + + # skip non-inouts or non-col_outs + _is_col_out = ( + arg.content_type == parameter.TYPE.COLLECTION + and param.direction == parameter.DIRECTION.OUT + ) + + _is_dict_col_out = ( + arg.content_type == parameter.TYPE.DICT_COLLECTION + and param.direction == parameter.DIRECTION.OUT + ) + + _is_inout = param.direction in ( + parameter.DIRECTION.INOUT, + parameter.DIRECTION.COMMUTATIVE, + ) + + _is_out = ( + param.direction == parameter.DIRECTION.OUT + and arg.content_type != parameter.TYPE.EXTERNAL_STREAM + ) + + if not (_is_inout or _is_out or _is_col_out or _is_dict_col_out): + continue + + # Now it is "INOUT" or "OUT" or "COLLECTION_OUT" or + # "DICT_COLLECTION_OUT" or not has been delegated. + # object param, serialize to a file. + if arg.content_type == parameter.TYPE.COLLECTION: + if __debug__: + LOGGER.debug("Serializing collection: %s", str(arg.name)) + # handle collections recursively + for content, elem in get_collection_objects(arg.content, arg): + if elem.file_name: + _is_delegated = False + if CONTEXT.is_nesting_enabled(): + if self._check_if_delegated(elem): + # Skip serialization + _is_delegated = True + + if not _is_delegated: + f_name = elem.file_name.original_path + if __debug__: + LOGGER.debug( + "\t - Serializing element: %s to %s", + str(arg.name), + str(f_name), + ) + if python_mpi: + serialize_to_file_mpienv( + content, f_name, False, LOGGER + ) + else: + serialize_to_file(content, f_name, LOGGER) + self.update_object_in_cache(content, arg) + else: + # It is None --> PSCO + pass + elif arg.content_type == parameter.TYPE.DICT_COLLECTION: + if __debug__: + LOGGER.debug( + "Serializing dictionary collection: " + str(arg.name) + ) + # handle dictionary collections recursively + for content, elem in get_dict_collection_objects( + arg.content, arg + ): + if elem.file_name: + _is_delegated = False + if CONTEXT.is_nesting_enabled(): + if self._check_if_delegated(elem): + # Skip serialization + _is_delegated = True + + if not _is_delegated: + f_name = elem.file_name.original_path + if __debug__: + LOGGER.debug( + "\t - Serializing element: %s to %s", + str(arg.name), + str(f_name), + ) + if python_mpi: + serialize_to_file_mpienv( + content, f_name, False, LOGGER + ) + else: + serialize_to_file(content, f_name, LOGGER) + self.update_object_in_cache(content, arg) + else: + # It is None --> PSCO + pass + else: + # NOTE: Synchronization with nesting is not needed anymore + # since we are delegating the writing of the file to + # the nested tasks. + if CONTEXT.is_nesting_enabled(): + if self._check_if_delegated(arg): + # Skip serialization + continue + + f_name = arg.file_name.original_path + + if __debug__: + LOGGER.debug( + "Serializing object: %s to %s", + str(arg.name), + str(f_name), + ) + + if python_mpi: + serialize_to_file_mpienv( + arg.content, f_name, False, LOGGER + ) + else: + serialize_to_file(arg.content, f_name, LOGGER) + self.update_object_in_cache(arg.content, arg) + + @staticmethod + def _check_if_delegated(arg: Parameter) -> bool: + """Check if the argument is delegated. + + CAUTION: Modifies arg if is delegated! + + :param arg: Parameter to be checked (and updated if necessary). + :return: True if delegated (serialization must be skipped). + False otherwise. + """ + obj_id = OT.is_tracked(arg.content) + _is_delegated = obj_id != "" + if _is_delegated: + arg.is_future = _is_delegated + # Update the file path to the delegated file name + obj_file_path = OT.get_file_name(obj_id) + arg.file_name.source_path = obj_file_path + arg.file_name.destination_name = obj_id + old_original_path = arg.file_name.original_path + arg.file_name.original_path = obj_file_path + if __debug__: + LOGGER.debug( + "Parameter %s serialization has been delegated " + "to a nested task (%s).", + old_original_path, + arg.file_name.original_path, + ) + # Skip serialization + return True + return False + + def update_object_in_cache( + self, content: typing.Any, argument: Parameter + ) -> None: + """Update the object into cache if possible. + + :param content: Object to be updated. + :param argument: Parameter object for the argument to be updated. + :return: None. + """ + name = argument.name + original_path = argument.file_name.original_path + + cache = ( + self.cache.in_queue is not None + and self.cache.out_queue is not None + ) + if ( + not self.cache.profiler + and name in self.decorator_arguments.parameters + ): + use_cache = self.decorator_arguments.parameters[name].cache + elif self.cache.profiler: + use_cache = True + else: + # if not explicitly said, the object is candidate to be cached + use_cache = False + if NP and cache and use_cache: + if CACHE_TRACKER.in_cache(LOGGER, original_path, self.cache.ids): + CACHE_TRACKER.replace_object_into_cache( + LOGGER, + self.cache.in_queue, + self.cache.out_queue, + content, + original_path, + name, + self.user_function, + ) + else: + with EventInsideWorker(TRACING_WORKER.cache_miss_event): + pass + CACHE_TRACKER.insert_object_into_cache( + LOGGER, + self.cache.in_queue, + self.cache.out_queue, + content, + original_path, + name, + self.user_function, + ) + + def manage_returns( + self, + num_returns: int, + user_returns: typing.Any, + ret_params: list, + python_mpi: bool, + ) -> typing.Any: + """Manage task returns. + + WARNING: Modifies ret_params, which is included into args. + + :param num_returns: Number of returns. + :param user_returns: User returns. + :param ret_params: Return parameters. + :param python_mpi: Boolean if is python mpi code. + :return: User returns. + """ + if __debug__: + LOGGER.debug("Dealing with returns: %s", str(num_returns)) + if num_returns > 0: + if num_returns == 1: + # Generalize the return case to multi-return to simplify the + # code + user_returns = [user_returns] + elif num_returns > 1 and python_mpi: + user_returns = [user_returns] + ret_params = get_ret_rank(ret_params) + # Note that we are implicitly assuming that the length of the user + # returns matches the number of return parameters + for obj, param in zip(user_returns, ret_params): + # Store the object int ret_params (included in args) + param.content = obj + param.direction = parameter.DIRECTION.OUT + + f_name = param.file_name.original_path + + # If the object is a PSCO, do not serialize to file + if ( + param.content_type == parameter.TYPE.EXTERNAL_PSCO + or is_psco(obj) + ): + continue + + if CONTEXT.is_nesting_enabled(): + obj_id = OT.is_tracked(param.content) + _is_delegated = obj_id != "" + if _is_delegated: + param.is_future = _is_delegated + # Update the file path to the delegated file name + obj_file_path = OT.get_file_name(obj_id) + param.file_name.destination_name = obj_id + param.file_name.original_path = obj_file_path + + if isinstance(param.content, Future) or param.is_future: + if __debug__: + LOGGER.debug( + "Return %s delegated to a nested task (%s).", + f_name, + param.file_name.original_path, + ) + continue + + # Serialize the object + # Note that there is no "command line optimization" in the + # returns, as we always pass them as files. + # This is due to the asymmetry in worker-master communications + # and because it also makes it easier for us to deal with + # returns in that format + + if __debug__: + LOGGER.debug("Serializing return: %s", str(f_name)) + if python_mpi: + if num_returns > 1: + rank_zero_reduce = False + else: + rank_zero_reduce = True + + serialize_to_file_mpienv( + obj, f_name, rank_zero_reduce, LOGGER + ) + else: + serialize_to_file(obj, f_name, LOGGER) + if ( + self.cache.in_queue is not None + and self.cache.out_queue is not None + and ( + self.cache.profiler + or self.decorator_arguments.cache_returns + ) + and not CACHE_TRACKER.in_cache( + LOGGER, f_name, self.cache.ids + ) + ): + with EventInsideWorker(TRACING_WORKER.cache_miss_event): + if __debug__: + LOGGER.debug("Storing return in cache") + CACHE_TRACKER.insert_object_into_cache( + LOGGER, + self.cache.in_queue, + self.cache.out_queue, + obj, + f_name, + "Return", + self.user_function, + ) + return user_returns + + def is_parameter_an_object(self, name: str) -> bool: + """Given the name of a parameter, determine if it is an object or not. + + :param name: Name of the parameter. + :return: True if the parameter is a (serializable) object. + """ + original_name = get_name_from_kwarg(name) + # Get the args parameter object + if is_vararg(original_name): + varargs_direction = self.decorator_arguments.varargs_type + param = get_new_parameter(varargs_direction) + return param.content_type == -1 + # Is this parameter annotated in the decorator? + if original_name in self.decorator_arguments.parameters: + annotated = [ + parameter.TYPE.COLLECTION, + parameter.TYPE.DICT_COLLECTION, + parameter.TYPE.EXTERNAL_STREAM, + parameter.TYPE.OBJECT, + -1, + ] + return ( + self.decorator_arguments.parameters[original_name].content_type + in annotated + ) + # The parameter is not annotated in the decorator, so return default + return True + + def is_parameter_file_collection(self, name: str) -> bool: + """Determine if the given parameter name it's a file collection or not. + + :param name: Name of the parameter. + :return: True if the parameter is a file collection. + """ + original_name = get_name_from_kwarg(name) + # Get the args parameter object + if is_vararg(original_name): + varargs_direction = self.decorator_arguments.varargs_type + param = get_new_parameter(varargs_direction) + return param.is_file_collection + # Is this parameter annotated in the decorator? + if original_name in self.decorator_arguments.parameters: + return self.decorator_arguments.parameters[ + original_name + ].is_file_collection + # The parameter is not annotated in the decorator, so (by default) + # return False + return False + + def manage_new_types_values( + self, + num_returns: int, + user_returns: typing.Any, + args: tuple, + ret_params: list, + has_self: bool, + self_type: int, + self_value: typing.Any, + ) -> typing.Tuple[list, list]: + """Manage new types and values. + + We must notify COMPSs when types are updated + Potential update candidates are returns and INOUTs + But the whole types and values list must be returned + new_types and new_values correspond to "parameters self returns" + + :param num_returns: Number of returns. + :param user_returns: User returns. + :param args: Arguments. + :param ret_params: Return parameters. + :param has_self: If has self. + :param self_type: Self type. + :param self_value: Self value. + :return: List new types, List new values. + """ + new_types, new_values = [], [] + if __debug__: + LOGGER.debug("Building types update") + + def build_collection_types_values( + _content: typing.Any, _arg: Parameter, direction: int + ) -> list: + """Retrieve collection type-value recursively. + + :param _content: Object or list of objects. + :param _arg: Argument or list of arguments of the given objects. + :param direction: Direction of the object/s. + :returns: The collection representation. + """ + coll = [] # type: list + for _cont, _elem in zip(_arg.content, _arg.collection_content): + if isinstance(_elem, str): + coll.append([parameter.TYPE.FILE, "null"]) + else: + if _elem.content_type == parameter.TYPE.COLLECTION: + coll.append( + build_collection_types_values( + _cont, _elem, direction + ) + ) + elif ( + _elem.content_type == parameter.TYPE.EXTERNAL_PSCO + and is_psco(_cont) + and direction != parameter.DIRECTION.IN + ): + coll.append([_elem.content_type, _cont.getID()]) + elif ( + _elem.content_type == parameter.TYPE.FILE + and is_psco(_cont) + and direction != parameter.DIRECTION.IN + ): + coll.append( + [parameter.TYPE.EXTERNAL_PSCO, _cont.getID()] + ) + else: + if CONTEXT.is_nesting_enabled(): + if _elem.is_future: + coll.append( + [ + _elem.content_type, + _elem.file_name.original_path, + ] + ) + else: + coll.append([_elem.content_type, "null"]) + else: + coll.append([_elem.content_type, "null"]) + return coll + + # Add parameter types and value + params_start = 1 if has_self else 0 + params_end = len(args) - num_returns + 1 + # Update new_types and new_values with the args list + # The results parameter is a boolean to distinguish the error message. + for arg in args[params_start : params_end - 1]: # noqa: E203 + # Loop through the arguments and update new_types and new_values + if not isinstance(arg, Parameter): + raise PyCOMPSsException( + "ERROR: A task parameter arrived as an object instead as" + " a TaskParameter when building the task result message." + ) + original_name = get_name_from_kwarg(arg.name) + real_direction = get_default_direction( + original_name, self.decorator_arguments, self.param_args + ) + param = self.decorator_arguments.get_parameter( + original_name, real_direction + ) + if arg.content_type in ( + parameter.TYPE.EXTERNAL_PSCO, + parameter.TYPE.FILE, + ): + # It was originally a persistent object + if is_psco(arg.content): + new_types.append(parameter.TYPE.EXTERNAL_PSCO) + new_values.append(arg.content.getID()) + else: + new_types.append(arg.content_type) + if CONTEXT.is_nesting_enabled(): + if arg.is_future: + new_values.append(arg.file_name.original_path) + else: + new_values.append("null") + else: + new_values.append("null") + elif arg.content_type == parameter.TYPE.COLLECTION: + # There is a collection that can contain persistent objects + collection_new_values = build_collection_types_values( + arg.content, arg, param.direction + ) + new_types.append(parameter.TYPE.COLLECTION) + new_values.append(collection_new_values) + else: + # Any other return object: same type and null value + new_types.append(arg.content_type) + if CONTEXT.is_nesting_enabled(): + if arg.is_future: + new_values.append(arg.file_name.original_path) + else: + new_values.append("null") + else: + new_values.append("null") + + # Add self type and value if exist + if has_self: + if ( + self.decorator_arguments.target_direction + == parameter.INOUT.key + ): + # Check if self is a PSCO that has been persisted inside the + # task and target_direction. + # Update self type and value + self_type = get_compss_type(args[0]) + if self_type == parameter.TYPE.EXTERNAL_PSCO: + self_value = args[0].getID() + else: + # Self can only be of type FILE, so avoid the last update + # of self_type + if is_psco(args[0]): + self_type = parameter.TYPE.EXTERNAL_PSCO + self_value = args[0].getID() + else: + self_type = parameter.TYPE.FILE + self_value = "null" + new_types.append(self_type) + new_values.append(self_value) + + # Add return types and values + # Loop through the rest of the arguments and update new_types and + # new_values. + # assert len(args[params_end - 1:]) == len(user_returns) + # add_parameter_new_types_and_values(args[params_end - 1:], True) + if num_returns > 0: + for ret, param in zip(user_returns, ret_params): + ret_type = get_compss_type(ret) + ret_value = "null" + if ret_type == parameter.TYPE.EXTERNAL_PSCO: + ret_value = ret.getID() + elif ret_type == parameter.TYPE.COLLECTION: + collection_ret_values = [] + for elem in ret: + if elem.type in ( + parameter.TYPE.EXTERNAL_PSCO, + parameter.TYPE.FILE, + ): + if is_psco(elem.content): + collection_ret_values.append(elem.key) + else: + if CONTEXT.is_nesting_enabled(): + if param.is_future: + collection_ret_values.append( + elem.file_name.original_path + ) + else: + collection_ret_values.append("null") + else: + collection_ret_values.append("null") + else: + if CONTEXT.is_nesting_enabled(): + if param.is_future: + collection_ret_values.append( + elem.file_name.original_path + ) + else: + collection_ret_values.append("null") + else: + collection_ret_values.append("null") + new_types.append(parameter.TYPE.COLLECTION) + new_values.append(collection_ret_values) + else: + # Returns can only be of type FILE, so avoid the last + # update of ret_type + ret_type = parameter.TYPE.FILE + if CONTEXT.is_nesting_enabled(): + if param.is_future: + ret_value = param.file_name.original_path + else: + ret_value = "null" + else: + ret_value = "null" + new_types.append(ret_type) + new_values.append(ret_value) + + return new_types, new_values + + +####################### +# AUXILIARY FUNCTIONS # +####################### + + +def clean_cupy_env(): + """Clean cupy environment. + + :return: None + """ + try: + import cupy + + CACHE_TRACKER.close_cupy_mem_handles() + except ImportError: + pass + + +def get_collection_objects( + content: typing.Any, argument: Parameter +) -> typing.Generator[typing.Tuple[typing.Any, Parameter], None, None]: + """Retrieve collection objects recursively generator. + + WARNING! Updates the collection with any modification from content. + + :param content: Object or list of objects. + :param argument: Argument or list of arguments of the given objects. + :return: The collection representation. + """ + if argument.content_type == parameter.TYPE.COLLECTION: + for new_con, _elem in zip( + argument.content, argument.collection_content + ): + # Update the sub-parameter content with the existing content + # to keep track of the synchronized. + _elem.content = new_con + for sub_el, sub_param in get_collection_objects(new_con, _elem): + # Update the sub-parameter content with the existing content + # to keep track of the synchronized. + sub_param.content = sub_el + yield sub_el, sub_param + else: + # Update the sub-parameter content with the existing content + # to keep track of the synchronized. + argument.content = content + # NOTE: Synchronization with nesting is not needed anymore since + # we are delegating the writing of the file to the nested + # task. + yield content, argument + + +def get_dict_collection_objects( + content: typing.Any, argument: Parameter +) -> typing.Generator[typing.Tuple[typing.Any, Parameter], None, None]: + """Retrieve the dictionary collection objects recursively generator. + + WARNING! Updates the dictionary collection with any modification + from content. + + :param content: Object or list of objects. + :param argument: Argument or list of arguments of the given objects. + :return: The collection representation. + """ + if argument.content_type == parameter.TYPE.DICT_COLLECTION: + elements = [] + for content_k, content_v in argument.content.items(): + elements.extend([content_k, content_v]) + # Prepare dict_collection_content per key + element_parameters_preproc = {} + for ( + dict_coll_k, + dict_coll_v, + ) in argument.dict_collection_content.items(): + element_parameters_preproc[dict_coll_k.content] = [ + dict_coll_k, + dict_coll_v, + ] + # Ensure that the element parameters are in the same order as + # argument.content + elements_parameters = [] + for content_k in argument.content.keys(): + elements_parameters.extend( + [ + element_parameters_preproc[content_k][0], + element_parameters_preproc[content_k][1], + ] + ) + # Loop recursively + for new_con, _elem in zip(elements, elements_parameters): + _elem.content = new_con + for sub_el, sub_param in get_dict_collection_objects( + new_con, _elem + ): + # Update the sub-parameter content with the existing content + # to keep track of the synchronized. + sub_param.content = sub_el + yield sub_el, sub_param + else: + # Update the sub-parameter content with the existing content + # to keep track of the synchronized. + argument.content = content + # NOTE: Synchronization with nesting is not needed anymore since + # we are delegating the writing of the file to the nested + # task. + yield content, argument + + +def get_ret_rank(_ret_params: list) -> list: + """Retrieve the rank id within MPI. + + :param _ret_params: Return parameters. + :return: Integer return rank. + """ + from mpi4py import MPI # pylint: disable=import-outside-toplevel + + return [_ret_params[MPI.COMM_WORLD.rank]] diff --git a/examples/dds/pycompss/runtime/task/wrappers/__init__.py b/examples/dds/pycompss/runtime/task/wrappers/__init__.py new file mode 100644 index 00000000..64fcec88 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/wrappers/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the PyCOMPSs runtime task wrappers.""" + +# ########################################################################### +# ### THIS IS TEMPORAL UNTIL THE EXTERNAL PSCO STREAM TYPE IS IMPLEMENTED ### +# ########################################################################### diff --git a/examples/dds/pycompss/runtime/task/wrappers/psco_stream.py b/examples/dds/pycompss/runtime/task/wrappers/psco_stream.py new file mode 100644 index 00000000..72541f13 --- /dev/null +++ b/examples/dds/pycompss/runtime/task/wrappers/psco_stream.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs runtime - Task - Wrappers - psco_stream. + +This file contains the PSCO Stream wrapper class. +""" + +# ########################################################################### +# ### THIS IS TEMPORAL UNTIL THE EXTERNAL PSCO STREAM TYPE IS IMPLEMENTED ### +# ########################################################################### + +from pycompss.util.typing_helper import typing + + +class PscoStreamWrapper: + """PSCO Stream wrapper definition. + + This class represents a PSCO Stream object. + """ + + __slots__ = [ + "psco_id", + ] + + def __init__(self, psco_id: typing.Union[str, None]) -> None: + """Set the persistent object identifier in the placeholder.""" + self.psco_id = psco_id + + def get_psco_id(self) -> typing.Union[str, None]: + """Retrieve the persistent object identifier.""" + return self.psco_id diff --git a/examples/dds/pycompss/streams/__init__.py b/examples/dds/pycompss/streams/__init__.py new file mode 100644 index 00000000..3642f645 --- /dev/null +++ b/examples/dds/pycompss/streams/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the Streaming API functions, constants and classes.""" + +# For * imports +__all__ = ["distro_stream", "environment"] diff --git a/examples/dds/pycompss/streams/components/__init__.py b/examples/dds/pycompss/streams/components/__init__.py new file mode 100644 index 00000000..512b318b --- /dev/null +++ b/examples/dds/pycompss/streams/components/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This package contains the Streaming components helpers. + +Where the helpers are functions, constants and classes. +""" + +__all__ = ["distro_stream_client"] diff --git a/examples/dds/pycompss/streams/components/distro_stream_client.py b/examples/dds/pycompss/streams/components/distro_stream_client.py new file mode 100644 index 00000000..ecbdfa16 --- /dev/null +++ b/examples/dds/pycompss/streams/components/distro_stream_client.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - Streams - Components. + +This file contains the distro stream components code. +""" + +# Imports +import queue +import socket +from threading import Thread + +# Project imports +from pycompss.streams.types.requests import STOP +from pycompss.streams.types.requests import StopRequest +from pycompss.util.typing_helper import typing + +# +# Logger definition +# + +if __debug__: + import logging + + logger = logging.getLogger("pycompss.streams.distro_stream_client") + + +# +# Client Handler +# + + +class DistroStreamClientHandler: + """Handler to use the DistroStreamClient. + + This is a static class. + + Attributes: + - CLIENT: Distributed Stream Client to handle. + + type: DistroStreamClient + """ + + CLIENT = None # type: typing.Any + + def __init__(self) -> None: + """Create a new handler instance. + + Should never be called directly since all attributes are static. + """ + # Nothing to do since this is a static handler + + @staticmethod + def init_and_start( + master_ip: typing.Optional[str] = None, + master_port: typing.Optional[str] = None, + ) -> None: + """Initialize and starts the client. + + :param master_ip: Master IP. + :param master_port: Master port. + :return: None. + """ + DistroStreamClientHandler.CLIENT = DistroStreamClient( + master_ip=master_ip, master_port=master_port + ) + DistroStreamClientHandler.CLIENT.start() + + @staticmethod + def set_stop() -> None: + """Mark the client to stop. + + :return: None. + """ + req = StopRequest() + DistroStreamClientHandler.CLIENT.add_request(req) + + @staticmethod + def request(req: typing.Any) -> None: + """Add a new request to the client. + + :param req: Client request (Subclass of Request) + :return: None. + """ + DistroStreamClientHandler.CLIENT.add_request(req) + + +# +# Client definition +# +class DistroStreamClient(Thread): + """Distro Stream Client definition. + + Attributes: + - master_ip: Master IP address. + + type: string + - master_port: Master port. + + type: int + - running: Whether the client thread is running or not + + type: boolean + - requests: Queue of pending client requests + + type: Queue.Queue + """ + + BUFFER_SIZE = 4096 + + def __init__( + self, + master_ip: typing.Optional[str], + master_port: typing.Optional[str], + ) -> None: + """Create a new Client associated to the given master properties. + + :param master_ip: Master IP address. + :param master_port: Master port. + """ + super().__init__() + + if __debug__: + logger.info( + "Initializing DS Client on %s:%s", master_ip, master_port + ) + + # Register information + self.master_ip = master_ip + self.master_port = int(str(master_port)) + + # Initialize internal structures + self.running = True + self.requests = None # type: typing.Any + self.requests = queue.Queue() + + def run(self) -> None: + """Run method of the internal thread. + + :return: None. + """ + if __debug__: + logger.info("DS Client started") + + while self.running: + # Process requests + req = self.requests.get(block=True) + + if __debug__: + logger.debug("Processing request: %s", str(req.get_type())) + + if req.get_type() == STOP: + if __debug__: + logger.info("DS Client asked to stop") + self.running = False + req.set_response("DONE") + else: + self._process_request(req) + + # Mark the request as processed + req.set_processed() + self.requests.task_done() + + if __debug__: + logger.info("DS Client stopped") + + def _process_request(self, req: typing.Any) -> None: + """Process requests to the server. + + :param req: Request. + :return: None. + """ + # Open socket connection + try: + comm_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + comm_socket.connect((self.master_ip, self.master_port)) + + # Send request message + req_msg = req.get_request_msg() + if __debug__: + logger.debug("Sending request to server: %s", str(req_msg)) + req_msg = req_msg + "\n" + req_msg = req_msg.encode() + comm_socket.sendall(req_msg) + if __debug__: + logger.debug("Sent request to server") + + # Receive answer + chunk = comm_socket.recv(DistroStreamClient.BUFFER_SIZE) + answer = chunk + if __debug__: + logger.debug("Received answer from server: %s", str(answer)) + while ( + chunk is not None + and chunk + and not chunk.endswith("\n".encode()) + ): + if __debug__: + logger.debug( + "Received chunk answer from server with size = %s", + str(len(chunk)), + ) + chunk = comm_socket.recv(DistroStreamClient.BUFFER_SIZE) + if chunk is not None and chunk: + answer = answer + chunk + answer_str = answer.decode(encoding="UTF-8").strip() + if __debug__: + logger.debug( + "Received answer from server: %s", str(answer_str) + ) + req.set_response(answer_str) + except Exception as general_exception: # pylint: disable=broad-except + if __debug__: + logger.error( + "ERROR: Cannot process request \n %s", + str(general_exception), + ) + # Some error occurred, mark request as failed and keep going + req.set_error(1, str(general_exception)) + + def add_request(self, req: typing.Any) -> None: + """Add a new request to the client. + + :param req: Request to add (Request subclass). + :return: None. + """ + if __debug__: + logger.debug( + "Adding new request to client queue: %s", str(req.get_type()) + ) + self.requests.put(req, block=True) diff --git a/examples/dds/pycompss/streams/components/objects/__init__.py b/examples/dds/pycompss/streams/components/objects/__init__.py new file mode 100644 index 00000000..a5597a75 --- /dev/null +++ b/examples/dds/pycompss/streams/components/objects/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This package contains the Streaming component objects helpers. + +Where the helpers are functions, constants and classes. +""" + +__all__ = ["kafka_connectors"] diff --git a/examples/dds/pycompss/streams/components/objects/kafka_connectors.py b/examples/dds/pycompss/streams/components/objects/kafka_connectors.py new file mode 100644 index 00000000..c01d7292 --- /dev/null +++ b/examples/dds/pycompss/streams/components/objects/kafka_connectors.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - Streams - Components - Objects. + +This file contains the distro stream components objects code. +""" + +# Imports +import pickle +import socket + +from pycompss.util.typing_helper import typing + +# +# Logger definition +# + +if __debug__: + import logging + + logger = logging.getLogger("pycompss.streams.distro_stream") + +# +# Globals +# + +API_VERSION = (3, 8, 0) + +# +# ODSPublisher definition +# + + +class ODSPublisher: + """ODS Publisher connector implementation. + + Attributes: + - kafka_producer: KafkaProducer instance + + type: KafkaProducer + """ + + def __init__(self, bootstrap_server: str) -> None: + """Create a new ODSPublisher instance. + + :param bootstrap_server: Associated boostrap server. + """ + if __debug__: + logger.debug("Creating Publisher...") + + # Create internal producer + from kafka3 import KafkaProducer + + bootstrap_server_info = str(bootstrap_server).split(":") + bootstrap_server_ip = str( + socket.gethostbyname(bootstrap_server_info[0]) + ) + bootstrap_server_port = str(bootstrap_server_info[1]) + self.kafka_producer = KafkaProducer( + bootstrap_servers=f"{bootstrap_server_ip}:{bootstrap_server_port}", + acks="all", + retries=0, + batch_size=16384, + linger_ms=0, + api_version=API_VERSION, + max_block_ms=120000, + ) + # Other flags: + # auto_commit_interval_ms=2, + # key_serializer="org.apache.kafka.common.serialization.StringSerializer", # noqa: E501 + # value_serializer="org.apache.kafka.common.serialization.StringSerializer", # noqa: E501 + # block_on_buffer_full=True + + if __debug__: + logger.debug("DONE Creating Publisher") + + def publish(self, topic: typing.Union[bytes, str], message: str) -> None: + """Publish the given message to the given topic. + + :param topic: Message topic. + :param message: Message to publish. + :return: None. + """ + if __debug__: + logger.debug("Publishing Message to %s ...", str(topic)) + + # Fix topic if required + if isinstance(topic, bytes): + topic_fix = str(topic.decode("utf-8")) + else: + topic_fix = str(topic) + + # Serialize message + serialized_message = pickle.dumps(message) + + # Send message + self.kafka_producer.send(topic_fix, value=serialized_message) + self.kafka_producer.flush() + + if __debug__: + logger.debug("DONE Publishing Message") + + +# +# ODSConsumer definition +# + + +class ODSConsumer: + """ODS Consumer connector implementation. + + Attributes: + - topic: Registered topic name on the Kafka backend + + type: string + - access_mode: Consumer access mode + + type: string + - kafka_consumer: KafkaConsumer instance + + type: KafkaConsumer + """ + + def __init__( + self, bootstrap_server: str, topic: str, access_mode: str + ) -> None: + """Create a new ODSConsumer instance. + + :param bootstrap_server: Associated boostrap server. + :param topic: Topic where to consume records. + :param access_mode: Consumer access mode. + """ + if __debug__: + logger.debug("Creating Consumer...") + + if isinstance(topic, bytes): + topic_fix = str(topic.decode("utf-8")) # noqa + else: + topic_fix = str(topic) + self.topic = topic_fix + self.access_mode = access_mode + + # Parse configuration + + # Create internal consumer + from kafka3 import KafkaConsumer + + bootstrap_server_info = str(bootstrap_server).split(":") + bootstrap_server_ip = str( + socket.gethostbyname(bootstrap_server_info[0]) + ) # noqa: E501 + bootstrap_server_port = str(bootstrap_server_info[1]) + if __debug__: + logger.debug( + "Bootstrap server: %s - info: %s - %s:%s", + bootstrap_server, + bootstrap_server_info, + bootstrap_server_ip, + bootstrap_server_port, + ) + self.kafka_consumer = KafkaConsumer( + bootstrap_servers=f"{bootstrap_server_ip}:{bootstrap_server_port}", + enable_auto_commit=True, + auto_commit_interval_ms=200, + group_id=self.topic, + auto_offset_reset="earliest", + session_timeout_ms=10000, + fetch_min_bytes=1, + receive_buffer_bytes=262144, + max_partition_fetch_bytes=2097152, + api_version=API_VERSION, + ) + # Other flags: + # key_deserializer="org.apache.kafka.common.serialization.StringSerializer", # noqa: E501 + # value_deserializer="org.apache.kafka.common.serialization.StringSerializer", # noqa: E501 + + # Subscribe consumer + self.kafka_consumer.subscribe([self.topic]) + + if __debug__: + logger.debug("DONE Creating Consumer") + + def poll(self, timeout: int) -> list: + """Poll messages from the subscribed topics. + + :param timeout: Poll timeout. + :return: List of polled messages (strings - can be empty but not None). + """ + if __debug__: + logger.debug("Polling Messages from %s ...", str(self.topic)) + + new_messages = [] + # First _ was tp. + for _, records in self.kafka_consumer.poll( + timeout_ms=timeout + ).items(): # noqa: E501 + for record in records: + if record.topic == self.topic: + deserialized_message = pickle.loads(record.value) + new_messages.append(deserialized_message) + else: + logger.warning( + "Ignoring received message on unregistered topic %s", + str(record.topic), + ) + + if __debug__: + logger.debug( + "DONE Polling Messages (%s elements)", str(len(new_messages)) + ) + + return new_messages diff --git a/examples/dds/pycompss/streams/distro_stream.py b/examples/dds/pycompss/streams/distro_stream.py new file mode 100644 index 00000000..4e02264e --- /dev/null +++ b/examples/dds/pycompss/streams/distro_stream.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - Streams - Distro stream. + +This file contains the distro stream code. +""" + +import uuid +from abc import abstractmethod + +from pycompss.streams.components.distro_stream_client import ( + DistroStreamClientHandler, +) +from pycompss.streams.components.objects.kafka_connectors import ( + ODSPublisher, + ODSConsumer, +) +from pycompss.streams.types.requests import BootstrapServerRequest +from pycompss.streams.types.requests import CloseStreamRequest +from pycompss.streams.types.requests import PollRequest +from pycompss.streams.types.requests import PublishRequest + +# Project imports +from pycompss.streams.types.requests import RegisterStreamRequest +from pycompss.streams.types.requests import StreamStatusRequest + +# Imports +from pycompss.util.typing_helper import typing + +# +# Logger definition +# +if __debug__: + import logging + + logger = logging.getLogger("pycompss.streams.distro_stream") + +# +# Type Enums +# +# class StreamType(Enum): +FILE = "FILE" +OBJECT = "OBJECT" +PSCO = "PSCO" + +# class ConsumerMode(Enum): +AT_MOST_ONCE = "AT_MOST_ONCE" +AT_LEAST_ONCE = "AT_LEAST_ONCE" + +# Common messages +POLLING_MSG = "Polling new stream items..." + + +def str2bool(val: str) -> bool: + """Convert string to boolean. + + :param val: String to analyse. + :return: If val means true or false. + """ + return val.lower() in ("yes", "true", "t", "1") + + +# +# Interface definition +# + + +class DistroStream: + """Interface for File and Object Distributed Streams.""" + + def __init__(self) -> None: + """Create a new DistroStream instance.""" + + @abstractmethod + def get_stream_id(self) -> str: + """Return the internal stream id. + + :return: The internal stream id (str) + """ + + @abstractmethod + def get_stream_alias(self) -> str: + """Return the internal stream alias. + + :return: The internal stream alias (str) + """ + + @abstractmethod + def get_stream_type(self) -> str: + """Return the internal stream type. + + :return: The internal stream type (StreamType) + """ + + @abstractmethod + def publish(self, message: str) -> None: + """Publish the given message on the stream. + + :param message: Message to publish. + :return: None. + """ + + @abstractmethod + def publish_list(self, messages: list) -> None: + """Publish the given list of messages on the stream. + + :param messages: List of messages to publish. + :return: None. + """ + + @abstractmethod + def poll(self, timeout: int = 0) -> list: + """Poll the produced messages. + + If there are registered messages, returns immediately. Otherwise, + waits until a record is produced or the timeout is exceeded. + + :param timeout: Maximum request time to poll new messages. + :return: List of polled messages (List). + """ + return [] + + @abstractmethod + def close(self) -> None: + """Close the current stream. + + :return: None. + """ + + @abstractmethod + def is_closed(self) -> bool: + """Return whether the stream is closed or not. + + :return: True if the stream is closed, False otherwise. + """ + + +# +# Common Implementation +# + + +class DistroStreamImpl(DistroStream): + """Implementation of the common methods of the DistroStream Interface. + + Attributes: + - alias: Stream Alias. + + type: string + - id: Stream Id. + + type: string containing UUID + - stream_type: Internal stream type. + + type: StreamType + - access_mode: Stream consumer access mode. + + type: ConsumerMode + """ + + def __init__( + self, + alias: typing.Optional[str] = None, + stream_type: typing.Optional[str] = None, + internal_stream_info: typing.Optional[list] = None, + access_mode: str = AT_MOST_ONCE, + ) -> None: + """Create a new DistroStream instance. + + :param alias: Stream alias. + :param stream_type: Internal stream type (StreamType). + :param internal_stream_info: Implementation specific information + (List). + :param access_mode: Stream access mode (ConsumerMode). + :raise RegistrationException: When client cannot register the stream + into the server. + """ + super().__init__() + + if __debug__: + logger.debug("Registering new stream...") + + self.alias = alias + self.stream_type = stream_type + self.access_mode = access_mode + + # Retrieve registration id + req = RegisterStreamRequest( + self.alias, + self.stream_type, + self.access_mode, + internal_stream_info, + ) + DistroStreamClientHandler.request(req) + + req.wait_processed() + error = req.get_error_code() + if error != 0: + raise RegistrationException(error, req.get_error_msg()) + self.id = req.get_response_msg() # pylint: disable=invalid-name + + def get_stream_id(self) -> str: + """Retrieve the stream identifier. + + :returns: The stream identifier. + """ + return self.id + + def get_stream_alias(self) -> str: + """Retrieve the stream alias. + + :returns: The stream alias. + """ + return self.alias + + def get_stream_type(self) -> str: + """Retrieve the stream alias. + + :returns: The stream type. + """ + return self.stream_type + + @abstractmethod + def publish(self, message: str) -> None: + """Publish a message. + + :param message: Message to publish. + :returns: None. + """ + + @abstractmethod + def publish_list(self, messages: list) -> None: + """Publish a list of messages. + + :param messages: List of message to publish. + :returns: None. + """ + + @abstractmethod + def poll(self, timeout: int = 0) -> list: + """Poll the stream. + + :param timeout: Waiting time. + :returns: None. + """ + return [] + + def close(self) -> None: + """Close the stream. + + :returns: None. + """ + if __debug__: + logger.debug("Closing stream %s", str(self.id)) + + # Ask for stream closure + req = CloseStreamRequest(self.id) + DistroStreamClientHandler.request(req) + + req.wait_processed() + error = req.get_error_code() + if error != 0 and __debug__: + logger.error("ERROR: Cannot close stream") + logger.error(" - Internal Error Code: %s", str(error)) + logger.error(" - Internal Error Msg: %s", str(req.get_error_msg())) + + # No need to process the answer message, checking the error is enough. + + def is_closed(self) -> bool: + """Check if the stream is closed. + + :returns: If the stream is closed. + """ + if __debug__: + logger.debug("Checking if stream %s is closed", str(self.id)) + + # Ask for stream status + req = StreamStatusRequest(self.id) + DistroStreamClientHandler.request(req) + + req.wait_processed() + error = req.get_error_code() + if error != 0 and __debug__: + logger.error("ERROR: Cannot retrieve stream status") + logger.error(" - Internal Error Code: %s", str(error)) + logger.error(" - Internal Error Msg: %s", str(req.get_error_msg())) + + return str2bool(req.get_response_msg()) + + +# +# FileDistroStream definition +# + + +class FileDistroStream(DistroStreamImpl): + """File Distributed Stream implementation. + + Attributes: + - base_dir: Base directory path for the streaming. + + type: string + """ + + def __init__( + self, + alias: typing.Optional[str] = None, + base_dir: typing.Optional[str] = None, + access_mode: str = AT_MOST_ONCE, + ) -> None: + """Create a new FileDistroStream instance. + + :param alias: Stream alias. + :param base_dir: Base directory for the file stream. + :param access_mode: Stream access mode (ConsumerMode) + :raise RegistrationException: When client cannot register the stream + into the server. + """ + super().__init__( + alias=alias, + stream_type=FILE, + internal_stream_info=[base_dir], + access_mode=access_mode, + ) + self.base_dir = base_dir + + def publish(self, message: str) -> None: + """Publish message. + + Nothing to do since server automatically publishes the written files. + + :param message: Message to publish. + :return: None. + """ + if __debug__: + logger.warning( + "WARN: Unnecessary call on publish on FileDistroStream" + ) + + def publish_list(self, messages: list) -> None: + """Publish a list of messages. + + Nothing to do since server automatically publishes the written files. + + :param messages: List of messages to publish. + :return: None. + """ + if __debug__: + logger.warning( + "WARN: Unnecessary call on publish on FileDistroStream" + ) + + def poll(self, timeout: int = 0) -> list: + """Poll the stream. + + :param timeout: Waiting time. + :return: List of messages. + """ + if __debug__: + logger.info(POLLING_MSG) + + # Send request to server + req = PollRequest(self.id) + DistroStreamClientHandler.request(req) + + # Retrieve answer + req.wait_processed() + error = req.get_error_code() + if error != 0: + raise BackendException(error, req.get_error_msg()) + + # Parse answer + info = req.get_response_msg() + if __debug__: + logger.debug("Retrieved stream items: %s", str(info)) + if info is not None and info and info != "null": + return info.split() + return [] + + +# +# ObjectDistroStream definition +# + + +class ObjectDistroStream(DistroStreamImpl): + """Object Distributed Stream implementation. + + Attributes: + - kafka_topic_name: Registered topic name on the Kafka backend + + type: string + - bootstrap_server: Bootstrap server information + + type: string + - publisher: Internal Kafka connector for publish + + type: ODSPublisher + - consumer: Internal Kafka connector for consume + + type: ODSConsumer + """ + + TOPIC_REGULAR_MESSAGES_PREFIX = "regular-messages" + TOPIC_SYSTEM_MESSAGES = "system-messages" + DEFAULT_KAFKA_TIMEOUT = 200 # ms + + def __init__( + self, + alias: typing.Optional[str] = None, + access_mode: str = AT_MOST_ONCE, + ) -> None: + """Create a new ObjectDistroStream instance. + + :param alias: Stream alias. + :param access_mode: Stream access mode (ConsumerMode) + :raise RegistrationException: When client cannot register the stream + into the server. + """ + super().__init__( + alias=alias, + stream_type=OBJECT, + internal_stream_info=[], + access_mode=access_mode, + ) + self.kafka_topic_name = alias + if alias != "": + self.kafka_topic_name = ( + ObjectDistroStream.TOPIC_REGULAR_MESSAGES_PREFIX + + "-" + + self.id + ) # noqa: E501 + + self.bootstrap_server = "None" # type: str + self.publisher = None # type: typing.Any + self.consumer = None # type: typing.Any + + def _register_publisher(self) -> None: + """Register publisher. + + :return: None. + """ + if self.publisher is None: + if self.bootstrap_server == "None": + self.bootstrap_server = ( + ObjectDistroStream._request_bootstrap_server_info() + ) # noqa: E501 + if __debug__: + logger.info("Creating internal producer...") + self.publisher = ODSPublisher(self.bootstrap_server) + + def _register_consumer(self) -> None: + """Register consumer. + + :return: None. + """ + if self.consumer is None: + if self.bootstrap_server == "None": + self.bootstrap_server = ( + ObjectDistroStream._request_bootstrap_server_info() + ) # noqa: E501 + if __debug__: + logger.info("Creating internal consumer...") + self.consumer = ODSConsumer( + self.bootstrap_server, self.kafka_topic_name, self.access_mode + ) + + @staticmethod + def _request_bootstrap_server_info() -> str: + """Request bootstrap server information. + + :return: String with the retrieved information. + """ + if __debug__: + logger.info("Requesting bootstrap server...") + req = BootstrapServerRequest() + DistroStreamClientHandler.request(req) + + # Retrieve answer + req.wait_processed() + error = req.get_error_code() + if error != 0: + raise BackendException(error, req.get_error_msg()) + + # Parse answer + answer = req.get_response_msg() + if __debug__: + logger.debug("Retrieved bootstrap server information: %s", answer) + + return answer + + def publish(self, message: str) -> None: + """Publish message. + + :param message: Message to publish. + :return: None. + """ + if __debug__: + logger.info("Publishing new object...") + self._register_publisher() + self.publisher.publish(self.kafka_topic_name, message) + if __debug__: + logger.info("Publishing new object") + + def publish_list(self, messages: list) -> None: + """Publish message. + + :param messages: List of messages to publish. + :return: None. + """ + if __debug__: + logger.info("Publishing new List of objects...") + self._register_publisher() + for msg in messages: + self.publisher.publish(self.kafka_topic_name, msg) + if __debug__: + logger.info("Published new List of objects") + + def poll(self, timeout: int = DEFAULT_KAFKA_TIMEOUT) -> list: + """Poll server. + + :param timeout: Maximum waiting time. + :return: None. + """ + if __debug__: + logger.info(POLLING_MSG) + + self._register_consumer() + return self.consumer.poll(timeout) + + +# +# PscoDistroStream definition +# + + +class PscoDistroStream(DistroStreamImpl): + """PSCO Distributed Stream implementation.""" + + def __init__(self, alias: str, access_mode: str = AT_MOST_ONCE) -> None: + """Create a new PscoDistroStream instance. + + :param alias: Stream alias. + :param access_mode: Stream access mode (ConsumerMode). + :raise RegistrationException: When client cannot register the stream + into the server. + """ + super().__init__( + alias=alias, + stream_type=PSCO, + internal_stream_info=[], + access_mode=access_mode, + ) + + def publish(self, message: str) -> None: + """Publish message. + + :param message: Message to publish. + :return: None. + """ + if __debug__: + logger.info("Publishing new PSCO object...") + self._psco_publish(message) + if __debug__: + logger.info("Publishing new PSCO object") + + def publish_list(self, messages: list) -> None: + """Publish message. + + :param messages: List of messages to publish. + :return: None. + """ + if __debug__: + logger.info("Publishing new List of PSCOs...") + for msg in messages: + self._psco_publish(msg) + if __debug__: + logger.info("Published new List of PSCOs") + + def _psco_publish(self, psco: typing.Any) -> None: + """Publish message. + + :param psco: Persistent object to publish. + :return: None. + """ + # Persist the psco if its not + if __debug__: + logger.debug("Persisting user PSCO...") + if psco.getID() is None: + alias = str(uuid.uuid4()) + psco.makePersistent(alias) + psco_id = psco.getID() + + # Register the psco on the server + if __debug__: + logger.debug("Registering PSCO publish...") + req = PublishRequest(self.id, psco_id) + DistroStreamClientHandler.request(req) + + # Retrieve answer + req.wait_processed() + error = req.get_error_code() + if error != 0: + raise BackendException(error, req.get_error_msg()) + + # Parse answer + answer = req.get_response_msg() # noqa + if __debug__: + logger.debug("Publish stream answer: %s", str(answer)) + + def poll(self, timeout: int = 0) -> list: + """Poll server. + + :param timeout: Maximum waiting time. + :return: None. + """ + if __debug__: + logger.info(POLLING_MSG) + + # Send request to server + req = PollRequest(self.id) + DistroStreamClientHandler.request(req) + + # Retrieve answer + req.wait_processed() + error = req.get_error_code() + if error != 0: + raise BackendException(error, req.get_error_msg()) + + # Parse answer + info = req.get_response_msg() + if __debug__: + logger.debug("Retrieved stream items: %s", str(info)) + + from pycompss.util.storages.persistent import ( + get_by_id, + ) # pylint: disable=import-outside-toplevel + + retrieved_pscos = [] + if info is not None and info and info != "null": + for psco_id in info.split(): + psco = get_by_id(psco_id) + retrieved_pscos.append(psco) + return retrieved_pscos + + +# +# Exception Class +# + + +class RegistrationException(Exception): + """Registration exception class.""" + + def __init__( + self, + code: typing.Optional[int] = None, + message: typing.Optional[str] = None, + ) -> None: + """Create a new RegistrationException instance. + + :param code: Internal request error code. + :param message: Internal request error message. + """ + super().__init__() + self.code = code + self.message = message + + def __str__(self) -> str: + """Retrieve the string repr. of the RegistrationException object. + + :return: The string representation. + """ + message = ( + f"ERROR: Registration Exception.\n" + f" - Internal error code: {str(self.code)}\n" + f" - Internal error message: {str(self.message)}" + ) + return message + + +class BackendException(Exception): + """Backend exception class.""" + + def __init__( + self, + code: typing.Optional[int] = None, + message: typing.Optional[str] = None, + ) -> None: + """Create a new BackendException instance. + + :param code: Internal request error code. + :param message: Internal request error message. + """ + super().__init__() + self.code = code + self.message = message + + def __str__(self) -> str: + """Retrieve the string representation of the BackendException object. + + :return: The string representation. + """ + message = ( + f"ERROR: Backend Exception.\n" + f" - Internal error code: {str(self.code)}\n" + f" - Internal error message: {str(self.message)}" + ) + return message diff --git a/examples/dds/pycompss/streams/environment.py b/examples/dds/pycompss/streams/environment.py new file mode 100644 index 00000000..6e5b47fe --- /dev/null +++ b/examples/dds/pycompss/streams/environment.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Binding - Streams - Environment. + +This file contains the methods to setup the environment when using streaming. +It is called from interactive.py and launch.py scripts. +""" + +from pycompss.streams.components.distro_stream_client import ( + DistroStreamClientHandler, +) + +if __debug__: + import logging + + logger = logging.getLogger(__name__) + + +def init_streaming( + streaming_backend: str, + streaming_master_name: str, + streaming_master_port: str, +) -> bool: + """Initialize the streaming client. + + :param streaming_backend: Streaming backend. + :param streaming_master_name: Streaming backend master node name. + :param streaming_master_port: Streaming backend master port. + :return: True if initialized successfully, False otherwise. + """ + # Fix options if necessary + if ( + streaming_master_name == "" + or not streaming_master_name + or streaming_master_name == "null" + ): + streaming_master_name = "localhost" + if ( + streaming_master_port == "" + or not streaming_master_port + or streaming_master_port == "null" + ): + streaming_master_port = "49049" + + # Check if the stream backend is enabled + streaming_enabled = streaming_backend not in ("", "null", "None", "NONE") + + # Init stream backend if needed + if streaming_enabled: + if __debug__: + logger.debug("Starting DistroStream library") + DistroStreamClientHandler.init_and_start( + master_ip=streaming_master_name, + master_port=streaming_master_port, + ) + + # Return whether the streaming backend is enabled or not + return streaming_enabled + + +def stop_streaming() -> None: + """Stop the streaming backend. + + :return: None. + """ + if __debug__: + logger.debug("Stopping DistroStream library") + DistroStreamClientHandler.set_stop() diff --git a/examples/dds/pycompss/streams/types/__init__.py b/examples/dds/pycompss/streams/types/__init__.py new file mode 100644 index 00000000..0414c701 --- /dev/null +++ b/examples/dds/pycompss/streams/types/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This package contains the Streaming types helpers. + +Where the helpers are functions, constants and classes. +""" + +__all__ = ["requests"] diff --git a/examples/dds/pycompss/streams/types/requests.py b/examples/dds/pycompss/streams/types/requests.py new file mode 100644 index 00000000..47198c90 --- /dev/null +++ b/examples/dds/pycompss/streams/types/requests.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - Streams - Types. + +This file contains the distro stream types code. +""" + +# Imports +from abc import abstractmethod +from threading import Semaphore + +# +# Logger definition +# + +if __debug__: + import logging + + logger = logging.getLogger("pycompss.streams.distro_stream") + +# +# Type Enums +# +# class RequestType(Enum): +REGISTER_CLIENT = "REGISTER_CLIENT" +UNREGISTER_CLIENT = "UNREGISTER_CLIENT" +BOOTSTRAP_SERVER = "BOOTSTRAP_SERVER" +REGISTER_STREAM = "REGISTER_STREAM" +STREAM_STATUS = "STREAM_STATUS" +CLOSE_STREAM = "CLOSE_STREAM" +POLL = "POLL" +PUBLISH = "PUBLISH" +STOP = "STOP" + + +# +# Interface definition +# +class Request: + """Interface for Client-Server requests. + + Attributes: + - request_type: Request type. + + type: RequestType + - has_been_processed: Whether the request has been processed or not. + + type: boolean + - wait_sem: Internal waiting semaphore for request completion. + + type: Semaphore + - error_code: Request error code. + + type: int + - error_msg: Request error message. + + type: string + - response_msg: Client response message. + + type: string + """ + + def __init__(self, request_type: str) -> None: + """Create a new Request instance. + + :param request_type: Request type (RequestType). + """ + self.request_type = request_type + + self.has_been_processed = False + self.wait_sem = Semaphore(0) + + self.error_code = -1 + self.error_msg = "None" + self.response_msg = "None" + + def get_type(self) -> str: + """Return the request type. + + :return: The request type (RequestType). + """ + return self.request_type + + def is_processed(self) -> bool: + """Return whether the request has been processed or not. + + :return: True if the request has been processed, False otherwise. + """ + return self.has_been_processed + + def wait_processed(self) -> None: + """Lock the current thread until the request has been processed. + + :return: None. + """ + self.wait_sem.acquire() # pylint: disable=consider-using-with + + def get_error_code(self) -> int: + """Return the request error code. + + :return: The request error code. + """ + return self.error_code + + def get_error_msg(self) -> str: + """Return the request error message. + + :return: The request error message. + """ + return self.error_msg + + def get_response_msg(self) -> str: + """Return the request response message. + + :return: The request response message. + """ + return self.response_msg + + @abstractmethod + def get_request_msg(self) -> str: + """Return the request message to send to the server. + + :return: The request message to send to the server. + """ + + def set_processed(self) -> None: + """Mark the request as processed. + + :return: None. + """ + self.has_been_processed = True + self.wait_sem.release() + + def set_error(self, error_code: int, error_msg: str) -> None: + """Set a new error code and message to the current request. + + :param error_code: Error code. + :param error_msg: Error message. + :return: None. + """ + self.error_code = error_code + self.error_msg = error_msg + + def set_response(self, msg: str) -> None: + """Set a new response message to the current request. + + :param msg: New response message. + :return: None. + """ + self.error_code = 0 + self.response_msg = msg + + +# +# Specific requests implementations +# + + +class RegisterStreamRequest(Request): + """Request to register a new stream. + + Attributes: + - alias: Associated stream alias. + + type: string + - stream_type: Associated stream type. + + type: StreamType + - access_mode: Associated stream access mode. + + type: ConsumerMode + - internal_stream_info: Associated information about the internal + stream implementation. + + type: List + """ + + def __init__( + self, + alias: str, + stream_type: str, + access_mode: str, + internal_stream_info: list, + ) -> None: + """Create a new RegisterStreamRequest instance. + + :param alias: Associated stream alias. + :param stream_type: Associated stream type (StreamType). + :param access_mode: Associated stream access mode (ConsumerMode). + :param internal_stream_info: Associated information about the internal + stream implementation (List). + """ + super().__init__(request_type=REGISTER_STREAM) + self.alias = alias + self.stream_type = stream_type + self.access_mode = access_mode + self.internal_stream_info = internal_stream_info + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + message = " ".join( + ( + str(self.request_type), + str(self.stream_type), + str(self.access_mode), + str(self.alias), + ) + ) + if self.internal_stream_info is not None: + for info in self.internal_stream_info: + message = message + " " + str(info) + return message + + +class StopRequest(Request): + """Request to stop the client.""" + + def __init__(self) -> None: + """Create a new StopRequest instance.""" + super().__init__(request_type=STOP) + + def get_request_msg(self) -> str: + """Get request message. + + :returns: Mesage. + """ + message = str(self.request_type) + return message + + +class BootstrapServerRequest(Request): + """Request to retrieve the bootstrap server information.""" + + def __init__(self) -> None: + """Create a new BootstrapServerRequest instance.""" + super().__init__(request_type=BOOTSTRAP_SERVER) + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + message = str(self.request_type) + return message + + +class StreamStatusRequest(Request): + """Request to retrieve the status of the given stream. + + Attributes: + - stream_id : Stream Id. + + type: UUID + """ + + def __init__(self, stream_id: str) -> None: + """Create a new StreamStatusRequest instance. + + :param stream_id: Stream Id. + """ + super().__init__(request_type=STREAM_STATUS) + self.stream_id = stream_id + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + return f"{str(self.request_type)} {str(self.stream_id)}" + + +class CloseStreamRequest(Request): + """Request to close the given stream. + + Attributes: + - stream_id : Stream Id. + + type: UUID + """ + + def __init__(self, stream_id: str) -> None: + """Create a new CloseStreamRequest instance. + + :param stream_id: Stream Identifier. + """ + super().__init__(request_type=CLOSE_STREAM) + self.stream_id = stream_id + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + return f"{str(self.request_type)} {str(self.stream_id)}" + + +class PollRequest(Request): + """Request to poll the given stream. + + Attributes: + - stream_id : Stream Id. + + type: UUID + """ + + def __init__(self, stream_id: str) -> None: + """Create a new PollRequest instance. + + :param stream_id: Stream Identifier. + """ + super().__init__(request_type=POLL) + self.stream_id = stream_id + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + return f"{str(self.request_type)} {str(self.stream_id)}" + + +class PublishRequest(Request): + """Request to publish a new element to the given stream. + + Attributes: + - stream_id : Stream Id. + + type: UUID + - msg : Message to publish + + type: str + """ + + def __init__(self, stream_id: str, msg: str) -> None: + """Create a new PublishRequest instance. + + :param stream_id: Stream Id (UUID). + :param msg: Message to publish. + """ + super().__init__(request_type=PUBLISH) + self.stream_id = stream_id + self.msg = msg + + def get_request_msg(self) -> str: + """Get request message. + + :return: Message. + """ + return " ".join( + (str(self.request_type), str(self.stream_id), str(self.msg)) + ) diff --git a/examples/dds/pycompss/tests/__init__.py b/examples/dds/pycompss/tests/__init__.py new file mode 100644 index 00000000..53c00a15 --- /dev/null +++ b/examples/dds/pycompss/tests/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the tests.""" diff --git a/examples/dds/pycompss/tests/integration/__init__.py b/examples/dds/pycompss/tests/integration/__init__.py new file mode 100644 index 00000000..cb3e9059 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the integration tests.""" diff --git a/examples/dds/pycompss/tests/integration/dds/__init__.py b/examples/dds/pycompss/tests/integration/dds/__init__.py new file mode 100644 index 00000000..20568158 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/dds/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the dds integration tests.""" diff --git a/examples/dds/pycompss/tests/integration/dds/dds/__init__.py b/examples/dds/pycompss/tests/integration/dds/dds/__init__.py new file mode 100644 index 00000000..e9685bd4 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/dds/dds/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the dds integration tests examples.""" diff --git a/examples/dds/pycompss/tests/integration/dds/dds/dds_examples.py b/examples/dds/pycompss/tests/integration/dds/dds/dds_examples.py new file mode 100644 index 00000000..6671e024 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/dds/dds/dds_examples.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +PyCOMPSs Tests - Integration - DDS - Examples. + +This file contains the dds integration examples tests. +""" + +import os +import shutil +import sys +import tempfile +import time + +from pycompss.dds.examples import inverted_indexing +from pycompss.dds.examples import pi_estimation +from pycompss.dds.examples import terasort +from pycompss.dds.examples import transitive_closure +from pycompss.dds.examples import word_count +from pycompss.dds.examples import wordcount_k_means +from pycompss.runtime.binding import barrier +from pycompss.util.context import CONTEXT + +# The DDS examples unittest only checks functionality (not the validity of +# the results). +# TODO: check that the results are OK. + +EXAMPLES_NAME = "examples.py" + + +def pi_estimation_example(): + """Run the pi estimation example. + + :returns: None. + """ + pi_estimation() + + +def transitive_closure_example(): + """Run the transitive closure example. + + :returns: None. + """ + transitive_closure(2) + + +def wordcount_example(): + """Run the wordcount example. + + :returns: None. + """ + current_path = os.path.dirname(os.path.abspath(__file__)) + wordcount_dataset_path = os.path.join( + current_path, "../../../unittests/dds/dataset", "wordcount" + ) + argv_backup = sys.argv + sys.argv = [EXAMPLES_NAME, wordcount_dataset_path] + word_count() + sys.argv = argv_backup + + +def wordcount_k_means_example(): + """Run the wordcount K-means example. + + :returns: None. + """ + if sys.version_info >= (3, 0): + current_path = os.path.dirname(os.path.abspath(__file__)) + wordcount_k_means_dataset_path = os.path.join( + current_path, "../../../unittests/dds/dataset", "wordcount" + ) + argv_backup = sys.argv + sys.argv = [EXAMPLES_NAME, wordcount_k_means_dataset_path] + wordcount_k_means(dim=155) + sys.argv = argv_backup + else: + print("NOTE: Spacy fails in Python 2 [deprecating].") + + +def terasort_example(): + """Run the terasort example. + + :returns: None. + """ + result_path = tempfile.mkdtemp() + current_path = os.path.dirname(os.path.abspath(__file__)) + terasort_dataset_path = os.path.join( + current_path, "../../../unittests/dds/dataset", "terasort" + ) + argv_backup = sys.argv + sys.argv = [EXAMPLES_NAME, terasort_dataset_path, result_path] + terasort() + sys.argv = argv_backup + if CONTEXT.in_pycompss(): + barrier() + time.sleep(5) # TODO: Why is this sleep needed? + # Clean the results directory + shutil.rmtree(result_path) + + +def inverted_indexing_example(): + """Run the inverted indexing example. + + :returns: None. + """ + current_path = os.path.dirname(os.path.abspath(__file__)) + inverted_indexing_dataset_path = os.path.join( + current_path, "../../../unittests/dds/dataset", "wordcount" + ) + argv_backup = sys.argv + sys.argv = [EXAMPLES_NAME, inverted_indexing_dataset_path] + inverted_indexing() + sys.argv = argv_backup + + +def main(): + """Run all examples. + + :returns: None. + """ + # Examples to run + pi_estimation_example() + transitive_closure_example() + wordcount_example() + # Next test @unittest.skip("ERROR WITH SPACY (python 3.10 + spacy 2.3.2)") + # wordcount_k_means_example() + terasort_example() + inverted_indexing_example() diff --git a/examples/dds/pycompss/tests/integration/dds/test_dds_examples.py b/examples/dds/pycompss/tests/integration/dds/test_dds_examples.py new file mode 100644 index 00000000..7f91f413 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/dds/test_dds_examples.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + + +def test_launch_dds_examples(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "dds/dds_examples.py") + launch_pycompss_application(app, "main", debug=True, app_name="main") diff --git a/examples/dds/pycompss/tests/integration/launch/__init__.py b/examples/dds/pycompss/tests/integration/launch/__init__.py new file mode 100644 index 00000000..c371b820 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the launch integration tests.""" diff --git a/examples/dds/pycompss/tests/integration/launch/resources/__init__.py b/examples/dds/pycompss/tests/integration/launch/resources/__init__.py new file mode 100644 index 00000000..9038baed --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the launch integration tests resources.""" diff --git a/examples/dds/pycompss/tests/integration/launch/resources/api_tester.py b/examples/dds/pycompss/tests/integration/launch/resources/api_tester.py new file mode 100644 index 00000000..ed01cd20 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/api_tester.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench for the API.""" + +import os +import shutil +import time + +# from pycompss.api.api import compss_get_number_of_resources +# from pycompss.api.api import compss_request_resources +# from pycompss.api.api import compss_free_resources +from pycompss.api.api import TaskGroup +from pycompss.api.api import compss_barrier +from pycompss.api.api import compss_barrier_group +from pycompss.api.api import compss_delete_file +from pycompss.api.api import compss_delete_object +from pycompss.api.api import compss_file_exists +from pycompss.api.api import compss_open +from pycompss.api.api import compss_wait_on +from pycompss.api.api import compss_wait_on_directory +from pycompss.api.api import compss_wait_on_file +from pycompss.api.parameter import FILE +from pycompss.api.parameter import FILE_INOUT +from pycompss.api.parameter import FILE_OUT +from pycompss.api.parameter import DIRECTORY_INOUT +from pycompss.api.parameter import DIRECTORY_IN +from pycompss.api.parameter import DIRECTORY_OUT +from pycompss.api.task import task + + +@task(fin=FILE, returns=str) +def file_in(fin): + """Read the given file. + + :param fin: Input file path. + :returns: The input file contents. + """ + print("TEST FILE IN") + # Open the file and read the content + fin_d = open(fin, "r") + content = fin_d.read() + print("- In file content:\n", content) + # Close and return the content + fin_d.close() + return content + + +@task(finout=FILE_INOUT, returns=str) +def file_inout(finout): + """Read and modify the given file. + + CAUTION! Modifies finout. + + :param finout: Input output file path. + :returns: The given file contents. + """ + print("TEST FILE INOUT") + # Open the file and read the content + finout_d = open(finout, "r+") + content = finout_d.read() + print("- Inout file content:\n", content) + # Add some content + content += "\n===> INOUT FILE ADDED CONTENT" + finout_d.write("\n===> INOUT FILE ADDED CONTENT") + print("- Inout file content after modification:\n", content) + # Close and return with the modification + finout_d.close() + return content + + +@task(fout=FILE_OUT, returns=str) +def file_out(fout, content): + """Read the given file and returns as OUT and in return. + + :param fout: Input file path. + :param content: Output content. + :returns: The given file contents. + """ + print("TEST FILE OUT") + # Open the file for writing and write some content + with open(fout, "w") as fout_d: + fout_d.write(content) + print("- Out file content added:\n", content) + return content + + +def file_checker(filename, direction): + """Check if file exists. + + :param filename: File to be checked. + :param direction: File direction. + :returns: None. + """ + must_exist = compss_file_exists(filename) + compss_delete_file(filename) + must_not_exist = compss_file_exists(filename) + assert must_exist is True, "File %s that must exist not found." % direction + assert must_not_exist is False, ( + "File %s that must NOT exist is found." % direction + ) + + +def multiple_file_checker(filenames, directions): + """Check if multiple files exist. + + :param filenames: Files to be checked. + :param directions: Files direction. + :returns: None. + """ + must_exist = compss_file_exists(*filenames) + compss_delete_file(*filenames) + must_not_exist = compss_file_exists(*filenames) + assert all( + must_exist + ), "Multiple files %s that must exist not found." % str(directions) + assert not any( + must_not_exist + ), "Multiple files %s that must NOT exist is found." % str(directions) + + +def files(): + """Test files functionality. + + :returns: None. + """ + # Test FILE_IN + fin = "infile" + content = "IN FILE CONTENT" + with open(fin, "w") as f: + f.write(content) + res = file_in(fin) + res = compss_wait_on(res) + assert res == content, "strings are not equal: {}, {}".format(res, content) + + # Check if file exists: + file_checker(fin, "IN") + # Remove object + compss_delete_object(res) + + # Test Multiple FILE_IN + fin_1 = "infile_1" + fin_2 = "infile_2" + fin_3 = "infile_3" + fins = [fin_1, fin_2, fin_3] + content = "IN FILE CONTENT" + results = [] + for fin in fins: + with open(fin, "w") as f: + f.write(content) + results.append(file_in(fin)) + results = compss_wait_on(results) + for res in results: + assert res == content, "strings are not equal: {}, {}".format( + res, content + ) + + # Check if file exists: + multiple_file_checker(fins, "IN") + # Remove objects + compss_delete_object(*results) + + # Test FILE_INOUT + finout = "inoutfile" + content = "INOUT FILE CONTENT" + with open(finout, "w") as f: + f.write(content) + res = file_inout(finout) + res = compss_wait_on(res) + compss_wait_on_file(finout) + with compss_open(finout, "r") as finout_r: + content_r = finout_r.read() + content += "\n===> INOUT FILE ADDED CONTENT" + assert res == content, "strings are not equal: {}, {}".format(res, content) + assert content_r == content, "strings are not equal: {}, {}".format( + content_r, content + ) + + # Check if file exists: + file_checker(finout, "INOUT") + # Remove object + compss_delete_object(res) + + # Test Multiple FILE_INOUT + finout_1 = "inoutfile_1" + finout_2 = "inoutfile_2" + finout_3 = "inoutfile_3" + finouts = [finout_1, finout_2, finout_3] + content = "INOUT FILE CONTENT" + results = [] + for finout in finouts: + with open(finout, "w") as f: + f.write(content) + results.append(file_inout(finout)) + results = compss_wait_on(results) + compss_wait_on_file(*finouts) + i = 0 + content += "\n===> INOUT FILE ADDED CONTENT" + for finout in finouts: + with compss_open(finout, "r") as finout_r: + content_r = finout_r.read() + assert results[i] == content, "strings are not equal: {}, {}".format( + results[i], content + ) + assert content_r == content, "strings are not equal: {}, {}".format( + content_r, content + ) + i += 1 + + # Check if file exists: + multiple_file_checker(finouts, "INOUT") + # Remove objects + compss_delete_object(*results) + + # Test FILE_OUT + fout = "outfile" + content = "OUT FILE CONTENT" + res = file_out(fout, content) + res = compss_wait_on(res) + compss_wait_on_file(fout) + with compss_open(fout, "r") as fout_r: + content_r = fout_r.read() + # The final file is only stored after the execution. + # During the execution, you have to use the compss_open, which will + # provide the real file where the output file is. + # fileInFolder = os.path.exists(fout) + # assert fileInFolder is True, "FILE_OUT is not in the final location" + assert res == content, "strings are not equal: {}, {}".format(res, content) + assert content_r == content, "strings are not equal: {}, {}".format( + content_r, content + ) + + # Check if file exists: + file_checker(fout, "OUT") + # Remove object + compss_delete_object(res) + + # Test Multiple FILE_OUT + fout_1 = "outfile_1" + fout_2 = "outfile_2" + fout_3 = "outfile_3" + fouts = [fout_1, fout_2, fout_3] + content = "OUT FILE CONTENT" + results = [] + for fout in fouts: + results.append(file_out(fout, content)) + results = compss_wait_on(results) + compss_wait_on_file(*fouts) + i = 0 + for fout in fouts: + with compss_open(fout, "r") as fout_r: + content_r = fout_r.read() + # The final file is only stored after the execution. + # During the execution, you have to use the compss_open, which will + # provide the real file where the output file is. + # fileInFolder = os.path.exists(fout) + # assert fileInFolder is True, "FILE_OUT is not in the final location" + assert results[i] == content, "strings are not equal: {}, {}".format( + results[i], content + ) + assert content_r == content, "strings are not equal: {}, {}".format( + content_r, content + ) + i += 1 + + # Check if file exists: + file_checker(fout, "OUT") + # Remove object + compss_delete_object(*results) + + +@task(dir_inout=DIRECTORY_INOUT, returns=list) +def dir_inout_task(dir_inout, i): + """Write to an INOUT directory. + + :param dir_inout: Directory with INOUT direction. + :param i: Value to be written in the file within the directory. + :returns: None. + """ + res = list() + for _ in os.listdir(dir_inout): + with open("{}{}{}".format(dir_inout, os.sep, _), "r") as fd: + res.append(fd.read()) + f_inout = "{}{}{}".format(dir_inout, os.sep, i) + with open(f_inout, "w") as fd: + fd.write("written by inout task #" + str(i)) + return res + + +@task(dir_in=DIRECTORY_IN) +def dir_in_task(dir_in): + """Read all files within the given directory. + + :param dir_in: Source directory. + :returns: List with the contents per file. + """ + res = list() + for _ in os.listdir(dir_in): + _fp = dir_in + os.sep + _ + with open(_fp, "r") as fd: + res.append(fd.read()) + return res + + +@task(dir_out=DIRECTORY_OUT) +def dir_out_task(dir_out, i): + """Write to an OUT directory. + + :param dir_out: Directory with OUT direction. + :param i: Value to be written in the file within the directory. + :returns: None. + """ + if os.path.exists(dir_out): + shutil.rmtree(dir_out) + os.mkdir(dir_out) + f_out = "{}{}{}".format(dir_out, os.sep, i) + with open(f_out, "w") as fd: + fd.write("written in dir out #{}".format(i)) + + +def directories(): + """Check directories functionalities. + + :returns: None. + """ + cur_path = "{}{}".format(os.getcwd(), os.sep) + dir_t = "{}{}".format(cur_path, "some_dir_t") + dir_t_1 = "{}{}".format(cur_path, "some_dir_t_1") + dir_t_2 = "{}{}".format(cur_path, "some_dir_t_2") + dir_t_3 = "{}{}".format(cur_path, "some_dir_t_3") + dir_ts = [dir_t_1, dir_t_2, dir_t_3] + if os.path.exists(dir_t): + shutil.rmtree(dir_t) + os.mkdir(dir_t) + for dir_t_elem in dir_ts: + if os.path.exists(dir_t_elem): + shutil.rmtree(dir_t_elem) + os.mkdir(dir_t_elem) + + # len(phase[i] = i) + res_phase_0 = [] + for i in range(0, 5, 1): + res_phase_0.append(dir_inout_task(dir_t, i)) + + res_multiple_dirs = [] + value = 11 + for dir_t_elem in dir_ts: + res_multiple_dirs.append(dir_inout_task(dir_t_elem, value)) + value += 11 + + # len(phase[i] = 5) + res_phase_1 = [] + for i in range(0, 5, 1): + res_phase_1.append(dir_in_task(dir_t)) + + # len(phase[i] = i + 5) + res_phase_2 = [] + for i in range(5, 10, 1): + res_phase_2.append(dir_inout_task(dir_t, i)) + + # len(phase[i] = 10) + res_phase_3 = [] + for i in range(0, 5, 1): + res_phase_3.append(dir_in_task(dir_t)) + + # dir out should contain only the last file + for i in range(0, 15, 1): + dir_out_task(dir_t, i) + + res_phase_0 = compss_wait_on(res_phase_0) + res_phase_1 = compss_wait_on(res_phase_1) + res_phase_2 = compss_wait_on(res_phase_2) + res_phase_3 = compss_wait_on(res_phase_3) + compss_wait_on_directory(dir_t) + res_multiple_dirs = compss_wait_on(res_multiple_dirs) + compss_wait_on_directory(*dir_ts) + + for i, res in enumerate(res_phase_0): + assert len(res) == i, "ERROR in task #{} of phase 0: {} != {}".format( + i, len(res), i + ) + + for res in res_multiple_dirs: + assert ( + len(res) == 0 + ), "ERROR in task of phase 0 multiple: {} != {}".format(len(res), 0) + + for i, res in enumerate(res_phase_1): + assert len(res) == 5, "ERROR in task #{} of phase 1: {} != 5".format( + i, len(res) + ) + + for i, res in enumerate(res_phase_2): + assert ( + len(res) == i + 5 + ), "ERROR in task #{} of phase 2: {} != {}".format(i, len(res), i + 5) + + for i, res in enumerate(res_phase_3): + assert len(res) == 10, "ERROR in task #{} of phase 3: {} != 10".format( + i, len(res) + ) + + time.sleep(3) # TODO: Why it is needed a sleep to find the directory? + assert 1 == len( + os.listdir(dir_t) + ), "Directory has fewer or more files than 1: {}".format( + len(os.listdir(dir_t)) + ) + shutil.rmtree(dir_t) + compressed_dir = "some_dir_t.zip" + if os.path.exists(compressed_dir): + os.remove(compressed_dir) + + i = 1 + for dir_t in dir_ts: + assert 1 == len( + os.listdir(dir_t) + ), "Directory has fewer or more files than 1: {}".format( + len(os.listdir(dir_t)) + ) + shutil.rmtree(dir_t) + compressed_dir = "some_dir_t_" + str(i) + ".zip" + if os.path.exists(compressed_dir): + os.remove(compressed_dir) + i += 1 + + +@task(returns=1) +def increment(value): + """Increment the given value with 1. + + :param value: Integer value. + :returns: value incremented with 1. + """ + return value + 1 + + +def test_task_groups(): + """Check task groups functionalities. + + :returns: None. + """ + num_tasks = 3 + num_groups = 3 + results = [] + with TaskGroup("bigGroup", True): + # Inside a big group, more groups are created + for i in range(num_groups): + with TaskGroup("group" + str(i), False): + for j in range(num_tasks): + results.append(increment(i)) + + # Barrier for groups + for i in range(num_groups): + compss_barrier_group("group" + str(i)) + + +def main(): + """Check all API functionalities. + + :returns: None. + """ + files() + compss_barrier() + directories() + compss_barrier() + test_task_groups() + compss_barrier() + + +# Uncomment for command line check: +# if __name__ == "__main__": +# main() diff --git a/examples/dds/pycompss/tests/integration/launch/resources/app_collection.py b/examples/dds/pycompss/tests/integration/launch/resources/app_collection.py new file mode 100644 index 00000000..5b44c363 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/app_collection.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench for Collections.""" + +from pycompss.api.api import compss_wait_on +from pycompss.api.parameter import COLLECTION +from pycompss.api.parameter import COLLECTION_IN +from pycompss.api.parameter import COLLECTION_INOUT +from pycompss.api.parameter import DICTIONARY_IN +from pycompss.api.parameter import DICTIONARY_INOUT +from pycompss.api.reduction import reduction +from pycompss.api.task import task + + +class Polygon(object): + """Polygon class.""" + + def __init__(self, sides): + """Create a new Polygon with the given sides. + + :param sides: Number of sides. + :returns: None. + """ + self.sides = sides + + def increment(self, amount): + """Increment the number of sides. + + :param amount: Number of sides to be incremented. + :returns: None + """ + self.sides += amount + + def get_sides(self): + """Retrieve the number of sides. + + :returns: The number of sides. + """ + return self.sides + + +# COLLECTIONS + + +def generate_collection(value): + """Populate the given list with three polygons. + + :param value: List to be populated with three polygons. + :returns: None. + """ + value.append(Polygon(2)) + value.append(Polygon(10)) + value.append(Polygon(20)) + + +@task(value=COLLECTION_INOUT) +def update_collection(value): + """Update the number of sides of all polygons in the given collection. + + :param value: List of polygons. + :returns: None. + """ + for c in value: + c.increment(1) + + +@task(returns=1, value=COLLECTION_IN) +def sum_all_sides(value): + """Sum all sides from all polygons from the given collection. + + :param value: List of polygons. + :returns: Total number of sides of all polygons. + """ + result = 0 + for c in value: + result += c.get_sides() + return result + + +# DICTIONARY COLLECTIONS + + +def generate_dictionary(value): + """Populate the given dictionary with three polygons. + + :param value: Dictionary to be populated with three polygons. + :returns: None. + """ + value["a"] = Polygon(3) + value["b"] = Polygon(10) + value["c"] = Polygon(20) + + +@task(value=DICTIONARY_INOUT) +def update_dictionary(value): + """Update the number of sides of all polygons in the given dictionary. + + :param value: Dictionary of polygons. + :returns: None. + """ + for key in value.keys(): + value[key].increment(1) + + +@task(returns=2, value=DICTIONARY_IN) +def sum_all_sides_of_dictionary(value): + """Sum all sides from all polygons from the given dictionary. + + :param value: Dictionary of polygons. + :returns: Total number of sides of all polygons. + """ + keys = "" + result = 0 + for k, v in value.items(): + keys += k + result += v.get_sides() + return keys, result + + +# REDUCE WITH COLLECTIONS + + +@reduction(chunk_size="2") +@task(returns=1, col=COLLECTION_IN) +def my_reduction(col): + """Accumulate all elements in the given collection. + + :param col: Collection of integers. + :returns: Accumulated value of all integers in the given collection. + """ + r = 0 + for i in col: + r += i + return r + + +@task(returns=1) +def increment(v): + """Increment the given value with 1. + + :param v: Integer to increment. + :returns: Incremented value with 1. + """ + return v + 1 + + +@task(returns=COLLECTION) +def generate_collection_return(): + """Create a return collection with three polygons. + + :returns: A collection with three polygons. + """ + value = [Polygon(2), Polygon(10), Polygon(20)] + return value + + +def main(): + """Execute all collection tests. + + :returns: None. + """ + initial = [] + generate_collection(initial) + update_collection(initial) + result = sum_all_sides(initial) + result = compss_wait_on(result) + assert result == 35, "ERROR: Unexpected result (%s != 35)." % str(result) + + initial = {} + generate_dictionary(initial) + update_dictionary(initial) + keys, result = sum_all_sides_of_dictionary(initial) + keys = compss_wait_on(keys) + result = compss_wait_on(result) + assert ( + len(keys) == 3 and "a" in keys and "b" in keys and "c" in keys + ), "ERROR: Unexpected keys (%s != abc (in any order))." % str(keys) + assert result == 36, "ERROR: Unexpected result (%s != 36)." % str(result) + + # Reduction + num_tasks = 5 + a = [x for x in range(1, num_tasks + 1)] + result = [] + for element in a: + result.append(increment(element)) + # Reduction task + final = my_reduction(result) + final = compss_wait_on(final) + assert final == 20, "ERROR: Unexpected result (%s != 20)." % str(result) + + # Collection return + result = generate_collection_return() + results = compss_wait_on(result) + assert ( + len(results) == 3 + ), "ERROR: The generated collection does not have the expected length." # noqa: E501 + assert ( + results[0].get_sides() == 2 + and results[1].get_sides() == 10 + and results[2].get_sides() == 20 + ), "ERROR: The collection contents are not as expected" # noqa: E501 + + +# Uncomment for command line check: +# if __name__ == '__main__': +# main() diff --git a/examples/dds/pycompss/tests/integration/launch/resources/functions.py b/examples/dds/pycompss/tests/integration/launch/resources/functions.py new file mode 100644 index 00000000..38a39449 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/functions.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench for functions.""" + +from pycompss.functions.data import generator + + +def check_generator(): + """Check the generator function. + + :returns: None. + """ + random_data = generator((12, 12), 4, 5, "random", True) + normal_data = generator((12, 12), 4, 5, "normal", True) + uniform_data = generator((12, 12), 4, 5, "uniform", True) + + assert ( + random_data != normal_data != uniform_data + ), "The generator did not produce different data for different distributions" # noqa: E501 + + +def main(): + """Execute all function tests. + + :returns: None. + """ + check_generator() + + +# Uncomment for command line check: +# if __name__ == "__main__": +# main() diff --git a/examples/dds/pycompss/tests/integration/launch/resources/increment.py b/examples/dds/pycompss/tests/integration/launch/resources/increment.py new file mode 100644 index 00000000..9f45eac6 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/increment.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench increment functions.""" + +from pycompss.api.api import compss_wait_on +from pycompss.api.constraint import constraint +from pycompss.api.implement import implement +from pycompss.api.local import local +from pycompss.api.task import task + + +@implement(source_class="increment", method="increment") +@constraint(computing_units=2) +@task(returns=1) +def super_increment(value): + """Increment the given value with 1 (implementation with 2 CUs). + + :param value: Integer to increment. + :returns: Incremented value with 1. + """ + return value + 1 + + +@task(returns=1) +def increment(value): + """Increment the given value with 1. + + :param value: Integer to increment. + :returns: Incremented value with 1. + """ + return value + 1 + + +@constraint(computing_units=1) +@task(returns=1) +def decrement(value): + """Decrement the given value with 1. + + :param value: Integer to decrement. + :returns: Decremented value with 1. + """ + return value - 1 + + +@local +def power(value, **kwarg): + """Power the given value and multiply with kwarg["param"]. + + :param value: Integer to multiply. + :param kwarg: Dictionary containing "param" value. + :returns: The power of the given value multiplied by kwarg["param"]. + """ + return value * value * kwarg["param"] + + +def main(): + """Execute all increment/decrement/power functions. + + :returns: None. + """ + initial = 1 + partial = increment(initial) + result = decrement(partial) + result = compss_wait_on(result) + assert result == initial, "ERROR: Unexpected increment result." + local_result = power(partial, param=partial) + assert local_result == 8, "ERROR: Unexpected local result." + + +# Uncomment for command line check: +# if __name__ == '__main__': +# main() diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage/Object.py b/examples/dds/pycompss/tests/integration/launch/resources/storage/Object.py new file mode 100644 index 00000000..6a47b90b --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage/Object.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +Dummy SCO class. + +WARNING! Only for testing purposes. + Considers the persistence within the /tmp folder of the localhost. +""" + +from pycompss.tests.integration.launch.resources.storage import api + + +class SCO(object): + """Self-contained object representative.""" + + id = None + alias = None + + def __init__(self): + """Do nothing constructor.""" + pass + + # Functionality Not Supported! use getByName instead. + # def __init__(self, alias): + # self.alias = alias + + def get_id(self): + """Retrieve the object identifier. + + :returns: Object identifier. + """ + return self.id + + def set_id(self, id): # noqa + """Set the object identifier. + + :param id: Object identifier. + :returns: None. + """ + self.id = id + + def make_persistent(self, *args): + """Persist the object. + + :param args: Arguments redirected to api.makePersistent. + :returns: None. + """ + api.makePersistent(self, *args) + + def delete_persistent(self): + """Delete persistent object. + + :returns: None. + """ + api.removeById(self) # noqa + + def update_persistent(self): + """Update persistent object. + + :returns: None. + """ + api.updatePersistent(self) + + # Renaming + getID = get_id # noqa: N815 + setID = set_id # noqa: N815 + makePersistent = make_persistent # noqa: N815 + deletePersistent = delete_persistent # noqa: N815 + updatePersistent = update_persistent # noqa: N815 diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage/__init__.py b/examples/dds/pycompss/tests/integration/launch/resources/storage/__init__.py new file mode 100644 index 00000000..4f817328 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the launch integration tests storage resources.""" diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage/api.py b/examples/dds/pycompss/tests/integration/launch/resources/storage/api.py new file mode 100644 index 00000000..a9f8a8fe --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage/api.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +Storage dummy connector. + +This file contains the functions that any storage that wants to be used +with PyCOMPSs must implement. + +storage.api code example. +""" + +import os +import socket +import uuid + +from pycompss.tests.outlog import get_logger +from pycompss.util.serialization.serializer import deserialize_from_file +from pycompss.util.serialization.serializer import serialize_to_file +from pycompss.util.typing_helper import typing + +STORAGE_PATH = "/tmp/PSCO/" + str(socket.gethostname()) + "/" # NOSONAR +LOGGER = get_logger() + + +def init(config_file_path=None, **kwargs): # noqa + """Initialize dummy storage. + + :param config_file_path: Storage conf. + :param kwargs: Other arguments. + :return: None. + """ + # print("-----------------------------------------------------") + # print("| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |") + # print("| Call to: init function. |") + # print("| Parameters: config_file_path = None") + # for key in kwargs: + # print("| Kwargs: Key %s - Value %s" % (key, kwargs[key])) + # print("-----------------------------------------------------") + pass + + +def finish(**kwargs: dict) -> None: + """Finalize dummy storage. + + :param kwargs: Other arguments. + :return: None. + """ + # print("-----------------------------------------------------") + # print("| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |") + # print("| Call to: finish function. |") + # for key in kwargs: + # print("| Kwargs: Key %s - Value %s" % (key, kwargs[key])) + # print("-----------------------------------------------------") + pass + + +def init_worker(config_file_path: str = None, **kwargs: dict) -> None: + """Initialize dummy storage at worker. + + :param config_file_path: Storage conf. + :param kwargs: Other arguments. + :return: None. + """ + # print("-----------------------------------------------------") + # print("| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |") + # print("| Call to: init Worker function. |") + # print("| Parameters: config_file_path = None") + # for key in kwargs: + # print("| Kwargs: Key %s - Value %s" % (key, kwargs[key])) + # print("-----------------------------------------------------") + pass + + +def finish_worker(**kwargs: dict) -> None: + """Finalize dummy storage at worker. + + :param kwargs: Other arguments. + :return: None. + """ + # print("-----------------------------------------------------") + # print("| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |") + # print("| Call to: finish Worker function. |") + # for key in kwargs: + # print("| Kwargs: Key %s - Value %s" % (key, kwargs[key])) + # print("-----------------------------------------------------") + pass + + +def get_by_id(id: str) -> typing.Any: + """Retrieve an object from an external storage. + + This dummy returns the same object as submitted by the parameter obj. + + :param id: Key of the object to be retrieved. + :return: The real object. + """ + # # Warning message: + # print "-----------------------------------------------------" + # print "| WARNING!!! - YOU ARE USING THE DUMMY STORAGE API. |" + # print "| Call to: get_by_id |" + # print "| ********************************************* |" + # print "| *** Check that you really want to use the *** |" + # print "| ************* dummy storage api ************* |" + # print "| ********************************************* |" + # print "-----------------------------------------------------" + if id is not None: + try: + file_name = id + ".PSCO" + file_path = STORAGE_PATH + file_name + obj = deserialize_from_file(file_path, logger=LOGGER) + obj.setID(id) # noqa + return obj + except ValueError: + # The id does not complain uuid4 --> raise an exception + print( + "Error: the ID for get_by_id does not complain the " + "uuid4 format." + ) + raise ValueError( + "Using the dummy storage API get_by_id with wrong id." + ) + else: + # Using a None id --> raise an exception + print("Error: the ID for get_by_id is None.") + raise ValueError("Using the dummy storage API get_by_id with None id.") + + +def make_persistent(obj: typing.Any, *args: dict) -> None: + """Persist the given object. + + :param obj: object to persist. + :param args: Extra arguments. + :return: None. + """ + if obj.id is None: + if len(args) == 0: + # The user has not indicated the id + uid = uuid.uuid4() + elif len(args) == 1: + # The user has indicated the id + uid = args[0] + else: + raise ValueError("Too many arguments when calling makePersistent.") + obj.id = str(uid) + # Write ID file + file_name = str(uid) + ".ID" + file_path = STORAGE_PATH + file_name + print("MAKE PERSISTENT: Creating ID file " + file_path) + with open(file_path, "w") as f: + f.write(obj.id) + + # Write PSCO file + file_name = str(uid) + ".PSCO" + file_path = STORAGE_PATH + file_name + print("MAKE PERSISTENT: Serializing object to file " + file_path) + serialize_to_file(obj, file_path, logger=LOGGER) + else: + # The obj is already persistent + pass + + +def update_persistent(obj: typing.Any, *args: dict) -> None: + """Update the given object. + + :param obj: object to update. + :param args: Extra arguments. + :return: None. + """ + if obj.id is not None: + # The psco is already persistent + # Update PSCO file + file_name = str(obj.id) + ".PSCO" + file_path = STORAGE_PATH + file_name + # Remove old file + os.remove(file_path) + # Create a new one + serialize_to_file(obj, file_path, logger=LOGGER) + else: + # The obj is not persistent + pass + + +def remove_by_id(obj: str) -> None: + """Remove the given object. + + :param obj: Object to remove. + :return: None. + """ + if obj.id is not None: + # Remove ID file from /tmp/PSCO + file_name = str(obj.id) + ".ID" + file_path = STORAGE_PATH + file_name + try: + print("Removing ID file: " + file_path) + os.remove(file_path) + except OSError: + print("PSCO: " + file_path + " Does not exist!") + + # Remove PSCO file from /tmp/PSCO + file_name = str(obj.id) + ".PSCO" + file_path = STORAGE_PATH + file_name + try: + print("Removing Object file: " + file_path) + os.remove(file_path) + obj.id = None + except OSError: + print("PSCO: " + file_path + " Does not exist!") + else: + # The obj is not persistent yet + pass + + +class TaskContext(object): + """Dummy task context.""" + + def __init__(self, logger, values, config_file_path=None): + """Do nothing init. + + :param logger: Logger. + :param values: Values. + :param config_file_path: Configuration file path. + """ + self.logger = logger + self.values = values + self.config_file_path = config_file_path + + def __enter__(self): + """Do nothing at enter. + + :returns: None. + """ + # Do something prolog + # Ready to start the task + self.logger.info("Prolog finished") + + def __exit__(self, type, value, traceback): + """Do nothing at exit. + + :returns: None. + """ + # Do something epilog + # Finished + self.logger.info("Epilog finished") + + +# Renaming +initWorker = init_worker # noqa: N816 +finishWorker = finish_worker # noqa: N816 +getByID = get_by_id # noqa: N816 +makePersistent = make_persistent # noqa: N816 +updatePersistent = update_persistent # noqa: N816 +removeById = remove_by_id # noqa: N816 diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage_app/__init__.py b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/__init__.py new file mode 100644 index 00000000..8909d57d --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the launch integration tests storage app resources.""" diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage_app/models.py b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/models.py new file mode 100644 index 00000000..1b54b3c3 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/models.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench PSCO Models.""" + +import socket + +from pycompss.api.parameter import INOUT +from pycompss.api.task import task +from pycompss.tests.integration.launch.resources.storage.Object import SCO +from pycompss.tests.outlog import get_logger +from pycompss.util.serialization.serializer import serialize_to_file + +LOGGER = get_logger() + + +def update_file(obj): + """Update the object in its file (serialized). + + :param obj: Object to update. + :returns: None + """ + if obj.getID() is not None: + storage_path = ( + "/tmp/PSCO/" + str(socket.gethostname()) + "/" + ) # NOSONAR + serialize_to_file( + obj, storage_path + obj.getID() + ".PSCO", logger=LOGGER + ) + + +class MySO(SCO): + """Storage object. + + @ClassField value int + """ + + # For simple PSCO test + value = 0 + + def __init__(self, v): # noqa + """Create a MySO object containing an integer value. + + :param v: Integer value to be contained. + :returns: None. + """ + self.value = v + + def get(self): + """Retrieve the inner value. + + :returns: The inner value. + """ + return self.value + + def put(self, v): + """Put an inner value. + + :param v: Integer value to be contained. + :returns: None. + """ + self.value = v + update_file(self) + + @task(target_direction=INOUT) + def increment(self): + """Increment the inner value. + + :returns: None. + """ + self.value += 1 + self.updatePersistent() + + +class Words(SCO): + """Words storage object. + + @ClassField text dict <, word_info:str> + """ + + # For Wordcount Test + text = "" + + def __init__(self, t): # noqa + """Create a Words object containing the given text. + + :param t: Text to be contained. + :returns: None. + """ + self.text = t + + def get(self): + """Retrieve the inner text. + + :returns: The inner text. + """ + return self.text + + +class Result(SCO): + """Result storage object. + + @ClassField myd dict <,instances:atomicint> + """ + + myd = {} + + def get(self): + """Retrieve the inner result. + + :returns: The inner result. + """ + return self.myd + + def set(self, d): + """Set the inner result. + + :param d: Result to be contained. + :returns: None. + """ + self.myd = d + update_file(self) + + +# For Tiramisu mockup test + + +class InputData(SCO): + """Input data storage object. + + @ClassField images dict <, value:list> + """ + + images = {} + + def get(self): + """Retrieve the inner input data. + + :returns: The inner input data. + """ + return self.images + + def set(self, i): + """Set the inner input data. + + :param i: Data to be contained. + :returns: None. + """ + self.images = i + update_file(self) diff --git a/examples/dds/pycompss/tests/integration/launch/resources/storage_app/pscos.py b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/pscos.py new file mode 100644 index 00000000..b7d414dd --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/storage_app/pscos.py @@ -0,0 +1,654 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench for PSCOs.""" + +from pycompss.api.api import compss_wait_on +from pycompss.api.parameter import INOUT +from pycompss.api.parameter import OUT +from pycompss.api.task import task +from pycompss.tests.integration.launch.resources.storage_app.models import ( + InputData, +) +from pycompss.tests.integration.launch.resources.storage_app.models import MySO +from pycompss.tests.integration.launch.resources.storage_app.models import ( + Result, +) +from pycompss.tests.integration.launch.resources.storage_app.models import ( + Words, +) + +GENERIC_STRING = "This is a test" +FIRST = "first" +SECOND = "second" +THIRD = "third" +FOURTH = "fourth" +SCO_GET = "sco.get = " + + +@task(returns=int) +def simple_task(r): + """Get r value and it by itself. + + :param r: Storage object to multiply. + :returns: r value * r value + """ + val = r.get() + print("R: ", r) + print("R.get = ", val) + return val * val + + +@task(returns=MySO) +def complex_task(p): + """Persist p with the value multiplied by 10. + + :param p: Storage object to multiply and persist. + :returns: r value * r value + """ + v = p.get() + print("P: ", p) + print("P.get: ", v) + q = MySO(v * 10) + q.makePersistent() + print("Q: ", q) + print("Q.get = ", q.get()) + return q + + +@task(returns=dict) +def wordcount_task(words): + """Word count task. + + :param words: String. + :returns: Dictionary with the number of appearances per word. + """ + partial_result = {} + data = words.get() + for entry in data.split(): + if entry in partial_result: + partial_result[entry] += 1 + else: + partial_result[entry] = 1 + return partial_result + + +@task(dic1=INOUT) +def reduce_task(dic1, dic2): + """Word count reduction task. + + CAUTION! Modifies dic1. + + :param dic1: First partial dictionary. + :param dic2: Second partial dictionary. + :returns: None. + """ + for k in dic2: + if k in dic1: + dic1[k] += dic2[k] + else: + dic1[k] = dic2[k] + + +@task(final=INOUT) +def reduce_task_pscos(final, dic2): + """Word count reduction task with persistent objects. + + CAUTION! Modifies final. + + :param final: First partial dictionary. + :param dic2: Second partial dictionary. + :returns: None. + """ + dic1 = final.get() + for k in dic2: + if k in dic1: + dic1[k] += dic2[k] + else: + dic1[k] = dic2[k] + final.set(dic1) + + +def basic_test() -> bool: + """Test basic functionalities. + + This Test: + - Instantiates a SCO. + - Makes it persistent. + - Calls a task with that PSCO -> IN parameter. + - Waits for the task result. + - Deletes the PSCO. + + :return: True if success. False otherwise. + """ + o = MySO(10) + print("BEFORE MAKEPERSISTENT: o.id: ", o.getID()) + # Persist the object to disk (it will be at /tmp/uuid.PSCO) + o.makePersistent() + print("AFTER MAKEPERSISTENT: o.id: ", o.getID()) + v = simple_task(o) + v1 = compss_wait_on(v) + # Remove the persisted object from disk (from /tmp/uuid.PSCO) + o.deletePersistent() + if v1 == 100: + print("- Simple Test Python PSCOs: OK") + return True + else: + print("- Simple Test Python PSCOs: ERROR") + return False + + +def basic_2_test() -> bool: + """Test more basic functionalities. + + This Test: + - Instantiates a SCO. + - Makes it persistent. + - Calls a task with that PSCO -> IN parameter. + - The task receives the PSCO and gets its value. + - Instantiates another SCO and makes it persistent within the task. + - Returns the new PSCO. + - Calls another task with the input from the first one. + - Gets the output PSCO and receives it as input. + - Waits for the first task result. + - Waits for the second task result. + + :return: True if success. False otherwise. + """ + p = MySO(1) + p.makePersistent() + x = complex_task(p) + y = simple_task(x) + res1 = compss_wait_on(x) + res2 = compss_wait_on(y) + if p.get() == 1 and res1.get() == 10 and res2 == 100: + print("- Complex Test Python PSCOs: OK") + return True + else: + print("- Complex Test Python PSCOs: ERROR") + return False + + +def basic_3_test() -> bool: + """Test even more basic functionalities. + + This Test: + - Instantiates a SCO. + - Makes it persistent. + - Calls a task within that PSCO -> self INOUT + - The task increments its value. + - Calls it again. + - Calls it again. + - Waits for the self result. + + :return: True if success. False otherwise. + """ + p = MySO(1) + p.makePersistent() + p.increment() + p.increment() + p.increment() + p = compss_wait_on(p) + print("p: " + str(p.get())) + if p.get() == 4: + print("- Complex Test Python PSCOs: OK") + return True + else: + print("- Complex Test Python PSCOs: ERROR") + return False + + +def wordcount() -> bool: + """Wordcount Test. + + - Wordcount task receives a PSCO and returns a dictionary. + - Reduce task works with python dictionaries. + + :return: True if success. False otherwise. + """ + words = [ + Words(GENERIC_STRING), + Words(GENERIC_STRING), + Words(GENERIC_STRING), + Words(GENERIC_STRING), + ] + for w in words: + w.makePersistent() + result = {} + + for w in words: + partial_results = wordcount_task(w) + reduce_task(result, partial_results) + result = compss_wait_on(result) + + if ( + result["This"] == 4 + and result["is"] == 4 + and result["a"] == 4 + and result["test"] == 4 + ): + print("- Python Wordcount 1 with PSCOs: OK") + return True + else: + print("- Python Wordcount 1 with PSCOs: ERROR") + return False + + +def wordcount2() -> bool: + """Wordcount Test. + + - Wordcount task receives a PSCO and returns a dictionary. + - Reduce task receives a INOUT PSCO (result) where accumulates + the partial results. + + :return: True if success. False otherwise. + """ + words = [ + Words(GENERIC_STRING), + Words(GENERIC_STRING), + Words(GENERIC_STRING), + Words(GENERIC_STRING), + ] + for w in words: + w.makePersistent() + result = Result() + result.makePersistent() + + for w in words: + partial_results = wordcount_task(w) + reduce_task_pscos(result, partial_results) + + final = compss_wait_on(result) + + print(final.myd) + result = final.get() + + if ( + result["This"] == 4 + and result["is"] == 4 + and result["a"] == 4 + and result["test"] == 4 + ): + print("- Python Wordcount 2 with PSCOs: OK") + return True + else: + print("- Python Wordcount 2 with PSCOs: ERROR") + return False + + +@task(o2=INOUT) +def transform1(o1, o2): + """Do first transformation (INOUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: None. + """ + pow2 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value * value) + pow2[k] = lst + o2.set(pow2) + print("Function: Pow 2.") + print("Transformation 1 result in o2: ", o2.get()) + + +@task(o2=INOUT) +def transform2(o2, o1): + """Do second transformation (INOUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: None. + """ + add1 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value + 1) + add1[k] = lst + o2.set(add1) + print("Function: Add 1.") + print("Transformation 2 result in o2: ", o2.get()) + + +@task(returns=InputData, o2=INOUT) +def transform3(o1, o2): + """Do third transformation (returns and INOUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: Updated o2. + """ + mult3 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value * 3) + mult3[k] = lst + o2.set(mult3) + print("Function: Multiply per 3.") + print("Transformation 3 result in o2: ", o2.get()) + return o2 + + +def tiramisu_mockup() -> bool: + """Tiramisu Mockup Test. + + :return: True if success. False otherwise. + """ + my_obj = InputData() + my_obj.set( + { + FIRST: [1, 1, 1, 1], + SECOND: [2, 2, 2, 2], + THIRD: [3, 3, 3, 3], + FOURTH: [4, 4, 4, 4], + } + ) + out1 = InputData() + out2 = InputData() + out3 = InputData() + my_obj.makePersistent() + out1.makePersistent() + out2.makePersistent() + out3.makePersistent() + + transform1(my_obj, out1) + transform2(out2, out1) # INOUT in first position + result = transform3(out2, out3) + + result = compss_wait_on(result) + out1 = compss_wait_on(out1) + out2 = compss_wait_on(out2) + out3 = compss_wait_on(out3) + + print("INOUTS:") + return __check_transformations( + "Tiramisu Mockup", out1, out2, out3, result + ) # noqa + + +def __check_transformations( + transformation: str, + out1: InputData, + out2: InputData, + out3: InputData, + result: InputData, +) -> bool: + """Check the transformation to evaluate the result. + + :param transformation: Transformation name + :param out1: Transformation 1 output. + :param out2: Transformation 2 output. + :param out3: Transformation 3 output. + :param result: Result object. + :return: True if success. False otherwise. + """ + print("Transformation 1: ", out1.get()) + print("Transformation 2: ", out2.get()) + print("Transformation 3: ", out3.get()) + print("RESULT: ", result.get()) + + assert out1.get() == { + FIRST: [1, 1, 1, 1], + SECOND: [4, 4, 4, 4], + THIRD: [9, 9, 9, 9], + FOURTH: [16, 16, 16, 16], + } + assert out2.get() == { + FIRST: [2, 2, 2, 2], + SECOND: [5, 5, 5, 5], + THIRD: [10, 10, 10, 10], + FOURTH: [17, 17, 17, 17], + } + assert out3.get() == { + FIRST: [6, 6, 6, 6], + SECOND: [15, 15, 15, 15], + THIRD: [30, 30, 30, 30], + FOURTH: [51, 51, 51, 51], + } + + final_results = result.get() + message = "- Python " + transformation + " with PSCOs: " + if ( + all(x == 6 for x in final_results[FIRST]) + and all(x == 15 for x in final_results[SECOND]) + and all(x == 30 for x in final_results[THIRD]) + and all(x == 51 for x in final_results[FOURTH]) + ): + print(message + "OK") + return True + else: + print(message + "ERROR") + return False + + +@task(o2=OUT) +def transform1_2(o1, o2): + """Do first transformation second version (OUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: None. + """ + pow2 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value * value) + pow2[k] = lst + o2.set(pow2) + print("Function: Pow 2.") + print("Transformation 1 result in o2: ", o2.get()) + + +@task(o2=OUT) +def transform2_2(o2, o1): + """Do second transformation second version (OUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: None. + """ + add1 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value + 1) + add1[k] = lst + o2.set(add1) + print("Function: Add 1.") + print("Transformation 2 result in o2: ", o2.get()) + + +@task(returns=InputData, o2=OUT) +def transform3_2(o1, o2): + """Do third transformation second version (OUT). + + CAUTION! Modifies o2. + + :param o1: First persistent object. + :param o2: Second persistent object. + :results: None. + """ + mult3 = {} + images = o1.get() + for k, v in images.items(): + lst = [] + for value in v: + lst.append(value * 3) + mult3[k] = lst + o2.set(mult3) + print("Function: Multiply per 3.") + print("Transformation 3 result in o2: ", o2.get()) + return o2 + + +def tiramisu_mockup2() -> bool: + """Tiramisu Mockup Test 2. + + :return: True if success. False otherwise. + """ + my_obj = InputData() + my_obj.set( + { + FIRST: [1, 1, 1, 1], + SECOND: [2, 2, 2, 2], + THIRD: [3, 3, 3, 3], + FOURTH: [4, 4, 4, 4], + } + ) + out1 = InputData() + out2 = InputData() + out3 = InputData() + my_obj.makePersistent() + out1.makePersistent() + out2.makePersistent() + out3.makePersistent() + + transform1_2(my_obj, out1) + transform2_2(out2, out1) # OUT in first position + result = transform3_2(out2, out3) + + result = compss_wait_on(result) + + out1 = compss_wait_on(out1) + out2 = compss_wait_on(out2) + out3 = compss_wait_on(out3) + + print("OUTPUTS:") + return __check_transformations( + "Tiramisu Mockup 2", out1, out2, out3, result + ) # noqa + + +@task(sco=INOUT) +def make_persistent_in_task_as_parameter(sco): + """Make persistent the sco within the task as INOUT. + + :param sco: Non persisted persistent object. + :returns: None. + """ + print("sco: ", sco) + print(SCO_GET, sco.get()) + sco.put(27) + print("Update sco value:") + print(SCO_GET, sco.get()) + sco.makePersistent() + print("sco made persistent") + print("sco id: ", sco.getID()) + + +@task(returns=MySO) +def make_persistent_in_task_as_return(sco): + """Make persistent the sco within the task as return. + + :param sco: Non persisted persistent object. + :returns: None. + """ + print("sco: ", sco) + print(SCO_GET, sco.get()) + sco.put(37) + print("Update sco value:") + print(SCO_GET, sco.get()) + sco.makePersistent() + print("sco made persistent") + print("sco id: ", sco.getID()) + return sco + + +def evaluate_make_persistent_in_task() -> bool: + """Check sco persistence within task (as INOUT). + + This Test checks what happens when a not persisted persistent object + is passed an INOUT task parameter and made persistent within the task. + + :return: True if success. False otherwise. + """ + o = MySO(10) + make_persistent_in_task_as_parameter(o) + v = compss_wait_on(o) + o.deletePersistent() + if v.get() == 27: + print("- Persistence of PSCOS in task as INOUT parameter: OK") + return True + else: + print("- Persistence of PSCOS in task as INOUT parameter: ERROR") + return False + + +def evaluate_make_persistent_in_task2() -> bool: + """Check sco persistence within task (as return). + + This Test checks what happens when a not persisted persistent object + is passed as IN task parameter, made persistent within the task, and + returned. + + :return: True if success. False otherwise. + """ + o = MySO(10) + sco = make_persistent_in_task_as_return(o) + v = compss_wait_on(sco) + v.deletePersistent() + if v.get() == 37: + print("- Persistence of PSCOS in task as return: OK") + return True + else: + print("- Persistence of PSCOS in task as return: ERROR") + return False + + +def main(): + """Run all pscos tests. + + :returns: None. + """ + results = { + "basic": basic_test(), + "basic2": basic_2_test(), + "basic3": basic_3_test(), + "wordcount": wordcount(), + "wordcount2": wordcount2(), + "tiramisu": tiramisu_mockup(), + "tiramisu2": tiramisu_mockup2(), + "persistInTask": evaluate_make_persistent_in_task(), + "persistInTask2": evaluate_make_persistent_in_task2(), + } + + assert all( + x for x in list(results.values()) + ), "PSCOs TEST FINISHED WITH ERRORS: " + str(results) diff --git a/examples/dds/pycompss/tests/integration/launch/resources/synthetic.py b/examples/dds/pycompss/tests/integration/launch/resources/synthetic.py new file mode 100644 index 00000000..2d8c7506 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/resources/synthetic.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench synthetic functions.""" + +import os +import shutil +import tempfile + +from pycompss.api.api import compss_open +from pycompss.api.api import compss_wait_on_file +from pycompss.api.binary import binary +from pycompss.api.mpi import mpi +from pycompss.api.ompss import ompss +from pycompss.api.parameter import FILE_OUT_STDOUT +from pycompss.api.parameter import Type +from pycompss.api.task import task + +TEMPORARY_DIRECTORY = tempfile.mkdtemp() + + +@binary(binary="date", working_dir=TEMPORARY_DIRECTORY) +@task(result={Type: FILE_OUT_STDOUT}) +def check_binary(result): # noqa + pass + + +@mpi(binary="date", working_dir=TEMPORARY_DIRECTORY, runner="mpirun") +@task(result={Type: FILE_OUT_STDOUT}) +def check_mpi(result): # noqa + pass + + +@ompss(binary="date", working_dir=TEMPORARY_DIRECTORY) +@task(result={Type: FILE_OUT_STDOUT}) +def check_ompss(result): # noqa + pass + + +def check_decorators(): + """Check the binary, mpi and ompss decorators. + + :returns: None. + """ + binary_result = "binary_result.out" + mpi_result = "mpi_result.out" + ompss_result = "ompss_result.out" + check_binary(binary_result) + check_mpi(mpi_result) + check_ompss(ompss_result) + compss_wait_on_file(binary_result) + compss_wait_on_file(mpi_result) + compss_wait_on_file(ompss_result) + + binary_result_fd = compss_open(binary_result) + mpi_result_fd = compss_open(mpi_result) + ompss_result_fd = compss_open(ompss_result) + + binary_content = binary_result_fd.readlines() # noqa + mpi_content = mpi_result_fd.readlines() # noqa + ompss_content = ompss_result_fd.readlines() # noqa + + binary_result_fd.close() # noqa + mpi_result_fd.close() # noqa + ompss_result_fd.close() # noqa + + os.remove(binary_result) + os.remove(mpi_result) + os.remove(ompss_result) + + shutil.rmtree(TEMPORARY_DIRECTORY) + + assert len(binary_content) == 1 + assert len(mpi_content) == 1 + assert len(ompss_content) == 1 + + print(binary_content) + print(mpi_content) + print(ompss_content) + + +def main(): + """Execute all synthetic functionalities. + + :returns: None. + """ + check_decorators() + # add more to be tested + + +# Uncomment for command line check: +# if __name__ == "__main__": +# main() diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_0_basic1.py b/examples/dds/pycompss/tests/integration/launch/test_launch_0_basic1.py new file mode 100644 index 00000000..26180a4f --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_0_basic1.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys + + +def test_launch_test_0_basic1(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app_path = os.path.join( + current_path, + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "tests", + "sources", + "local", + "python", + "0_basic1", + "src", + ) + app = os.path.join(app_path, "test_mp.py") + sys.path.insert(0, app_path) + launch_pycompss_application( + app, "main_program", debug=True, app_name="test_0_basic1" + ) + sys.path.pop(0) + if os.path.exists("infile"): + os.remove("infile") + if os.path.exists("outfile"): + os.remove("outfile") + if os.path.exists("inoutfile"): + os.remove("inoutfile") diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_application.py b/examples/dds/pycompss/tests/integration/launch/test_launch_application.py new file mode 100644 index 00000000..ad6d429a --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_application.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + +APPLICATION_NAME = "increment" +APPLICATION = "increment.py" +FOLDER = "resources" +PACKAGE = "main" + + +def test_launch_increment(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, FOLDER, APPLICATION) + launch_pycompss_application( + app, PACKAGE, debug=True, trace=False, app_name=APPLICATION_NAME + ) + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, FOLDER, "api_tester.py") + launch_pycompss_application( + app, PACKAGE, debug=True, trace=False, app_name=APPLICATION_NAME + ) + + +# def test_launch_increment_with_cache(): +# if sys.version_info >= (3, 8): +# from pycompss.runtime.launch import launch_pycompss_application +# current_path = os.path.dirname(os.path.abspath(__file__)) +# app = os.path.join(current_path, "..", FOLDER, APPLICATION) +# launch_pycompss_application( +# app, PACKAGE, debug=True, trace=False, app_name=APPLICATION_NAME, +# worker_cache=True +# ) +# else: +# print("WARNING: Cache not tested because python version is not " +# ">= 3.8") +# +# +# def test_launch_increment_mpi_worker(): +# from pycompss.runtime.launch import launch_pycompss_application +# +# current_path = os.path.dirname(os.path.abspath(__file__)) +# app = os.path.join(current_path, "..", FOLDER, APPLICATION) +# launch_pycompss_application( +# app, PACKAGE, debug=True, trace=False, app_name=APPLICATION_NAME, +# mpi_worker=True +# ) diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_application_collection.py b/examples/dds/pycompss/tests/integration/launch/test_launch_application_collection.py new file mode 100644 index 00000000..5c0b68e2 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_application_collection.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "app_collection.py") + launch_pycompss_application(app, "main", debug=True, app_name="main") diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_application_storage.py b/examples/dds/pycompss/tests/integration/launch/test_launch_application_storage.py new file mode 100644 index 00000000..d58b6b01 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_application_storage.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import shutil +import subprocess +import sys + +STORAGE_CONF = "/tmp/storage.conf" # NOSONAR +TEMP_DIR = "/tmp/PSCO" # NOSONAR +JAVA_API_JAR = "" +STORAGE_API = None + + +def __initialize_storage() -> None: + """Initializes the dummy storage backend. + Compiles the dummy storage backend from the tests sources. + Sets the JAVA_API_JAR on the first call to this initialization. + + :return: None + """ + global JAVA_API_JAR + global STORAGE_API + current_path = os.path.dirname(os.path.abspath(__file__)) + # Add python storage api to sys.path + STORAGE_API = os.path.join(current_path, "resources") + sys.path.insert(0, STORAGE_API) + if JAVA_API_JAR == "": + # Compile jar + jar_source_path = os.path.join( + current_path, + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "..", + "utils", + "storage", + ) + compile_command = ["mvn", "clean", "package"] + process = subprocess.Popen( + compile_command, stdout=subprocess.PIPE, cwd=jar_source_path + ) + output, error = process.communicate() + if error: + print(output.decode()) + print(error.decode()) + # Set JAVA_API_JAR + JAVA_API_JAR = os.path.join( + jar_source_path, "dummyPSCO", "target", "compss-dummyPSCO.jar" + ) + print("Storage api jar: " + JAVA_API_JAR) + # Set global environment + os.environ["CLASSPATH"] = JAVA_API_JAR + ":" + os.environ["CLASSPATH"] + os.environ["PYTHONPATH"] = STORAGE_API + ":" + os.environ["PYTHONPATH"] + # Prepare temporary directory + if os.path.exists(TEMP_DIR): + shutil.rmtree(TEMP_DIR) + os.mkdir(TEMP_DIR) + # Prepare storage configuration + if not os.path.exists(STORAGE_CONF): + with open(STORAGE_CONF, "w") as fd: + fd.write("localhost") + + +def __clean_storage(): + # Remove python storage api from sys.path + sys.path.pop(0) + # Clean directories + if os.path.exists(STORAGE_CONF): + os.remove(STORAGE_CONF) + if os.path.exists(TEMP_DIR): + shutil.rmtree(TEMP_DIR) + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + global JAVA_API_JAR + global STORAGE_API + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "storage_app", "pscos.py") + __initialize_storage() + launch_pycompss_application( + app, + "main", + debug=True, + app_name="pscos", + storage_conf=STORAGE_CONF, + storage_impl=JAVA_API_JAR, + classpath=JAVA_API_JAR, + pythonpath=STORAGE_API, + ) + __clean_storage() diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_application_tracing.py b/examples/dds/pycompss/tests/integration/launch/test_launch_application_tracing.py new file mode 100644 index 00000000..88bede31 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_application_tracing.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "increment.py") + launch_pycompss_application( + app, "main", debug=True, app_name="increment", trace=True + ) + + +def test_launch_application_with_mpi_worker(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "increment.py") + launch_pycompss_application( + app, + "main", + debug=True, + app_name="increment", + trace=True, + mpi_worker=True, + ) diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_functions.py b/examples/dds/pycompss/tests/integration/launch/test_launch_functions.py new file mode 100644 index 00000000..6b4a7365 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_functions.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "functions.py") + launch_pycompss_application(app, "main", debug=True, app_name="functions") diff --git a/examples/dds/pycompss/tests/integration/launch/test_launch_synthetic_application.py b/examples/dds/pycompss/tests/integration/launch/test_launch_synthetic_application.py new file mode 100644 index 00000000..6eaf9af2 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/launch/test_launch_synthetic_application.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + + +def test_launch_application(): + from pycompss.runtime.launch import launch_pycompss_application + + current_path = os.path.dirname(os.path.abspath(__file__)) + app = os.path.join(current_path, "resources", "synthetic.py") + launch_pycompss_application(app, "main", debug=True, app_name="synthetic") diff --git a/examples/dds/pycompss/tests/integration/runcompss/__init__.py b/examples/dds/pycompss/tests/integration/runcompss/__init__.py new file mode 100644 index 00000000..14f3b0b1 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/runcompss/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the runcompss integration tests resources.""" diff --git a/examples/dds/pycompss/tests/integration/runcompss/test_runcompss_application.py b/examples/dds/pycompss/tests/integration/runcompss/test_runcompss_application.py new file mode 100644 index 00000000..36776e98 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/runcompss/test_runcompss_application.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import subprocess +import sys + + +def test_runcompss_increment(): + current_path = os.path.dirname(os.path.abspath(__file__)) + + # Call to runcompss for increment application + app = os.path.join( + current_path, "..", "launch", "resources", "increment.py" + ) + if sys.version_info < (3, 0): + raise Exception("Unsupported python version. Required Python 3.X") + else: + cmd = [ + "runcompss", + "--log_level=debug", + "--python_interpreter=python3", + app, + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + process.wait() + outs, errs = process.communicate() + + +def test_runcompss_increment_with_gat(): + current_path = os.path.dirname(os.path.abspath(__file__)) + + # Call to runcompss for increment application + app = os.path.join( + current_path, "..", "launch", "resources", "increment.py" + ) + if sys.version_info < (3, 0): + raise Exception("Unsupported python version. Required Python 3.X") + else: + cmd = [ + "runcompss", + "--log_level=debug", + "--python_interpreter=python3", + "--comm=es.bsc.compss.gat.master.GATAdaptor", + app, + ] + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + process.wait() + outs, errs = process.communicate() diff --git a/examples/dds/pycompss/tests/integration/runtime/__init__.py b/examples/dds/pycompss/tests/integration/runtime/__init__.py new file mode 100644 index 00000000..6466058e --- /dev/null +++ b/examples/dds/pycompss/tests/integration/runtime/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the runtime integration tests resources.""" diff --git a/examples/dds/pycompss/tests/integration/runtime/test_link.py b/examples/dds/pycompss/tests/integration/runtime/test_link.py new file mode 100644 index 00000000..6a17da07 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/runtime/test_link.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.runtime.management.link.separate import c_extension_link +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.process.manager import new_queue + + +def test_c_extension_link_wrong_message(): + in_queue = new_queue() + out_queue = new_queue() + in_queue.put("UNSUPPORTED") + is_ok = False + try: + c_extension_link(in_queue, out_queue, False, "None", "None") + except PyCOMPSsException: + is_ok = True + assert ( + is_ok + ), "ERROR: Exception not raised when undefined message received in link." diff --git a/examples/dds/pycompss/tests/integration/test_main.py b/examples/dds/pycompss/tests/integration/test_main.py new file mode 100644 index 00000000..c37afe62 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/test_main.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Tests - Integration - Main. + +This file contains the integration tests using __main__.py module. +""" + +import os +import sys + +from pycompss.__main__ import main +from pycompss.util.exceptions import PyCOMPSsException + +MAIN_NAME = "__main__.py" + + +def check_output(stdout, stderr, error_expected=False): + if ( + os.path.exists(stderr) + and os.path.getsize(stderr) > 0 + and not error_expected + ): + # Non empty file exists + raise PyCOMPSsException("An error happened. Please check " + stderr) + else: + os.remove(stdout) + os.remove(stderr) + + +def call_main(main_py, stdout, stderr): + backup_out = sys.stdout + backup_err = sys.stderr + f_out = open(stdout, "a") + f_err = open(stderr, "a") + sys.stdout = f_out + sys.stderr = f_err + sys.argv = [main_py + MAIN_NAME] + main() + f_out.close() + f_err.close() + sys.stdout = backup_out + sys.stderr = backup_err + + +def call_main_run(stdout, stderr): + backup_out = sys.stdout + backup_err = sys.stderr + f_out = open(stdout, "a") + f_err = open(stderr, "a") + sys.stdout = f_out + sys.stderr = f_err + sys.argv = [MAIN_NAME, "run"] + main() + f_out.close() + f_err.close() + sys.stdout = backup_out + sys.stderr = backup_err + + +def call_main_run_with_python_interpreter(stdout, stderr): + backup_out = sys.stdout + backup_err = sys.stderr + f_out = open(stdout, "a") + f_err = open(stderr, "a") + sys.stdout = f_out + sys.stderr = f_err + sys.argv = [MAIN_NAME, "run", "--python_interpreter=python3"] + main() + f_out.close() + f_err.close() + sys.stdout = backup_out + sys.stderr = backup_err + + +def call_main_run_without_run(stdout, stderr): + backup_out = sys.stdout + backup_err = sys.stderr + f_out = open(stdout, "a") + f_err = open(stderr, "a") + sys.stdout = f_out + sys.stderr = f_err + sys.argv = [MAIN_NAME, "undefined"] # assumes the user does not say run + main() + f_out.close() + f_err.close() + sys.stdout = backup_out + sys.stderr = backup_err + + +def call_main_enqueue(stdout, stderr): + backup_out = sys.stdout + backup_err = sys.stderr + f_out = open(stdout, "a") + f_err = open(stderr, "a") + sys.stdout = f_out + sys.stderr = f_err + sys.argv = [MAIN_NAME, "enqueue"] + main() + f_out.close() + f_err.close() + sys.stdout = backup_out + sys.stderr = backup_err + + +def test_main_script(): + current_path = os.path.dirname(os.path.abspath(__file__)) + stdout = current_path + "/../../../../std.out" + stderr = current_path + "/../../../../std.err" + main_py = current_path + "/../../../../src/pycompss/" + call_main(main_py, stdout, stderr) + check_output(stdout, stderr) + # Check that can call runcompss and enqueue_compss, but they will fail + # since they are not supposed to be available during unit testing. + call_main_run(stdout, stderr) + check_output(stdout, stderr, error_expected=True) + call_main_run_with_python_interpreter(stdout, stderr) + check_output(stdout, stderr, error_expected=True) + call_main_run_without_run(stdout, stderr) + check_output(stdout, stderr, error_expected=True) + call_main_enqueue(stdout, stderr) + check_output(stdout, stderr, error_expected=True) diff --git a/examples/dds/pycompss/tests/integration/worker/__init__.py b/examples/dds/pycompss/tests/integration/worker/__init__.py new file mode 100644 index 00000000..d3fceffe --- /dev/null +++ b/examples/dds/pycompss/tests/integration/worker/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the worker integration tests resources.""" diff --git a/examples/dds/pycompss/tests/integration/worker/common_piper_tester.py b/examples/dds/pycompss/tests/integration/worker/common_piper_tester.py new file mode 100644 index 00000000..6e727e79 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/worker/common_piper_tester.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""PyCOMPSs Testbench commons for the worker pipers testing.""" + +import os +import shutil +import sys +import tempfile +import time + +from pycompss.api.task import task +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.process.manager import create_process +from pycompss.util.serialization.serializer import deserialize_from_file +from pycompss.tests.outlog import get_logger + +# Globals +STD_OUT_FILE = "/../../../../std.out" +STD_ERR_FILE = "/../../../../std.err" +ERROR_MESSAGE = "An error happened. Please check: " + + +@task() +def simple(): + """Do nothing task. + + :returns: None. + """ + pass + + +@task(returns=1) +def increment(value): + """Increment the given value with 1. + + :param value: Integer to increment. + :returns: Incremented value with 1. + """ + return value + 1 + + +def setup_argv(argv, current_path): + """Set up the argv required with the given current path. + + :param argv: System argv. + :param current_path: Directory where to redirect stdout and stderr. + :returns: None. + """ + sys.argv = argv + sys.path.append(current_path) + sys.stdout = open(current_path + STD_OUT_FILE, "w") + sys.stderr = open(current_path + STD_ERR_FILE, "w") + + +def evaluate_piper_worker_common(worker_process, mpi_worker=False): + """Evaluate the piper worker result. + + Override sys.argv to mimic runtime call + + :params worker_process: Worker process. + :params mpi_worker: If the piper worker uses MPI for process spawning. + :returns: None. + """ + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + + files = create_files() + ( + temp_directory, + log_directory, + analysis_directory, + executor_outbound, + executor_inbound, + control_worker_outbound, + control_worker_inbound, + ) = files + + current_path = os.path.dirname(os.path.abspath(__file__)) + python_path = ( + current_path + + "/../../../tests/integration/worker/:" + + os.environ["PYTHONPATH"] + ) + + if mpi_worker: + sys.argv = [ + "mpirun", + "-np", + "2", + "-x", + "PYTHONPATH=" + python_path, + "python", + current_path + "/../../../worker/piper/mpi_piper_worker.py", + temp_directory, + log_directory, + analysis_directory, + "false", + "true", + "0", + "null", + "NONE", + "localhost", + "49049", + "false", + "false", + "false", + "1", + "1", + executor_outbound, + executor_inbound, + control_worker_outbound, + control_worker_inbound, + ] + else: + sys.argv = [ + "piper_worker.py", + temp_directory, + log_directory, + analysis_directory, + "false", + "true", + "0", + "null", + "NONE", + "localhost", + "49049", + "false", + "false", + "false", + "1", + "1", + executor_outbound, + executor_inbound, + control_worker_outbound, + control_worker_inbound, + ] + pipes = sys.argv[-4:] + # Create pipes + for pipe in pipes: + if os.path.exists(pipe): + os.remove(pipe) + os.mkfifo(pipe) + # Open pipes + executor_out = os.open(pipes[0], os.O_RDWR) + executor_in = os.open(pipes[1], os.O_RDWR) + worker_out = os.open(pipes[2], os.O_RDWR) + worker_in = os.open(pipes[3], os.O_RDWR) + + sys.path.append(current_path) + # Start the piper worker in a separate thread + worker = create_process( + target=worker_process, args=(sys.argv, current_path) + ) + if mpi_worker: + evaluate_worker( + worker, + "test_mpi_piper", + pipes, + files, + current_path, + executor_out, + executor_in, + worker_out, + worker_in, + ) + else: + evaluate_worker( + worker, + "test_piper", + pipes, + files, + current_path, + executor_out, + executor_in, + worker_out, + worker_in, + ) + + # Restore sys.argv and sys.path + sys.argv = sys_argv_backup + sys.path = sys_path_backup + + +def create_files(): + """Create initial testing files. + + :returns: A list with the temporary folder containing files. + """ + temp_directory = tempfile.mkdtemp() + os.mkdir(os.path.join(temp_directory, "temp")) + log_directory = tempfile.mkdtemp() + os.mkdir(os.path.join(log_directory, "log")) + analysis_directory = tempfile.mkdtemp() + os.mkdir(os.path.join(analysis_directory, "analysis")) + executor_outbound = tempfile.NamedTemporaryFile(delete=False).name + executor_inbound = tempfile.NamedTemporaryFile(delete=False).name + control_worker_outbound = tempfile.NamedTemporaryFile(delete=False).name + control_worker_inbound = tempfile.NamedTemporaryFile(delete=False).name + return ( + temp_directory, + log_directory, + analysis_directory, + executor_outbound, + executor_inbound, + control_worker_outbound, + control_worker_inbound, + ) + + +def evaluate_worker( + worker, + name, + pipes, + files, + current_path, + executor_out, + executor_in, + worker_out, + worker_in, +): + """Evaluate a worker. + + :param worker: Worker process. + :param name: Worker name. + :param pipes: Worker pipes. + :param files: Worker files. + :param current_path: Current working path. + :param executor_out: Executor output file. + :param executor_in: Executor input file. + :param worker_out: Worker output file. + :param worker_in: Worker input file. + :returns: None. + """ + ( + temp_directory, + log_directory, + analysis_directory, + executor_outbound, + executor_inbound, + control_worker_outbound, + control_worker_inbound, + ) = files + print("Starting " + name + " worker") + worker.start() + print("Log folder: " + log_directory) + # Wait 2 seconds to start the worker. + print("Waiting 2 seconds to send a task request") + time.sleep(2) + # Run a simple task + job1_out = tempfile.NamedTemporaryFile(delete=False).name + job1_err = tempfile.NamedTemporaryFile(delete=False).name + simple_task_message = [ + "EXECUTE_TASK", + "1", + str(os.getcwd()), + job1_out, + job1_err, + "0", + "1", + "true", + "null", + "METHOD", + "common_piper_tester", + "simple", + "0", + "1", + "localhost", + "1", + "false", + "None", + "0", + "0", + "-", + "0", + "0", + ] + simple_task_message_str = " ".join(simple_task_message) + print("Requesting: " + simple_task_message_str) + os.write(executor_out, (simple_task_message_str + "\n").encode()) # noqa + time.sleep(2) + # Run a increment task + job2_out = tempfile.NamedTemporaryFile(delete=False).name + job2_err = tempfile.NamedTemporaryFile(delete=False).name + job2_result = tempfile.NamedTemporaryFile(delete=False).name + increment_task_message = [ + "EXECUTE_TASK", + "2", + str(os.getcwd()), + job2_out, + job2_err, + "0", + "1", + "true", + "null", + "METHOD", + "common_piper_tester", + "increment", + "0", + "1", + "localhost", + "1", + "false", + "10", + "1", + "2", + "4", + "3", + "null", + "value", + "null", + "1", + "10", + "3", + "#", + "$return_0", + "null", + job2_result + ":d1v2_1599560599402.IT:false:true:" + job2_result, + "-", + "0", + "0", + ] + increment_task_message_str = " ".join(increment_task_message) + print("Requesting: " + increment_task_message_str) + os.write( + executor_out, (increment_task_message_str + "\n").encode() + ) # noqa + time.sleep(2) + # Send quit message + os.write(executor_out, b"QUIT\n") + os.write(worker_out, b"QUIT\n") + # Wait for the worker to finish + worker.join() + # Cleanup + # Close pipes + os.close(executor_out) + os.close(executor_in) + os.close(worker_out) + os.close(worker_in) + # Remove pipes + for pipe in pipes: + os.unlink(pipe) + if os.path.isfile(pipe): + os.remove(pipe) + # Check logs + out_log = os.path.join(log_directory, "binding_worker.out") + err_log = os.path.join(log_directory, "binding_worker.err") + if os.path.exists(err_log): + raise PyCOMPSsException(ERROR_MESSAGE + err_log) + with open(out_log, "r") as f: + if "ERROR" in f.read(): + raise PyCOMPSsException(ERROR_MESSAGE + out_log) + if "Traceback" in f.read(): + raise PyCOMPSsException(ERROR_MESSAGE + out_log) + # Check task 1 + check_task(job1_out, job1_err) + # Check task 2 + check_task(job2_out, job2_err) + result = deserialize_from_file(job2_result, get_logger()) + if result != 2: + raise PyCOMPSsException( + "Wrong result obtained for increment task. Expected 2, received: " + + str(result) # noqa: E501 + ) + + # Remove logs + os.remove(job1_out) + os.remove(job1_err) + os.remove(job2_out) + os.remove(job2_err) + os.remove(job2_result) + if os.path.isfile(err_log): + os.remove(err_log) + if os.path.isfile(out_log): + os.remove(out_log) + if os.path.isfile(current_path + STD_OUT_FILE): + os.remove(current_path + STD_OUT_FILE) + if os.path.isfile(current_path + STD_ERR_FILE): + os.remove(current_path + STD_ERR_FILE) + shutil.rmtree(log_directory) + if os.path.isfile(executor_outbound): + os.remove(executor_outbound) + if os.path.isfile(executor_inbound): + os.remove(executor_inbound) + if os.path.isfile(control_worker_outbound): + os.remove(control_worker_outbound) + if os.path.isfile(control_worker_inbound): + os.remove(control_worker_inbound) + + +def check_task(job_out, job_err): + """Check task output looking for errors. + + :param job_out: Task stdout file. + :param job_err: Task stderr file. + :returns: None. + :raises PyCOMPSsException: if error is found. + """ + if os.path.exists(job_err) and os.path.getsize(job_err) > 0: # noqa + # Non empty file exists + raise PyCOMPSsException( + "An error happened in the task. Please check " + job_err + ) # noqa + with open(job_out, "r") as f: + content = f.read() + if "ERROR" in content: + raise PyCOMPSsException( + "An error happened in the task. Please check " + job_out + ) # noqa + if "EXCEPTION" in content or "Exception" in content: + raise PyCOMPSsException( + "An exception happened in the task. Please check " + job_out + ) # noqa + if "END TASK execution. Status: Ok" not in content: + raise PyCOMPSsException( + "The task was supposed to be OK. Please check " + job_out + ) # noqa diff --git a/examples/dds/pycompss/tests/integration/worker/test_mpi_piper.py b/examples/dds/pycompss/tests/integration/worker/test_mpi_piper.py new file mode 100644 index 00000000..0f5c1b39 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/worker/test_mpi_piper.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import subprocess + +from pycompss.tests.integration.worker.common_piper_tester import ( + evaluate_piper_worker_common, +) +from pycompss.tests.integration.worker.common_piper_tester import setup_argv + + +def worker_thread(argv, current_path): + try: + from pycompss.worker.piper.mpi_piper_worker import main # noqa + except AttributeError: + raise Exception("UNSUPPORTED WITH MYPY") + # Start the piper worker + setup_argv(argv, current_path) + p = subprocess.Popen( + argv, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + p.wait() + + +def test_piper_worker(): + try: + evaluate_piper_worker_common(worker_thread, mpi_worker=True) + except Exception as e: + print("EXCEPTION: " + str(e)) + # raise Exception("UNSUPPORTED WITH MYPY - Happened because the " + # "worker can not start with mpi") diff --git a/examples/dds/pycompss/tests/integration/worker/test_piper.py b/examples/dds/pycompss/tests/integration/worker/test_piper.py new file mode 100644 index 00000000..95771599 --- /dev/null +++ b/examples/dds/pycompss/tests/integration/worker/test_piper.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.tests.integration.worker.common_piper_tester import ( + evaluate_piper_worker_common, +) +from pycompss.tests.integration.worker.common_piper_tester import setup_argv + + +def worker_thread(argv, current_path): + from pycompss.worker.piper.piper_worker import main + + # Start the piper worker + setup_argv(argv, current_path) + main() + + +def test_piper_worker(): + evaluate_piper_worker_common(worker_thread, mpi_worker=False) diff --git a/examples/dds/pycompss/tests/jupyter/__init__.py b/examples/dds/pycompss/tests/jupyter/__init__.py new file mode 100644 index 00000000..7ea748ad --- /dev/null +++ b/examples/dds/pycompss/tests/jupyter/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the jupyter notebook tests.""" diff --git a/examples/dds/pycompss/tests/jupyter/notebook/project.xml b/examples/dds/pycompss/tests/jupyter/notebook/project.xml new file mode 100644 index 00000000..4e34c5b6 --- /dev/null +++ b/examples/dds/pycompss/tests/jupyter/notebook/project.xml @@ -0,0 +1,8 @@ + + + + + /opt/COMPSs/ + /tmp/COMPSsWorker01/ + + diff --git a/examples/dds/pycompss/tests/jupyter/notebook/resources.xml b/examples/dds/pycompss/tests/jupyter/notebook/resources.xml new file mode 100644 index 00000000..f9d17b78 --- /dev/null +++ b/examples/dds/pycompss/tests/jupyter/notebook/resources.xml @@ -0,0 +1,28 @@ + + + + + 2 + + + + + + + + 43001 + 43002 + + + + + + sequential + + + + sshtrilead + + + + diff --git a/examples/dds/pycompss/tests/jupyter/notebook/simple.ipynb b/examples/dds/pycompss/tests/jupyter/notebook/simple.ipynb new file mode 100644 index 00000000..84b8b58d --- /dev/null +++ b/examples/dds/pycompss/tests/jupyter/notebook/simple.ipynb @@ -0,0 +1,348 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "# Test suite for Jupyter-notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## First step\n", + "Import ipycompss library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pycompss.interactive as ipycompss" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Second step \n", + "Initialize COMPSs runtime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ipycompss.start(graph=True, trace=True, debug=True, disable_external=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Third step\n", + "Import task module before annotating functions or methods " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pycompss.api.task import task" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Other declarations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "MY_GLOBAL = 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class myClass(object):\n", + " def __init__(self):\n", + " self.value = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def divide(a, b):\n", + " def is_zero(value):\n", + " if value == 0:\n", + " raise True\n", + " else:\n", + " return False\n", + "\n", + " if not is_zero(b):\n", + " return a / b\n", + " else:\n", + " raise Exception(\"Can not divide by 0.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def multiply(a, b):\n", + " return a * b" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fourth step\n", + "Declare functions and decorate with @task those that should be tasks " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@task(returns=int)\n", + "def test(val1):\n", + " return multiply(val1, val1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@task(returns=int)\n", + "def test2(val2, val3):\n", + " return val2 + val3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@task(returns=int)\n", + "def test3(val2, val3):\n", + " return divide(val2, val3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fifth step\n", + "Invoke tasks " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a = test(2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "b = test2(a, 5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "c = test3(a, 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sixt step \n", + "Import compss_wait_on module and synchronize tasks " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pycompss.api.api import compss_wait_on" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = compss_wait_on(b)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Only those results being sychronized with compss_wait_on will have a valid value " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Results: \")\n", + "print(\"a: \", a)\n", + "print(\"b: \", b)\n", + "print(\"c: \", c)\n", + "print(\"result: \", result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Stop COMPSs runtime. All data will be synchronized in the main program " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "ipycompss.stop(sync=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "print(\"Results after stopping PyCOMPSs: \")\n", + "print(\"a: \", a)\n", + "print(\"b: \", b)\n", + "print(\"c: \", c)\n", + "print(\"result: \", result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "### CHECK THE RESULTS FOR THE TEST " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pycompss.runtime.management.classes import Future\n", + "\n", + "if a == 4 and isinstance(b, Future) and result == 9 and c == 2:\n", + " print(\"RESULT=EXPECTED\")\n", + "else:\n", + " raise Exception(\"RESULT=UNEXPECTED\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.2" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/examples/dds/pycompss/tests/jupyter/test_notebook.py b/examples/dds/pycompss/tests/jupyter/test_notebook.py new file mode 100644 index 00000000..eaeb8dde --- /dev/null +++ b/examples/dds/pycompss/tests/jupyter/test_notebook.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys + +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor + + +def test_simple_notebook(): + current_path = os.path.dirname(os.path.abspath(__file__)) + simple_notebook = os.path.join( + current_path, "../integration", "resources", "notebook", "simple.ipynb" + ) + with open(simple_notebook) as f: + nb = nbformat.read(f, as_version=4) + if sys.version_info < (3, 0): + raise Exception("Unsupported python version. Required Python 3.X") + else: + ep = ExecutePreprocessor(timeout=600, kernel_name="python3") + ep.preprocess(nb) diff --git a/examples/dds/pycompss/tests/outlog.py b/examples/dds/pycompss/tests/outlog.py new file mode 100644 index 00000000..c86cc399 --- /dev/null +++ b/examples/dds/pycompss/tests/outlog.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the unittests default logger.""" + +import logging +import sys + + +def create_logger() -> logging.Logger: + """Create a logger for unit tests. + + Currently, creates a logger that redirects all messages to standard + output. + + :return: Logger. + """ + log_level = logging.DEBUG + + logger = logging.getLogger() + logger.setLevel(log_level) + + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(log_level) + msg_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + formatter = logging.Formatter(msg_format) + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +def get_logger() -> logging.Logger: + """Get current logger for unit tests. + + :return: Logger. + """ + logger = logging.getLogger() + return logger diff --git a/examples/dds/pycompss/tests/unittests/__init__.py b/examples/dds/pycompss/tests/unittests/__init__.py new file mode 100644 index 00000000..f6985279 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/api/__init__.py b/examples/dds/pycompss/tests/unittests/api/__init__.py new file mode 100644 index 00000000..8d0d786e --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the API unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/__init__.py b/examples/dds/pycompss/tests/unittests/api/dummy/__init__.py new file mode 100644 index 00000000..1256bbba --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the dummy API unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_binary.py b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_binary.py new file mode 100644 index 00000000..a3cebae6 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_binary.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT + +CONTEXT.set_out_of_scope() + +from pycompss.api.api import compss_wait_on # noqa +from pycompss.api.binary import Binary # noqa +from pycompss.api.task import Task # noqa + + +@Binary(binary="date", working_dir="/tmp") # NOSONAR +@Task(returns=1) +def check_date(): + # Intentionally empty since it is a binary task. + pass + + +def test_dummy_task(): + result = check_date() + result = compss_wait_on(result) + assert result == 0, "ERROR: Unexpected exit code with dummy @binary." diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_container.py b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_container.py new file mode 100644 index 00000000..5e3d8356 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_container.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.api.dummy.container import Container + + +@Container(engine="docker", image="dummy") +def increment(value): + return value + 1 + + +def test_dummy_task(): + result = increment(1) + assert result == 2, ( + "Unexpected result provided by the dummy container decorator. Expected: 2 Received: %s" # noqa: E501 + % str(result) + ) diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_on_failure.py b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_on_failure.py new file mode 100644 index 00000000..40af9a3e --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_on_failure.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT + +CONTEXT.set_out_of_scope() + +from pycompss.api.api import compss_wait_on # noqa +from pycompss.api.on_failure import on_failure # noqa +from pycompss.api.task import Task # noqa + + +@on_failure(management="IGNORE") # NOSONAR +@Task(returns=1) +def increment(value): + return value + 1 + + +def test_dummy_task(): + initial = 3 + result = increment(initial) + result = compss_wait_on(result) + assert ( + result == initial + 1 + ), "ERROR: Unexpected exit code with dummy @on_failure." diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_reduction.py b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_reduction.py new file mode 100644 index 00000000..d6982fbd --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_reduction.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.api.dummy.reduction import Reduction +from pycompss.api.dummy.task import Task + + +@Reduction() +@Task() +def increment(value): + return value + 1 + + +def test_dummy_task(): + result = increment(1) + assert result == 2, ( + "Unexpected result provided by the dummy task decorator. Expected: 2 Received: %s" # noqa: E501 + % str(result) + ) diff --git a/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_task.py b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_task.py new file mode 100644 index 00000000..2493b360 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/dummy/test_dummy_task.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.api.dummy.task import Task + + +@Task() +def increment(value): + return value + 1 + + +def test_dummy_task(): + result = increment(1) + assert result == 2, ( + "Unexpected result provided by the dummy task decorator. Expected: 2 Received: %s" # noqa: E501 + % str(result) + ) diff --git a/examples/dds/pycompss/tests/unittests/api/test_api.py b/examples/dds/pycompss/tests/unittests/api/test_api.py new file mode 100644 index 00000000..4c1c16b1 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_api.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + +# CAUTION: Unittest for real api functions are in integration tests. + + +def test_api_calling_dummy(): + from pycompss.util.context import CONTEXT + + CONTEXT.set_master() + # Load exposed api, not dummy + from pycompss.api.api import compss_start + from pycompss.api.api import compss_stop + from pycompss.api.api import compss_file_exists + from pycompss.api.api import compss_open + from pycompss.api.api import compss_delete_file + from pycompss.api.api import compss_wait_on_file + from pycompss.api.api import compss_wait_on_directory + from pycompss.api.api import compss_delete_object + from pycompss.api.api import compss_barrier + from pycompss.api.api import compss_barrier_group + from pycompss.api.api import compss_wait_on + from pycompss.api.api import compss_get_number_of_resources + from pycompss.api.api import compss_request_resources + from pycompss.api.api import compss_free_resources + from pycompss.api.api import TaskGroup + + # Force to use dummies + CONTEXT.set_out_of_scope() + + file_name = "simulated_file.txt" + file_names = ["simulated_file1.txt", "simulated_file2.txt"] + directory_name = "simulated_directory" + directory_names = ["simulated_directory1", "simulated_directory2"] + group_name = "simulated_group" + obj = [1, 2, 3] + num_resources = 1 + + with open(file_name, "w") as f: + f.write("some content") + os.mkdir(directory_name) + + for f_name in file_names: + with open(f_name, "w") as f: + f.write("some content") + for d_name in directory_names: + os.mkdir(d_name) + + compss_start(log_level="off", interactive=False) + compss_stop(code=0) + compss_file_exists(file_name) + compss_file_exists(*file_names) + compss_open(file_name, mode="r") + compss_delete_file(file_name) + compss_delete_file(*file_names) + compss_wait_on_file(file_name) + compss_wait_on_file(*file_names) + compss_wait_on_directory(directory_name) + compss_wait_on_directory(*directory_names) + compss_delete_object(obj) + compss_delete_object(*obj) + compss_barrier(no_more_tasks=False) + compss_barrier_group(group_name) + compss_wait_on(obj) + compss_wait_on(*obj) + compss_get_number_of_resources() + compss_request_resources(num_resources, group_name) + compss_free_resources(num_resources, group_name) + + with TaskGroup(group_name, implicit_barrier=True): + # Empty task group check + pass + + os.remove(file_name) + os.rmdir(directory_name) + + for f_name in file_names: + os.remove(f_name) + for d_name in directory_names: + os.rmdir(d_name) + + +def test_dummy_api(): + from pycompss.api.dummy.api import compss_start + from pycompss.api.dummy.api import compss_stop + from pycompss.api.dummy.api import compss_file_exists + from pycompss.api.dummy.api import compss_open + from pycompss.api.dummy.api import compss_delete_file + from pycompss.api.dummy.api import compss_wait_on_file + from pycompss.api.dummy.api import compss_wait_on_directory + from pycompss.api.dummy.api import compss_delete_object + from pycompss.api.dummy.api import compss_barrier + from pycompss.api.dummy.api import compss_barrier_group + from pycompss.api.dummy.api import compss_wait_on + from pycompss.api.dummy.api import compss_get_number_of_resources + from pycompss.api.dummy.api import compss_request_resources + from pycompss.api.dummy.api import compss_free_resources + from pycompss.api.dummy.api import TaskGroup + + file_name = "simulated_file.txt" + file_names = ["simulated_file1.txt", "simulated_file2.txt"] + directory_name = "simulated_directory" + directory_names = ["simulated_directory1", "simulated_directory2"] + group_name = "simulated_group" + obj = [1, 2, 3] + num_resources = 1 + + with open(file_name, "w") as f: + f.write("some content") + os.mkdir(directory_name) + + for f_name in file_names: + with open(f_name, "w") as f: + f.write("some content") + for d_name in directory_names: + os.mkdir(d_name) + + compss_start(log_level="off", interactive=False) + compss_stop(code=0) + compss_file_exists(file_name) + compss_file_exists(*file_names) + compss_open(file_name, mode="r") + compss_delete_file(file_name) + compss_delete_file(*file_names) + compss_wait_on_file(file_name) + compss_wait_on_file(*file_names) + compss_wait_on_directory(directory_name) + compss_wait_on_directory(*directory_names) + compss_delete_object(obj) + compss_delete_object(*obj) + compss_barrier(no_more_tasks=False) + compss_barrier_group(group_name) + compss_wait_on(obj) + compss_wait_on(*obj) + compss_get_number_of_resources() + compss_request_resources(num_resources, group_name) + compss_free_resources(num_resources, group_name) + + with TaskGroup(group_name, implicit_barrier=True): + # Empty task group check + pass + + os.remove(file_name) + os.rmdir(directory_name) + + for f_name in file_names: + os.remove(f_name) + for d_name in directory_names: + os.rmdir(d_name) diff --git a/examples/dds/pycompss/tests/unittests/api/test_binary.py b/examples/dds/pycompss/tests/unittests/api/test_binary.py new file mode 100644 index 00000000..c576a3c1 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_binary.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.binary import Binary +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_binary_instantiation(): + CONTEXT.set_master() + my_bin = Binary(binary="date") + CONTEXT.set_out_of_scope() + assert ( + my_bin.decorator_name == "@binary" + ), "The decorator name must be @binary." + + +def test_binary_call(): + CONTEXT.set_master() + my_bin = Binary(binary="date") + f = my_bin(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +# # Disabled due to support of dummy @binary +# def test_binary_call_outside(): +# CONTEXT.set_out_of_scope() +# my_bin = Binary(binary="date") +# f = my_bin(dummy_function) +# thrown = False +# try: +# _ = f() +# except Exception: # noqa +# thrown = True # this is OK! +# CONTEXT.set_out_of_scope() +# assert thrown, \ +# "The binary decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_binary_engine_parameter(): + CONTEXT.set_master() + engine = "my_engine" + my_bin = Binary(binary="date", engine=engine) + f = my_bin(dummy_function) + _ = f() + assert ( + "engine" in my_bin.kwargs + ), "Engine is not defined in kwargs dictionary." + assert ( + engine == my_bin.kwargs["engine"] + ), "Engine parameter has not been initialized." + + +def test_binary_image_parameter(): + CONTEXT.set_master() + image = "my_image" + my_bin = Binary(binary="date", image=image) + f = my_bin(dummy_function) + _ = f() + assert ( + "image" in my_bin.kwargs + ), "Image is not defined in kwargs dictionary." + assert ( + image == my_bin.kwargs["image"] + ), "Image parameter has not been initialized." + + +def test_binary_existing_core_element(): + CONTEXT.set_master() + my_bin = Binary(binary="date") + f = my_bin(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + assert ( + CORE_ELEMENT_KEY not in my_bin.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_compss.py b/examples/dds/pycompss/tests/unittests/api/test_compss.py new file mode 100644 index 00000000..e01f81c6 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_compss.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.compss import COMPSs +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_compss_instantiation(): + CONTEXT.set_master() + my_compss = COMPSs(app_name="date") + CONTEXT.set_out_of_scope() + assert ( + my_compss.decorator_name == "@compss" + ), "The decorator name must be @compss." + + +def test_compss_call(): + CONTEXT.set_master() + my_compss = COMPSs(app_name="date") + f = my_compss(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_compss_call_outside(): + CONTEXT.set_out_of_scope() + my_compss = COMPSs(app_name="date") + f = my_compss(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The compss decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_compss_appname_parameter(): # NOSONAR + CONTEXT.set_master() + app_name = "my_appName" # noqa + my_compss = COMPSs(app_name="date", appName=app_name) + f = my_compss(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "appName" in my_compss.kwargs + ), "appName is not defined in kwargs dictionary." + assert ( + app_name == my_compss.kwargs["appName"] + ), "appName parameter has not been initialized." + + +def test_compss_runcompss_parameter(): + CONTEXT.set_master() + runcompss = "my_runcompss" + my_compss = COMPSs(app_name="date", runcompss=runcompss) + f = my_compss(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "runcompss" in my_compss.kwargs + ), "Runcompss is not defined in kwargs dictionary." + assert ( + runcompss == my_compss.kwargs["runcompss"] + ), "Runcompss parameter has not been initialized." + + +def test_compss_flags_parameter(): + CONTEXT.set_master() + flags = "my_flags" + my_compss = COMPSs(app_name="date", flags=flags) + f = my_compss(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "flags" in my_compss.kwargs + ), "flags is not defined in kwargs dictionary." + assert ( + flags == my_compss.kwargs["flags"] + ), "flags parameter has not been initialized." + + +def test_compss_worker_in_master_parameter(): + CONTEXT.set_master() + worker_in_master = "my_worker_in_master" + my_compss = COMPSs(app_name="date", worker_in_master=worker_in_master) + f = my_compss(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "worker_in_master" in my_compss.kwargs + ), "worker_in_master is not defined in kwargs dictionary." + assert ( + worker_in_master == my_compss.kwargs["worker_in_master"] + ), "worker_in_master parameter has not been initialized." + + +def test_compss_workerinmaster_parameter(): # NOSONAR + CONTEXT.set_master() + worker_in_master = "my_workerInMaster" # noqa + my_compss = COMPSs(app_name="date", workerInMaster=worker_in_master) + f = my_compss(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "workerInMaster" in my_compss.kwargs + ), "workerInMaster is not defined in kwargs dictionary." + assert ( + worker_in_master == my_compss.kwargs["workerInMaster"] + ), "workerInMaster parameter has not been initialized." + + +def test_compss_existing_core_element(): + CONTEXT.set_master() + my_compss = COMPSs(app_name="date") + f = my_compss(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_compss.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_constraint.py b/examples/dds/pycompss/tests/unittests/api/test_constraint.py new file mode 100644 index 00000000..9d5e817d --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_constraint.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.constraint import Constraint +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_constraint_instantiation(): + CONTEXT.set_master() + my_constraint = Constraint() + CONTEXT.set_out_of_scope() + assert ( + my_constraint.decorator_name == "@constraint" + ), "The decorator name must be @constraint." + + +def test_constraint_call(): + CONTEXT.set_master() + my_constraint = Constraint() + f = my_constraint(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_constraint_call_outside(): + CONTEXT.set_out_of_scope() + my_constraint = Constraint() + f = my_constraint(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + # Does not raise exception (dummy instead) + + +def test_constraint_existing_core_element(): + CONTEXT.set_master() + my_constraint = Constraint() + f = my_constraint(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_constraint.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_container.py b/examples/dds/pycompss/tests/unittests/api/test_container.py new file mode 100644 index 00000000..5b5564b2 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_container.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.container import Container +from pycompss.api.binary import Binary +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_container_instantiation(): + CONTEXT.set_master() + my_bin = Container(engine="docker", image="dummy") + CONTEXT.set_out_of_scope() + assert ( + my_bin.decorator_name == "@container" + ), "The decorator name must be @container." + + +def test_container_call(): + CONTEXT.set_master() + my_bin = Container(engine="docker", image="dummy") + f = my_bin(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_container_call_binary(): + CONTEXT.set_master() + my_cont = Container(engine="docker", image="dummy") + my_bin = Binary(binary="date") + f = my_cont(my_bin(dummy_function)) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_container_call_outside(): + CONTEXT.set_out_of_scope() + my_bin = Container(engine="docker", image="dummy") + f = my_bin(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The container decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_container_engine_image_parameters(): + CONTEXT.set_master() + engine = "docker" + image = "dummy" + my_bin = Container(engine=engine, image=image) + f = my_bin(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "engine" in my_bin.kwargs + ), "Engine is not defined in kwargs dictionary." + assert ( + engine == my_bin.kwargs["engine"] + ), "Engine parameter has not been initialized." + assert ( + "image" in my_bin.kwargs + ), "Image is not defined in kwargs dictionary." + assert ( + image == my_bin.kwargs["image"] + ), "image parameter has not been initialized." + + +def test_container_existing_core_element(): + CONTEXT.set_master() + my_bin = Container(engine="docker", image="dummy") + f = my_bin(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_bin.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_decaf.py b/examples/dds/pycompss/tests/unittests/api/test_decaf.py new file mode 100644 index 00000000..90dc67fb --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_decaf.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.decaf import Decaf +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_decaf_instantiation(): + CONTEXT.set_master() + my_decaf = Decaf(df_script="date") + CONTEXT.set_out_of_scope() + assert ( + my_decaf.decorator_name == "@decaf" + ), "The decorator name must be @decaf." + + +def test_decaf_call(): + CONTEXT.set_master() + my_decaf = Decaf(df_script="date") + f = my_decaf(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_decaf_call_outside(): + CONTEXT.set_out_of_scope() + my_decaf = Decaf(df_script="date") + f = my_decaf(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The decaf decorator did not raise an exception when " + "invoked out of scope." + ) + + +def test_decaf_runner_parameter(): + CONTEXT.set_master() + runner = "my_runner" + my_decaf = Decaf(df_script="date", runner=runner) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "runner" in my_decaf.kwargs + ), "Runner is not defined in kwargs dictionary." + assert ( + runner == my_decaf.kwargs["runner"] + ), "Runner parameter has not been initialized." + + +def test_decaf_dfscript_parameter(): + CONTEXT.set_master() + df_script = "my_dfScript" # noqa + my_decaf = Decaf(df_script="date", dfScript=df_script) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "dfScript" in my_decaf.kwargs + ), "dfScript is not defined in kwargs dictionary." + assert ( + df_script == my_decaf.kwargs["dfScript"] + ), "dfScript parameter has not been initialized." + + +def test_decaf_df_executor_parameter(): + CONTEXT.set_master() + df_executor = "my_df_executor" + my_decaf = Decaf(df_script="date", df_executor=df_executor) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "df_executor" in my_decaf.kwargs + ), "df_executor is not defined in kwargs dictionary." + assert ( + df_executor == my_decaf.kwargs["df_executor"] + ), "df_executor parameter has not been initialized." + + +def test_decaf_dfexecutor_parameter(): # NOSONAR + CONTEXT.set_master() + df_executor = "my_dfExecutor" # noqa + my_decaf = Decaf(df_script="date", dfExecutor=df_executor) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "dfExecutor" in my_decaf.kwargs + ), "dfExecutor is not defined in kwargs dictionary." + assert ( + df_executor == my_decaf.kwargs["dfExecutor"] + ), "dfExecutor parameter has not been initialized." + + +def test_decaf_df_lib_parameter(): + CONTEXT.set_master() + df_lib = "my_df_lib" + my_decaf = Decaf(df_script="date", df_lib=df_lib) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "df_lib" in my_decaf.kwargs + ), "df_lib is not defined in kwargs dictionary." + assert ( + df_lib == my_decaf.kwargs["df_lib"] + ), "df_lib parameter has not been initialized." + + +def test_decaf_dflib_parameter(): # NOSONAR + CONTEXT.set_master() + df_lib = "my_dfLib" # noqa + my_decaf = Decaf(df_script="date", dfLib=df_lib) + f = my_decaf(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "dfLib" in my_decaf.kwargs + ), "dfLib is not defined in kwargs dictionary." + assert ( + df_lib == my_decaf.kwargs["dfLib"] + ), "dfLib parameter has not been initialized." + + +def test_decaf_existing_core_element(): + CONTEXT.set_master() + my_decaf = Decaf(df_script="date") + f = my_decaf(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_decaf.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_epilog.py b/examples/dds/pycompss/tests/unittests/api/test_epilog.py new file mode 100644 index 00000000..c83bb50e --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_epilog.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.epilog import epilog +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_epilog_instantiation(): + CONTEXT.set_master() + my_epilog = epilog(binary="date") + assert ( + my_epilog.decorator_name == "@epilog" + ), "The decorator name must be @epilog" + + +def test_epilog_call(): + CONTEXT.set_master() + my_epilog = epilog(binary="date") + f = my_epilog(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_epilog_call_outside(): + CONTEXT.set_out_of_scope() + my_epilog = epilog(binary="date") + f = my_epilog(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The epilog decorator did not raise an exception when " + "invoked out of scope." + ) + + +def test_epilog_existing_core_element(): + CONTEXT.set_master() + my_epilog = epilog(binary="date") + f = my_epilog(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_epilog.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_err_msgs.py b/examples/dds/pycompss/tests/unittests/api/test_err_msgs.py new file mode 100644 index 00000000..2371235b --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_err_msgs.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.api.commons.error_msgs import cast_env_to_int_error +from pycompss.api.commons.error_msgs import cast_string_to_int_error +from pycompss.api.commons.error_msgs import not_in_pycompss +from pycompss.api.commons.error_msgs import wrong_value + +DECORATOR_NAME = "@unittest" + + +def test_err_msgs_not_in_pycompss(): + decorator_name = DECORATOR_NAME + expected = ( + "The %s decorator only works within PyCOMPSs framework." + % decorator_name + ) + error = not_in_pycompss(decorator_name=decorator_name) + assert error == expected, "Received wrong NOT IN PYCOMPSS error message." + + +def test_err_msgs_cast_env_to_int_error(): + what = DECORATOR_NAME + expected = "ERROR: %s value cannot be cast from ENV variable to int" % what + error = cast_env_to_int_error(what=what) + assert error == expected, "Received wrong Cast env to int error message." + + +def test_err_msgs_cast_string_to_int_error(): + what = DECORATOR_NAME + expected = "ERROR: %s value cannot be cast from string to int" % what + error = cast_string_to_int_error(what=what) + assert ( + error == expected + ), "Received wrong Cast string to int error message." + + +def test_err_msgs_wrong_value(): + value_name = "unittest" + decorator_name = DECORATOR_NAME + expected = "ERROR: Wrong %s value at %s decorator." % ( + value_name, + decorator_name, + ) + error = wrong_value(value_name=value_name, decorator_name=decorator_name) + assert error == expected, "Received wrong error message." diff --git a/examples/dds/pycompss/tests/unittests/api/test_exceptions.py b/examples/dds/pycompss/tests/unittests/api/test_exceptions.py new file mode 100644 index 00000000..8ff3246a --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_exceptions.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.api.exceptions import COMPSsException + + +def test_compss_exception_instantiation(): # NOSONAR + exc = COMPSsException("my message") + is_exception = False + if issubclass(type(exc), Exception): + is_exception = True + assert is_exception, "The COMPSsException does not inherit from Exception" diff --git a/examples/dds/pycompss/tests/unittests/api/test_http.py b/examples/dds/pycompss/tests/unittests/api/test_http.py new file mode 100644 index 00000000..83df6a82 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_http.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.http import http +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.serialization import serializer + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_http_instantiation(): + CONTEXT.set_master() + my_http = http( + service_name="service", resource="resource", request="request" + ) + serializer.FORCED_SERIALIZER = -1 + assert ( + my_http.decorator_name == "@http" + ), "The decorator name must be @http" + + +def test_http_call(): + CONTEXT.set_master() + my_http = http( + service_name="service", resource="resource", request="request" + ) + f = my_http(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + serializer.FORCED_SERIALIZER = -1 + assert result == 1, "Wrong expected result (should be 1)." + + +def test_http_call_outside(): + CONTEXT.set_out_of_scope() + my_http = http( + service_name="service", resource="resource", request="request" + ) + f = my_http(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + serializer.FORCED_SERIALIZER = -1 + + +def test_http_existing_core_element(): + CONTEXT.set_master() + my_http = http( + service_name="service", resource="resource", request="request" + ) + f = my_http(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + serializer.FORCED_SERIALIZER = -1 + assert ( + CORE_ELEMENT_KEY not in my_http.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_implement.py b/examples/dds/pycompss/tests/unittests/api/test_implement.py new file mode 100644 index 00000000..fae9e602 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_implement.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.implement import Implement +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_implement_instantiation(): + CONTEXT.set_master() + my_implementation = Implement(source_class="s_class", method="s_method") + CONTEXT.set_out_of_scope() + assert ( + my_implementation.decorator_name == "@implement" + ), "The decorator name must be @implement." + + +def test_implement_call(): + CONTEXT.set_master() + my_implementation = Implement(source_class="s_class", method="s_method") + f = my_implementation(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_implement_call_old_mode(): + CONTEXT.set_master() + my_implementation = Implement(sourceClass="s_class", method="s_method") + f = my_implementation(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_implement_call_outside(): + CONTEXT.set_out_of_scope() + my_implementation = Implement(source_class="s_class", method="s_method") + f = my_implementation(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The implement decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_implement_existing_core_element(): + CONTEXT.set_master() + my_implementation = Implement(source_class="s_class", method="s_method") + # Hack to mimic registered + my_implementation.first_register = True + f = my_implementation(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_implementation.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_io.py b/examples/dds/pycompss/tests/unittests/api/test_io.py new file mode 100644 index 00000000..61ee6d7a --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_io.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.IO import IO +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_io_instantiation(): + CONTEXT.set_master() + my_io = IO() + CONTEXT.set_out_of_scope() + assert my_io.decorator_name == "@io", "The decorator name must be @io." + + +def test_io_call(): + CONTEXT.set_master() + my_io = IO() + f = my_io(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_io_call_outside(): + CONTEXT.set_out_of_scope() + my_io = IO() + f = my_io(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The ompss decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_io_existing_core_element(): + CONTEXT.set_master() + my_io = IO() + f = my_io(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_io.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_local.py b/examples/dds/pycompss/tests/unittests/api/test_local.py new file mode 100644 index 00000000..3098cbac --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_local.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT + +using_mypy = False +try: + from pycompss.api.local import local +except ImportError: + + def local(func): + return func + + using_mypy = True + + +def test_local_instantiation(): + if using_mypy: + raise Exception("UNSUPPORTED WITH MYPY") + CONTEXT.set_master() + + @local + def dummy_function(*args, **kwargs): # noqa + return sum(args) + + result = dummy_function(1, 2, other=3) + CONTEXT.set_out_of_scope() + assert result == 3, "Wrong expected result (should be 3)." + + +def test_local_instantiation_outside(): + if using_mypy: + raise Exception("UNSUPPORTED WITH MYPY") + CONTEXT.set_out_of_scope() + + @local + def dummy_function(*args, **kwargs): # noqa + return sum(args) + + result = dummy_function(1, 2) + CONTEXT.set_out_of_scope() + assert result == 3, "Wrong expected result (should be 3)." diff --git a/examples/dds/pycompss/tests/unittests/api/test_mpi.py b/examples/dds/pycompss/tests/unittests/api/test_mpi.py new file mode 100644 index 00000000..903cde56 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_mpi.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.mpi import Mpi +from pycompss.runtime.task.definitions.core_element import CE + +MPI_RUNNER = "mpirun" +ERROR_EXPECTED_1 = "Wrong expected result (should be 1)." + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_mpi_instantiation(): + CONTEXT.set_master() + my_mpi = Mpi(runner=MPI_RUNNER) + CONTEXT.set_out_of_scope() + assert my_mpi.decorator_name == "@mpi", "The decorator name must be @mpi." + + +def test_mpi_call(): + CONTEXT.set_master() + my_mpi = Mpi(runner=MPI_RUNNER) + f = my_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_mpi_call_outside(): + CONTEXT.set_out_of_scope() + my_mpi = Mpi(runner=MPI_RUNNER, processes=2, binary="date") + f = my_mpi(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The mpi decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_mpi_call_outside_with_computing_nodes_old_style(): + CONTEXT.set_out_of_scope() + my_mpi = Mpi(runner=MPI_RUNNER, computingNodes=2, binary="date") + f = my_mpi(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The mpi decorator did not raise an exception when invoked out of scope (computingNodes)." # noqa: E501 + + +def test_mpi_call_outside_with_computing_nodes(): + CONTEXT.set_out_of_scope() + my_mpi = Mpi(runner=MPI_RUNNER, computing_nodes=2, binary="date") + f = my_mpi(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The mpi decorator did not raise an exception when invoked out of scope (computing_nodes)." # noqa: E501 + + +def test_mpi_layout_empty_parameter(): + CONTEXT.set_master() + layout = dict() + my_mpi = Mpi(runner=MPI_RUNNER, _layout={"_layout": layout}) + f = my_mpi(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "_layout" in my_mpi.kwargs + ), "_layout is not defined in kwargs dictionary." + + +def test_mpi_binary(): + CONTEXT.set_master() + my_mpi = Mpi(runner=MPI_RUNNER, binary="date", flags="flags") + f = my_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_mpi_binary_scale_bool_true(): + CONTEXT.set_master() + my_mpi = Mpi( + runner=MPI_RUNNER, binary="date", flags="flags", scale_by_cu=True + ) + f = my_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_mpi_binary_scale_bool_false(): + CONTEXT.set_master() + my_mpi = Mpi( + runner=MPI_RUNNER, binary="date", flags="flags", scale_by_cu=False + ) + f = my_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_mpi_binary_scale_str(): + CONTEXT.set_master() + my_mpi = Mpi( + runner=MPI_RUNNER, binary="date", flags="flags", scale_by_cu="true" + ) + f = my_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_mpi_binary_scale_incorrect(): + CONTEXT.set_master() + my_mpi = Mpi( + runner=MPI_RUNNER, binary="date", flags="flags", scale_by_cu=1 + ) + f = my_mpi(dummy_function) + exception = False + try: + _ = f() + except Exception: # noqa + exception = True + CONTEXT.set_out_of_scope() + assert exception, "Unsupported scale_by_cu value exception not raised." + + +def test_mpi_existing_core_element(): + CONTEXT.set_master() + my_mpi = Mpi(runner=MPI_RUNNER) + f = my_mpi(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_mpi.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_mpmd_mpi.py b/examples/dds/pycompss/tests/unittests/api/test_mpmd_mpi.py new file mode 100644 index 00000000..51227155 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_mpmd_mpi.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.mpmd_mpi import mpmd_mpi +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.exceptions import PyCOMPSsException + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_mpmd_mpi_instantiation(): + CONTEXT.set_master() + my_mpmd_mpi = mpmd_mpi(runner="runner") + assert ( + my_mpmd_mpi.decorator_name == "@mpmdmpi" + ), "The decorator name must be @mpmdmpi" + + +def test_mpmd_mpi_call(): + CONTEXT.set_master() + my_mpmd_mpi = mpmd_mpi( + runner="runner", programs=[{"binary": "binary"}], _layout="_layout" + ) + f = my_mpmd_mpi(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_mpmd_mpi_call_outside(): + CONTEXT.set_out_of_scope() + my_mpmd_mpi = mpmd_mpi(runner="runner", programs=[{"binary": "binary"}]) + f = my_mpmd_mpi(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The mpmd_mpi decorator did not raise an exception when " + "invoked out of scope." + ) + + +def test_mpmd_mpi_call_outside_invalid_program(): + CONTEXT.set_master() + my_mpmd_mpi = mpmd_mpi(runner="runner", programs="programs") + f = my_mpmd_mpi(dummy_function) + thrown = False + try: + _ = f() + except PyCOMPSsException: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The mpmd_mpi decorator did not raise an exception " + "for an incorrect program." + ) + + +def test_mpmd_mpi_call_outside_not_binary(): + CONTEXT.set_master() + my_mpmd_mpi = mpmd_mpi(runner="runner", programs=[{"other": "other"}]) + f = my_mpmd_mpi(dummy_function) + thrown = False + try: + _ = f() + except PyCOMPSsException: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The mpmd_mpi decorator did not raise an exception when no binary " + "is provided in program." + ) + + +def test_mpmd_mpi_existing_core_element(): + CONTEXT.set_master() + my_mpmd_mpi = mpmd_mpi(runner="runner", programs=[{"binary": "binary"}]) + f = my_mpmd_mpi(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_mpmd_mpi.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_multinode.py b/examples/dds/pycompss/tests/unittests/api/test_multinode.py new file mode 100644 index 00000000..a2f48f97 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_multinode.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.multinode import MultiNode +from pycompss.runtime.task.definitions.core_element import CE + +ERROR_EXPECTED_1 = "Wrong expected result (should be 1)." + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_multinode_instantiation(): + CONTEXT.set_master() + my_multinode = MultiNode() + CONTEXT.set_out_of_scope() + assert ( + my_multinode.decorator_name == "@multinode" + ), "The decorator name must be @multinode." + + +def test_multinode_call_outside(): + CONTEXT.set_out_of_scope() + my_multinode = MultiNode() + f = my_multinode(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The multinode decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_multinode_call_master(): + CONTEXT.set_master() + my_multinode = MultiNode() + f = my_multinode(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_multinode_call_worker(): + CONTEXT.set_worker() + # prepare test setup + os.environ["COMPSS_NUM_NODES"] = "2" + os.environ["COMPSS_NUM_THREADS"] = "2" + os.environ["COMPSS_HOSTNAMES"] = "hostnames" + # call + my_multinode = MultiNode() + f = my_multinode(dummy_function) + result = f() + # clean test setup + del os.environ["COMPSS_NUM_NODES"] + del os.environ["COMPSS_NUM_THREADS"] + del os.environ["COMPSS_HOSTNAMES"] + # Check result + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_multinode_call_worker_with_slurm(): + CONTEXT.set_worker() + # prepare test setup + os.environ["COMPSS_NUM_NODES"] = "2" + os.environ["COMPSS_NUM_THREADS"] = "2" + os.environ["COMPSS_HOSTNAMES"] = "hostname1,hostname2" + os.environ["SLURM_NTASKS"] = "2" + os.environ["SLURM_NNODES"] = "2" + os.environ["SLURM_NODELIST"] = "hostname1,hostname2" + os.environ["SLURM_TASKS_PER_NODE"] = "2" + # call + my_multinode = MultiNode() + f = my_multinode(dummy_function) + result = f() + # clean test setup + del os.environ["COMPSS_NUM_NODES"] + del os.environ["COMPSS_NUM_THREADS"] + del os.environ["COMPSS_HOSTNAMES"] + del os.environ["SLURM_NTASKS"] + del os.environ["SLURM_NNODES"] + del os.environ["SLURM_NODELIST"] + del os.environ["SLURM_TASKS_PER_NODE"] + # Check result + CONTEXT.set_out_of_scope() + assert result == 1, ERROR_EXPECTED_1 + + +def test_multinode_existing_core_element(): + CONTEXT.set_master() + my_multinode = MultiNode() + f = my_multinode(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_multinode.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_ompss.py b/examples/dds/pycompss/tests/unittests/api/test_ompss.py new file mode 100644 index 00000000..133db3c8 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_ompss.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.ompss import OmpSs +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_ompss_instantiation(): + CONTEXT.set_master() + my_ompss = OmpSs(binary="date") + assert ( + my_ompss.decorator_name == "@ompss" + ), "The decorator name must be @ompss." + + +def test_ompss_call(): + CONTEXT.set_master() + my_ompss = OmpSs(binary="date") + f = my_ompss(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_ompss_call_outside(): + CONTEXT.set_out_of_scope() + my_ompss = OmpSs(binary="date") + f = my_ompss(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The ompss decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_ompss_existing_core_element(): + CONTEXT.set_master() + my_ompss = OmpSs(binary="date") + f = my_ompss(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_ompss.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_on_failure.py b/examples/dds/pycompss/tests/unittests/api/test_on_failure.py new file mode 100644 index 00000000..bfda9cbc --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_on_failure.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.on_failure import on_failure +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_on_failure_instantiation(): + CONTEXT.set_master() + my_on_failure = on_failure(management="IGNORE") + assert ( + my_on_failure.decorator_name == "@onfailure" + ), "The decorator name must be @onfailure: " + + +def test_on_failure_call(): + CONTEXT.set_master() + my_on_failure = on_failure(management="IGNORE") + f = my_on_failure(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_on_failure_unsupported_call(): + CONTEXT.set_master() + thrown = False + try: + _ = on_failure(management="UNDEFINED") + except Exception: # noqa + thrown = True + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The on_failure decorator did not raised an exception with unsupported management value." # noqa: E501 + + +def test_on_failure_call_outside(): + CONTEXT.set_out_of_scope() + my_on_failure = on_failure(management="IGNORE") + f = my_on_failure(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + + +def test_on_failure_existing_core_element(): + CONTEXT.set_master() + my_on_failure = on_failure(management="IGNORE") + f = my_on_failure(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_on_failure.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_opencl.py b/examples/dds/pycompss/tests/unittests/api/test_opencl.py new file mode 100644 index 00000000..3f251701 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_opencl.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.opencl import OpenCL +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_opencl_instantiation(): + CONTEXT.set_master() + my_opencl = OpenCL(kernel="date") + CONTEXT.set_out_of_scope() + assert ( + my_opencl.decorator_name == "@opencl" + ), "The decorator name must be @opencl." + + +def test_opencl_call(): + CONTEXT.set_master() + my_opencl = OpenCL(kernel="date") + f = my_opencl(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_opencl_call_outside(): + CONTEXT.set_out_of_scope() + my_opencl = OpenCL(kernel="date") + f = my_opencl(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert ( + thrown + ), "The opencl decorator did not raise an exception when invoked out of scope." # noqa: E501 + + +def test_opencl_existing_core_element(): + CONTEXT.set_master() + my_opencl = OpenCL(kernel="date") + f = my_opencl(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_opencl.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_prolog.py b/examples/dds/pycompss/tests/unittests/api/test_prolog.py new file mode 100644 index 00000000..cb5e0ad8 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_prolog.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.prolog import prolog +from pycompss.runtime.task.definitions.core_element import CE + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_prolog_instantiation(): + CONTEXT.set_master() + my_prolog = prolog(binary="date") + assert ( + my_prolog.decorator_name == "@prolog" + ), "The decorator name must be @prolog" + + +def test_prolog_call(): + CONTEXT.set_master() + my_prolog = prolog(binary="date") + f = my_prolog(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_prolog_call_outside(): + CONTEXT.set_out_of_scope() + my_prolog = prolog(binary="date") + f = my_prolog(dummy_function) + thrown = False + try: + _ = f() + except Exception: # noqa + thrown = True # this is OK! + CONTEXT.set_out_of_scope() + assert thrown, ( + "The prolog decorator did not raise an exception when " + "invoked out of scope." + ) + + +def test_prolog_existing_core_element(): + CONTEXT.set_master() + my_prolog = prolog(binary="date") + f = my_prolog(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_prolog.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/api/test_reduction.py b/examples/dds/pycompss/tests/unittests/api/test_reduction.py new file mode 100644 index 00000000..a7368f06 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/api/test_reduction.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + +from pycompss.util.context import CONTEXT +from pycompss.api.commons.decorator import CORE_ELEMENT_KEY +from pycompss.api.reduction import reduction +from pycompss.runtime.task.definitions.core_element import CE +from pycompss.util.exceptions import PyCOMPSsException + +CHUNK_SIZE_ERROR = "chunk_size is not defined in kwargs dictionary." +CHUNK_SIZE_NOT_INIT_ERROR = "chunk_size parameter has not been initialized." +EXPECTED_EXCEPTION_ERROR = "ERROR: Expected Exception not raised." + + +def dummy_function(*args, **kwargs): # noqa + return 1 + + +def test_reduction_instantiation(): + CONTEXT.set_master() + my_reduction = reduction() + assert ( + my_reduction.decorator_name == "@reduction" + ), "The decorator name must be @reduction." + CONTEXT.set_out_of_scope() + + +def test_reduction_call(): + CONTEXT.set_master() + my_reduction = reduction() + f = my_reduction(dummy_function) + result = f() + CONTEXT.set_out_of_scope() + assert result == 1, "Wrong expected result (should be 1)." + + +def test_reduction_call_outside(): + CONTEXT.set_out_of_scope() + my_reduction = reduction() + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + + +def test_reduction_chunk_size_parameter(): + CONTEXT.set_master() + chunk_size = 4 + my_reduction = reduction(chunk_size=chunk_size) + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert "chunk_size" in my_reduction.kwargs, CHUNK_SIZE_ERROR + assert ( + chunk_size == my_reduction.kwargs["chunk_size"] + ), CHUNK_SIZE_NOT_INIT_ERROR + + +def test_reduction_chunk_size_str_parameter(): + CONTEXT.set_master() + chunk_size = "4" + my_reduction = reduction(chunk_size=chunk_size) + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert "chunk_size" in my_reduction.kwargs, CHUNK_SIZE_ERROR + assert ( + int(chunk_size) == my_reduction.kwargs["chunk_size"] + ), CHUNK_SIZE_NOT_INIT_ERROR + + +def test_reduction_chunk_size_str_exception_parameter(): + CONTEXT.set_master() + chunk_size = "abc" + ok = False + try: + _ = reduction(chunk_size=chunk_size) + except PyCOMPSsException: + ok = True + CONTEXT.set_out_of_scope() + assert ok, EXPECTED_EXCEPTION_ERROR + + +def test_reduction_chunk_size_other_exception_parameter(): + CONTEXT.set_master() + chunk_size = [] + ok = False + try: + _ = reduction(chunk_size=chunk_size) + except PyCOMPSsException: + ok = True + CONTEXT.set_out_of_scope() + assert ok, EXPECTED_EXCEPTION_ERROR + + +def test_reduction_chunk_size_str_env_var_parameter(): + CONTEXT.set_master() + os.environ["MY_CHUNK_SIZE"] = "4" + chunk_size = "$MY_CHUNK_SIZE" + my_reduction = reduction(chunk_size=chunk_size) + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert "chunk_size" in my_reduction.kwargs, CHUNK_SIZE_ERROR + assert ( + int(os.environ[chunk_size[1:]]) == my_reduction.kwargs["chunk_size"] + ), CHUNK_SIZE_NOT_INIT_ERROR + + +def test_reduction_chunk_size_str_env_var_brackets_parameter(): + CONTEXT.set_master() + os.environ["MY_CHUNK_SIZE"] = "4" + chunk_size = "${MY_CHUNK_SIZE}" + my_reduction = reduction(chunk_size=chunk_size) + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert "chunk_size" in my_reduction.kwargs, CHUNK_SIZE_ERROR + assert ( + int(os.environ[chunk_size[2:-1]]) == my_reduction.kwargs["chunk_size"] + ), CHUNK_SIZE_NOT_INIT_ERROR + + +def test_reduction_chunk_size_str_env_var_exception_parameter(): + CONTEXT.set_master() + os.environ["MY_CHUNK_SIZE"] = "abc" + chunk_size = "$MY_CHUNK_SIZE" + ok = False + try: + _ = reduction(chunk_size=chunk_size) + except PyCOMPSsException: + ok = True + CONTEXT.set_out_of_scope() + assert ok, EXPECTED_EXCEPTION_ERROR + + +def test_reduction_is_reduce_parameter(): + CONTEXT.set_master() + is_reduce = False + my_reduction = reduction(is_reduce=is_reduce) + f = my_reduction(dummy_function) + _ = f() + CONTEXT.set_out_of_scope() + assert ( + "is_reduce" in my_reduction.kwargs + ), "is_reduce is not defined in kwargs dictionary." + assert ( + is_reduce == my_reduction.kwargs["is_reduce"] + ), "is_reduce parameter has not been initialized." + + +def test_reduction_existing_core_element(): + CONTEXT.set_master() + my_reduction = reduction() + f = my_reduction(dummy_function) + # a higher level decorator would place the compss core element as follows: + _ = f(compss_core_element=CE()) + CONTEXT.set_out_of_scope() + assert ( + CORE_ELEMENT_KEY not in my_reduction.kwargs + ), "Core Element is not defined in kwargs dictionary." diff --git a/examples/dds/pycompss/tests/unittests/dds/__init__.py b/examples/dds/pycompss/tests/unittests/dds/__init__.py new file mode 100644 index 00000000..4f7a5570 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the DDS unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/pickles/00000 b/examples/dds/pycompss/tests/unittests/dds/dataset/pickles/00000 new file mode 100644 index 00000000..133b5aac Binary files /dev/null and b/examples/dds/pycompss/tests/unittests/dds/dataset/pickles/00000 differ diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/1.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/1.txt new file mode 100644 index 00000000..4f58dbda --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/1.txt @@ -0,0 +1,21 @@ +41429356830947402,0.7496260276670429 +4201970245368552306,0.5245202951682624 +574567563519463045,0.54710200644667 +1226254430389334203,0.5593763292076754 +2994519650955480264,0.5372690590519902 +7896879200635324511,0.7255689472079304 +4863997140642197736,0.5369844433293645 +5827159886458313661,0.6165326638893279 +2064280025585992929,0.4542743623708815 +4958463666295645436,0.22738587977946112 +7649675870278871041,0.35488119714508115 +4245274666464863305,0.9850428880270804 +5551134503685135807,0.3944320878668499 +663627960438999579,0.33538336779513434 +6476950352471350642,0.3292511866952884 +1163648332752024339,0.9664451245731124 +2332646553154413280,0.27937295902820336 +3122824164808409102,0.1425226973626723 +1614187145001184443,0.7558532856073786 +7582277018532412239,0.15130712163003435 +2574719539510083918,0.8665567658601387 diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/2.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/2.txt new file mode 100644 index 00000000..7408cdf0 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/2.txt @@ -0,0 +1,41 @@ +8674967308402108661,0.7701515660034416 +2512515317776329999,0.5298157576825062 +4827922885830109717,0.8542642292038164 +9049624203309339979,0.9754283017483261 +7107059541148838804,0.28718932330910985 +2417670716240944041,0.022291937896435132 +1588814763467641697,0.8759715796434223 +685019607602713268,0.5836246145499416 +5654404383068681251,0.9600503578132014 +3770679156133323437,0.60492457053877 +7324262963491925487,0.37549208442210136 +1358256593048748882,0.3321176961849829 +7153221006077928579,0.9343522175276643 +1510149420417424630,0.763173144539664 +2550352749398877269,0.04275406351915878 +6789258707596638918,0.06142159221119081 +1958285888030650735,0.8599924679877035 +686986040506723990,0.5530743302930959 +5365645027831817815,0.04760034228762022 +2045064933725301098,0.7886563069832314 +901917754997388146,0.5267916016347636 +4071922571777198165,0.6066273175574454 +5244000790233053333,0.8736296643994086 +3154071694324224176,0.8104599133073889 +4357816233771263912,0.14824471446354914 +8395522958177990682,0.6836453252791227 +7737766603108240169,0.08298851191595302 +7891896125212214577,0.7254501460560986 +6390102541465729119,0.5937530471966516 +1694425450402191568,0.21802869598415497 +7965691915468783912,0.06991903472191596 +119874364727067593,0.6619863025374749 +2483901449623356953,0.7845130935008681 +622041666938801290,0.11546521985820268 +4476889118949236120,0.2300333382303006 +3352451074006120321,0.20114688001510783 +5413584076866390746,0.5193116760111183 +5693553505572431392,0.016534075133918913 +5185881089353187877,0.19162352544582484 +777386896185610733,0.9452558782471361 +7173071257256800598,0.7791155043815785 diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/3.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/3.txt new file mode 100644 index 00000000..ee72e0b5 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/3.txt @@ -0,0 +1,15 @@ +1170474835342764701,0.17202197648490114 +1862507474546007843,0.5680601508054569 +2358667960527279676,0.9927699584753775 +7621008117715220464,0.8891106961510301 +4519416478752878903,0.3925347088282415 +429925256565564335,0.024411855560488638 +6909603136154045405,0.7921267846126342 +8341986170742424151,0.2921887071854711 +8923046034878116813,0.5484661134788997 +5075721185024672596,0.7475361069739503 +942014138398425189,0.42381029087994326 +6227912894351990711,0.5031523782140853 +8677858391402478896,0.6028056316827486 +8330758764363680486,0.4352983482002448 +8061739803713224390,0.5932164293900186 diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/4.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/4.txt new file mode 100644 index 00000000..020da13e --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/terasort/4.txt @@ -0,0 +1,17 @@ +5031935913777591113,0.6480980926602797 +1844685366258146719,0.06570062954563227 +8736154513084582925,0.8378310529200694 +7778268886207639767,0.8785593277806847 +1332264270827272393,0.533228752672977 +3344804883036219935,0.8989293479996422 +3507173626651916051,0.8120608801053442 +6005057084422825188,0.2903724800516857 +3921100360022912833,0.09869276632539237 +5778704711098331669,0.0017955457908438444 +5175454174934360138,0.25534825661881677 +8423697846056763886,0.8164287374658094 +8620694610613749791,0.43214266954399383 +3119965677860962681,0.060266160900709353 +8272627662232138080,0.4756054551395986 +2509209393631485715,0.1950117638768989 +7369657272234869563,0.4214777201610571 diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/tmp/00000 b/examples/dds/pycompss/tests/unittests/dds/dataset/tmp/00000 new file mode 100644 index 00000000..a12dca2f --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/tmp/00000 @@ -0,0 +1 @@ +Risus magna potenti nibh ridiculus pharetra cursus proin tincidunt natoque libero habitasse purus vitae condimentum in.Velit purus libero sollicitudin.Velit augue odio imperdiet hac facilisis accumsan diam hac.Dolor risus lobortis per in pharetra.Velit lacus nascetur nibh est ad tortor pellentesque elementum etiam.Vitae lacus eni orci senectus sed nibh cursus tincidunt.Nulla vitae lacinia. Fusce augue lobortis leo litora nisi.Nulla class turpis neque inceptos urna semper.Class purus.Porta neque luctus libero est eu facilisi dis class amet proin cursus pellentesque phasellus dictum eu. \ No newline at end of file diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/1.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/1.txt new file mode 100644 index 00000000..79a09f3e --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/1.txt @@ -0,0 +1,15 @@ +Risus magna potenti nibh ridiculus pharetra cursus proin tincidunt natoque libero habitasse purus vitae condimentum in. +Velit purus libero sollicitudin. +Velit augue odio imperdiet hac facilisis accumsan diam hac. +Dolor risus lobortis per in pharetra. +Velit lacus nascetur nibh est ad tortor pellentesque elementum etiam. +Vitae lacus eni orci senectus sed nibh cursus tincidunt. +Nulla vitae lacinia. Fusce augue lobortis leo litora nisi. +Nulla class turpis neque inceptos urna semper.Class purus. +Porta neque luctus libero est eu facilisi dis class amet proin cursus pellentesque phasellus dictum eu. +Fames lorem netus fames magna urna hac ante nunc purus duis a velit dapibus cum. +Ipsum vitae. +Velit vitae pede mus fusce laoreet. +Purus risus sociis facilisis natoque porta cum. +Velit justo integer nonummy ligula est auctor habitasse donec habitasse. +Nulla lorem integer auctor mus leo dignissim turpis. \ No newline at end of file diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/2.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/2.txt new file mode 100644 index 00000000..bbcc32ab --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/2.txt @@ -0,0 +1,14 @@ +Vitae nulla leo nunc taciti molestie aliquam sociis. +Velit nulla. +Felis augue eleifend amet euismod. +Donec risus varius ac montes vehicula leo porta hac amet. +Class purus integer. +Risus nulla litora at mauris. +Fames lacus class magna. +Lorem porta montes ut luctus sodales dui praesent. +Velit magna. +Purus magna odio duis diam urna hac a aenean ante erat potenti ve. +Porta magna torquent netus magna mus ridiculus ultrices aptent congue. +Purus metus ullamcorper maecenas fames justo sodales cras id tincidunt dolor nulla suscipit amet. +Fames magna class nostra blandit nibh amet sed pellentesque class. +Felis morbi ultricies dui. Class fames est donec lorem neque. diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/3.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/3.txt new file mode 100644 index 00000000..da577fcc --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/3.txt @@ -0,0 +1,11 @@ +Velit curae augue per ultricies tempus arcu amet senectus lacinia rhoncus. +Ipsum purus mus elit. +Curae lorem varius metus vestibulum placerat vitae eros a eget. +Massa lorem cursus metus sagittis parturient. +Proin netus. +Risus fusce quis proin quisque facilisis. +Augue class. +Fusce magna sed amet est pellentesque. +Augue ipsum duis mus per nec sit. +Dolor netus. +Porta lorem praesent ve velit. Nulla morbi. Class netus ligula curae. \ No newline at end of file diff --git a/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/4.txt b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/4.txt new file mode 100644 index 00000000..c46a5a14 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/dataset/wordcount/4.txt @@ -0,0 +1,7 @@ +Velit purus morbi tellus ac risus et tincidunt a est malesuada. +Lacus morbi. +Vitae fames auctor dolor vestibulum sociosqu. +Morbi dolor metus massa sed. +Velit lorem primis lorem vel parturient ac duis augue. +Massa nulla. +Dolor morbi. \ No newline at end of file diff --git a/examples/dds/pycompss/tests/unittests/dds/test_dds_class.py b/examples/dds/pycompss/tests/unittests/dds/test_dds_class.py new file mode 100644 index 00000000..d231129f --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/dds/test_dds_class.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from pycompss.dds.dds import DDS + +GENERIC_ERROR = "ERROR: Unexpected result from DDS." + +current_path = os.path.dirname(os.path.abspath(__file__)) +dataset_dir = os.path.join(current_path, "dataset") +wc_dir = os.path.join(dataset_dir, "wordcount") +fayl = os.path.join(wc_dir, "1.txt") + + +def test_data_loaders(): + dds = DDS() + assert not dds.partitions, GENERIC_ERROR + + data = list(range(10)) + assert len(dds.load(data).collect()) == 10, GENERIC_ERROR + + assert DDS().load_file(fayl).collect(), GENERIC_ERROR + assert DDS().load_file(fayl, worker_read=True).collect(), GENERIC_ERROR + assert DDS().load_text_file(fayl).collect(), GENERIC_ERROR + assert DDS().load_files_from_dir(wc_dir).collect(), GENERIC_ERROR + + pickles_dir = os.path.join(dataset_dir, "pickles") + DDS().load_text_file(fayl).save_as_pickle(pickles_dir) + assert DDS().load_pickle_files(pickles_dir).collect() + + tmp_dir = os.path.join(dataset_dir, "tmp") + DDS().load_text_file(fayl).save_as_text_file(tmp_dir) + assert DDS().load_files_from_dir(tmp_dir).collect() + + +def test_methods(): + data = list(range(10)) + + dds = DDS().load(data).map(lambda x: x * 2).collect() + assert 18 in dds + + unified = DDS().load(data).union(DDS().load(data)).collect() + assert len(unified) == 20 + + dds = DDS().load(data).flat_map(lambda x: [x, x * 2]).collect() + assert 18 in dds + + dds = DDS().load(data).filter(lambda x: x > 5).collect() + assert len(dds) == 4 + + dds = DDS().load(data).reduce(lambda x, y: x + y) + assert dds == 45 + + dds = DDS().load(data).flat_map(lambda x: list(range(x))) + assert dds.count_by_value().get(0, 0) == 9 + + dds = ( + DDS() + .load([("a", 1), ("b", 3)]) + .join(DDS().load([("a", 2), ("b", 4)])) + .collect() + ) + assert ("a", (1, 2)) in dds + + dds = DDS().load(data).take(4) + assert len(dds) == 4 + + +def test_k_v_operations(): + data = list(range(10)) + dds = ( + DDS() + .load(data) + .map(lambda x: (x, x * 2)) + .map_values(lambda x: x / 2) + .collect() + ) + for i in dds: + assert i[0] == i[1] + + dds = ( + DDS() + .load(data, num_of_parts=2) + .map(lambda x: (x, x)) + .partition_by(lambda x: x % 2) + .collect(keep_partitions=True) + ) + for i in range(5): + assert abs(dds[0][i][0] - dds[1][i][0]) == 1 + + dds = ( + DDS() + .load([("a", [1, 2]), ("b", [1])]) + .flatten_by_key(lambda x: x) + .collect() + ) + assert len(dds) == 3 + + dds = ( + DDS() + .load([("z", 1), ("b", 3), ("a", 1), ("c", 3)]) + .sort_by_key() + .collect() + ) + assert dds[0][0] == "a" diff --git a/examples/dds/pycompss/tests/unittests/functions/__init__.py b/examples/dds/pycompss/tests/unittests/functions/__init__.py new file mode 100644 index 00000000..5753e9cf --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/functions/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the Functions unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/functions/test_data.py b/examples/dds/pycompss/tests/unittests/functions/test_data.py new file mode 100644 index 00000000..94fa2425 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/functions/test_data.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_data_chunks(): + from pycompss.functions.data import chunks + + data = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + expected_unbalanced = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + chunked_unbalanced = list(chunks(data, 3, balanced=False)) + assert ( + expected_unbalanced == chunked_unbalanced + ), "ERROR: Got unexpected unbalanced chunking." + expected_balanced = [[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]] + chunked_balanced = list(chunks(data, 3, balanced=True)) + assert ( + expected_balanced == chunked_balanced + ), "ERROR: Got unexpected balanced chunking." diff --git a/examples/dds/pycompss/tests/unittests/functions/test_elapsed_time.py b/examples/dds/pycompss/tests/unittests/functions/test_elapsed_time.py new file mode 100644 index 00000000..ba5daeb3 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/functions/test_elapsed_time.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_elapsed_time(): + from pycompss.functions.elapsed_time import timeit + + @timeit() + def increment(value): + import time + + time.sleep(0.1) + return value + 1 + + result = increment(1) + assert len(result) == 2, "ERROR: Time it does not retrieve two elements." + assert result[0] == 2, "ERROR: Got unexpected result." + assert isinstance(result[1], float), "ERROR: Time is in incorrect format." + assert result[1] > 0, "ERROR: Time can not be 0 or negative." diff --git a/examples/dds/pycompss/tests/unittests/functions/test_profile.py b/examples/dds/pycompss/tests/unittests/functions/test_profile.py new file mode 100644 index 00000000..20e36cce --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/functions/test_profile.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_profile(): + from pycompss.functions.profile import profile + + @profile() + def increment(value): + return value + 1 + + result = increment(1) + assert result == 2, "ERROR: Got unexpected result." diff --git a/examples/dds/pycompss/tests/unittests/functions/test_reduce.py b/examples/dds/pycompss/tests/unittests/functions/test_reduce.py new file mode 100644 index 00000000..c786e242 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/functions/test_reduce.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_merge_reduce(): + from pycompss.functions.reduce import merge_reduce + + data = list(range(11)) + + def accumulate(a, b): + return a + b + + result = merge_reduce(accumulate, data) + + assert result == 55, "ERROR: Got unexpected result with merge_reduce." + + +def test_merge_n_reduce(): + from pycompss.functions.reduce import merge_n_reduce + + data = list(range(11)) + + def accumulate(*args): + return sum(args) + + result = merge_n_reduce(accumulate, 5, data) + + assert result == 55, "ERROR: Got unexpected result with merge_n_reduce." diff --git a/examples/dds/pycompss/tests/unittests/runtime/__init__.py b/examples/dds/pycompss/tests/unittests/runtime/__init__.py new file mode 100644 index 00000000..b3daee73 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/runtime/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the Runtime unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/runtime/test_COMPSs.py b/examples/dds/pycompss/tests/unittests/runtime/test_COMPSs.py new file mode 100644 index 00000000..191192d0 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/runtime/test_COMPSs.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.exceptions import PyCOMPSsException + + +def test_is_redirected(): + # Get a copy of the initial status + old_stdout = COMPSs.stdout + old_stderr = COMPSs.stderr + # First case: Both not initialized -> False + COMPSs.stdout = "" + COMPSs.stderr = "" + none_none = COMPSs.is_redirected() + # Second case: Both initialized -> True + COMPSs.stdout = "file.out" + COMPSs.stderr = "file.err" + something_something = COMPSs.is_redirected() + # Third case: One not initialized -> Raise exception + COMPSs.stderr = "" + is_ok = False + try: + _ = COMPSs.is_redirected() + except PyCOMPSsException: + is_ok = True + assert ( + none_none is False + ), "ERROR: Failed first case of is_redirected. Must return False." + assert ( + something_something + ), "ERROR: Failed second case of is_redirected. Must return True." + assert ( + is_ok + ), "ERROR: Failed third case of is_redirected. Must raise an Exception." + # Restore status + COMPSs.stdout = old_stdout + COMPSs.stderr = old_stderr + + +def test_get_redirection(): + # Get a copy of the initial status + old_stdout = COMPSs.stdout + old_stderr = COMPSs.stdout + # First case: Both not initialized -> Raise exception + COMPSs.stdout = "" + COMPSs.stderr = "" + is_ok = False + try: + _, _ = COMPSs.get_redirection_file_names() + except PyCOMPSsException: + is_ok = True + # Second case: Both initialized -> out, err + out_name = "file.out" + err_name = "file.err" + COMPSs.stdout = out_name + COMPSs.stderr = err_name + new_stdout, new_stderr = COMPSs.get_redirection_file_names() + assert ( + is_ok + ), "ERROR: Failed first case of get_redirection_file_names. Must raise an Exception." # noqa: E501 + assert ( + new_stdout == out_name + ), "ERROR: Failed second case of get_redirection_file_names. Must return stdout file name." # noqa: E501 + assert ( + new_stderr == err_name + ), "ERROR: Failed second case of get_redirection_file_names. Must return stdout file name." # noqa: E501 + # Restore status + COMPSs.stdout = old_stdout + COMPSs.stderr = old_stderr diff --git a/examples/dds/pycompss/tests/unittests/runtime/test_core_element.py b/examples/dds/pycompss/tests/unittests/runtime/test_core_element.py new file mode 100644 index 00000000..5101e0c4 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/runtime/test_core_element.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.runtime.task.definitions.core_element import CE + +ERROR_SIGNATURE = "ERROR: Wrong signature value." +ERROR_IMPL_SIGNATURE = "ERROR: Wrong impl_signature value." +ERROR_IMPL_CONSTRAINTS = "ERROR: Wrong impl_constraints value." +ERROR_IMPL_TYPE = "ERROR: Wrong impl_type value." +ERROR_IMPL_LOCAL = "ERROR: Wrong impl_local value." +ERROR_IMPL_IO = "ERROR: Wrong impl_io value." +ERROR_IMPL_PROLOG = "ERROR: Wrong impl_prolog value." +ERROR_IMPL_EPILOG = "ERROR: Wrong impl_epilog value." +ERROR_IMPL_CONTAINER = "ERROR: Wrong impl_container value." +ERROR_IMPL_TYPE_ARGS = "ERROR: Wrong impl_type_args value." + + +def test_core_element(): + signature = "my_signature" + impl_signature = "my_impl_signature" + impl_constraints = {"impl_constraints": ""} + impl_type = "impl_type" + impl_local = False + impl_io = False + impl_prolog = ["impl_prolog"] + impl_epilog = ["impl_epilog"] + impl_container = ["", "", ""] + impl_type_args = ["impl_type_args"] + core_element = CE( + signature, + impl_signature, + impl_constraints, + impl_type, + impl_local, + impl_io, + impl_prolog, + impl_epilog, + impl_container, + impl_type_args, + ) + + # Check signature + result = core_element.get_ce_signature() + assert result == signature, ERROR_SIGNATURE + new_signature = "my_new_signature" + core_element.set_ce_signature(new_signature) + result = core_element.get_ce_signature() + assert result == new_signature, ERROR_SIGNATURE + + # Check impl_signature + result = core_element.get_impl_signature() + assert result == impl_signature, ERROR_IMPL_SIGNATURE + new_impl_signature = "my_new_impl_signature" + core_element.set_impl_signature(new_impl_signature) + result = core_element.get_impl_signature() + assert result == new_impl_signature, ERROR_IMPL_SIGNATURE + + # Check impl_constraints + result = core_element.get_impl_constraints() + assert result == impl_constraints, ERROR_IMPL_CONSTRAINTS + new_impl_constraints = {"my_new_impl_constraints": "value"} + core_element.set_impl_constraints(new_impl_constraints) + result = core_element.get_impl_constraints() + assert result == new_impl_constraints, ERROR_IMPL_CONSTRAINTS + + # Check impl_type + result = core_element.get_impl_type() + assert result == impl_type, ERROR_IMPL_TYPE + new_impl_type = "my_new_impl_type" + core_element.set_impl_type(new_impl_type) + result = core_element.get_impl_type() + assert result == new_impl_type, ERROR_IMPL_TYPE + + # Check impl_local + result = core_element.get_impl_local() + assert result == impl_local, ERROR_IMPL_LOCAL + new_impl_local = True + core_element.set_impl_local(new_impl_local) + result = core_element.get_impl_local() + assert result == new_impl_local, ERROR_IMPL_LOCAL + + # Check impl_io + result = core_element.get_impl_io() + assert result == impl_io, ERROR_IMPL_IO + new_impl_io = True + core_element.set_impl_io(new_impl_io) + result = core_element.get_impl_io() + assert result == new_impl_io, ERROR_IMPL_IO + + # Check impl_prolog + result = core_element.get_impl_prolog() + assert result == impl_prolog, ERROR_IMPL_PROLOG + new_impl_prolog = ["my_new_impl_prolog"] + core_element.set_impl_prolog(new_impl_prolog) + result = core_element.get_impl_prolog() + assert result == new_impl_prolog, ERROR_IMPL_PROLOG + + # Check impl_epilog + result = core_element.get_impl_epilog() + assert result == impl_epilog, ERROR_IMPL_EPILOG + new_impl_epilog = ["my_new_impl_epilog"] + core_element.set_impl_epilog(new_impl_epilog) + result = core_element.get_impl_epilog() + assert result == new_impl_epilog, ERROR_IMPL_EPILOG + + # Check impl_container + result = core_element.get_impl_container() + assert result == impl_container, ERROR_IMPL_CONTAINER + new_impl_container = ["my_new_impl_container", "", ""] + core_element.set_impl_container(new_impl_container) + result = core_element.get_impl_container() + assert result == new_impl_container, ERROR_IMPL_CONTAINER + + # Check impl_type_args + result = core_element.get_impl_type_args() + assert result == impl_type_args, ERROR_IMPL_TYPE_ARGS + new_impl_type_args = ["my_new_impl_type_args"] + core_element.set_impl_type_args(new_impl_type_args) + result = core_element.get_impl_type_args() + assert result == new_impl_type_args, ERROR_IMPL_TYPE_ARGS + + # Check representation + representation = core_element.__repr__() + assert isinstance( + representation, str + ), "ERROR: Received wrong representation type." # noqa: E501 + expected = ( + "CORE ELEMENT: \n" + "\t - CE signature : my_new_signature\n" + "\t - Impl. signature : my_new_impl_signature\n" + "\t - Impl. constraints: my_new_impl_constraints:value;\n" + "\t - Impl. type : my_new_impl_type\n" + "\t - Impl. local : True\n" + "\t - Impl. io : True\n" + "\t - Impl. prolog : ['my_new_impl_prolog']\n" + "\t - Impl. epilog : ['my_new_impl_epilog']\n" + "\t - Impl. container : ['my_new_impl_container', '', '']\n" + "\t - Impl. type args : ['my_new_impl_type_args']\n" + ) + assert representation == expected, "ERROR: Wrong representation." + + # Reset + core_element.reset() + + # Check again representation + representation = core_element.__repr__() + assert isinstance( + representation, str + ), "ERROR: Received wrong representation type." # noqa: E501 + expected = ( + "CORE ELEMENT: \n" + "\t - CE signature : \n" + "\t - Impl. signature : \n" + "\t - Impl. constraints: {}\n" + "\t - Impl. type : \n" + "\t - Impl. local : False\n" + "\t - Impl. io : False\n" + "\t - Impl. prolog : []\n" + "\t - Impl. epilog : []\n" + "\t - Impl. container : []\n" + "\t - Impl. type args : []\n" + ) + assert representation == expected, "ERROR: Wrong empty representation." diff --git a/examples/dds/pycompss/tests/unittests/runtime/test_management_direction.py b/examples/dds/pycompss/tests/unittests/runtime/test_management_direction.py new file mode 100644 index 00000000..76c7a74b --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/runtime/test_management_direction.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_get_compss_direction(): + from pycompss.api.parameter import DIRECTION + from pycompss.runtime.management.direction import get_compss_direction + + write = get_compss_direction("w") + assert ( + write == DIRECTION.OUT + ), "ERROR: Wrong w direction. Expected: OUT" # noqa: E501 + read_write = get_compss_direction("r+") + assert ( + read_write == DIRECTION.INOUT + ), "ERROR: Wrong r+ direction. Expected: INOUT" # noqa: E501 + append = get_compss_direction("a") + assert ( + append == DIRECTION.INOUT + ), "ERROR: Wrong a direction. Expected: INOUT" # noqa: E501 + concurrent = get_compss_direction("c") + assert ( + concurrent == DIRECTION.CONCURRENT + ), "ERROR: Wrong c direction. Expected: CONCURRENT" # noqa: E501 + commutative = get_compss_direction("cv") + assert ( + commutative == DIRECTION.COMMUTATIVE + ), "ERROR: Wrong cv direction. Expected: COMMUTATIVE" # noqa: E501 + read = get_compss_direction("OTHER") + assert ( + read == DIRECTION.IN + ), "ERROR: Wrong other direction. Expected: IN" # noqa: E501 diff --git a/examples/dds/pycompss/tests/unittests/runtime/test_object_tracker.py b/examples/dds/pycompss/tests/unittests/runtime/test_object_tracker.py new file mode 100644 index 00000000..bdc05235 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/runtime/test_object_tracker.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + +from pycompss.runtime.management.object_tracker import ObjectTracker + +ERROR_ID_NONE = "The identifier can not be None." +ERROR_ID_STRING = "The identifier must be a string." +ERROR_ID_EMPTY = "The identifier must not be empty." +ERROR_ID_DIFFERENT = ( + "Tracked identifier differs from returned by track function." # noqa: E501 +) +ERROR_FILENAME_EMPTY = "The file name can not be empty." + + +class DummyObject(object): + def __init__(self): + self.value = 1 + + +def test_track(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, do_file_name = object_tracker.track(do) + assert object_tracker.is_tracked(do) is not None, ERROR_ID_NONE + assert isinstance(object_tracker.is_tracked(do), str), ERROR_ID_STRING + assert object_tracker.is_tracked(do) != "", ERROR_ID_EMPTY + assert object_tracker.is_tracked(do) == do_id, ERROR_ID_DIFFERENT + assert do_file_name != "", ERROR_FILENAME_EMPTY + + +def test_track_twice(): + object_tracker = ObjectTracker() + do = DummyObject() + _ = object_tracker.track(do) + do_id, do_file_name = object_tracker.track(do) + assert object_tracker.is_tracked(do) is not None, ERROR_ID_NONE + assert isinstance(object_tracker.is_tracked(do), str), ERROR_ID_STRING + assert object_tracker.is_tracked(do) != "", ERROR_ID_EMPTY + assert object_tracker.is_tracked(do) == do_id, ERROR_ID_DIFFERENT + assert do_file_name != "", ERROR_FILENAME_EMPTY + + +def test_track_collection(): + object_tracker = ObjectTracker() + my_collection = [DummyObject(), DummyObject()] + collection_id, collection_file_name = object_tracker.track( + my_collection, collection=True + ) + assert object_tracker.is_tracked(my_collection) is not None, ERROR_ID_NONE + assert isinstance( + object_tracker.is_tracked(my_collection), str + ), ERROR_ID_STRING # noqa: E501 + assert ( + object_tracker.is_tracked(my_collection) != "" + and object_tracker.is_tracked(my_collection) != "None" + ), ERROR_ID_EMPTY + assert ( + object_tracker.is_tracked(my_collection) == collection_id + ), ERROR_ID_DIFFERENT # noqa: E501 + assert ( + collection_file_name != "" or collection_file_name != "None" + ), "The file name must be None for collections." + + +def test_stop_tracking(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, do_file_name = object_tracker.track(do) + assert ( + object_tracker.is_tracked(do) != "" + and object_tracker.is_tracked(do) != "None" # noqa: E501 + ), ERROR_ID_NONE + assert isinstance(object_tracker.is_tracked(do), str), ERROR_ID_STRING + assert ( + object_tracker.is_tracked(do) != "" + and object_tracker.is_tracked(do) != "None" # noqa: E501 + ), ERROR_ID_EMPTY + assert object_tracker.is_tracked(do) == do_id, ERROR_ID_DIFFERENT + # The object do is being tracked + object_tracker.stop_tracking(do) + assert ( + object_tracker.is_tracked(do) == "" + or object_tracker.is_tracked(do) == "None" # noqa: E501 + ), "The identifier must be None after stop tracking" + assert do_file_name != "", ERROR_FILENAME_EMPTY + + +def test_stop_tracking_collection(): + object_tracker = ObjectTracker() + my_collection = [DummyObject(), DummyObject()] + collection_id, collection_file_name = object_tracker.track( + my_collection, collection=True + ) + assert object_tracker.is_tracked(my_collection) != "", ERROR_ID_NONE + assert isinstance( + object_tracker.is_tracked(my_collection), str + ), ERROR_ID_STRING # noqa: E501 + assert ( + object_tracker.is_tracked(my_collection) != "" + and object_tracker.is_tracked(my_collection) != "None" + ), ERROR_ID_EMPTY + assert ( + object_tracker.is_tracked(my_collection) == collection_id + ), ERROR_ID_DIFFERENT # noqa: E501 + # The collection is being tracked + object_tracker.stop_tracking(my_collection, collection=True) + assert ( + object_tracker.is_tracked(my_collection) == "" + ), "The identifier must be None after stop tracking" + assert ( + collection_file_name == "None" + ), "The file name must be None for collections." # noqa: E501 + + +def test_get_object_id(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, do_file_name = object_tracker.track(do) + do_get_obj_id = object_tracker.get_object_id(do) + assert do_id == do_get_obj_id, "The object identifiers are different!" + assert do_file_name != "", ERROR_FILENAME_EMPTY + + +def test_not_tracking_empty(): + object_tracker = ObjectTracker() + do = DummyObject() + assert ( + object_tracker.is_tracked(do) == "" + ), "The object seems to be tracked." # noqa: E501 + + +def test_not_tracking_not_empty(): + object_tracker = ObjectTracker() + do = DummyObject() + _, _ = object_tracker.track(do) + do2 = DummyObject() + assert ( + object_tracker.is_tracked(do2) == "" + ), "The object seems to be tracked." # noqa: E501 + + +def test_get_all_file_names(): + object_tracker = ObjectTracker() + do = DummyObject() + do2 = DummyObject() + _, _ = object_tracker.track(do) + _, _ = object_tracker.track(do2) + file_names = object_tracker.get_all_file_names() + assert ( + len(file_names) == 2 + ), "Two elements should be being tracked: %s" % str( # noqa: E501 + file_names + ) + + +def test_get_file_name(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, do_file_name = object_tracker.track(do) + file_name = object_tracker.get_file_name(do_id) + assert file_name is not None, "The file name can not be None." + assert isinstance(file_name, str), "The file name must be a string." + assert file_name != "", "The file name must not be empty." + assert do_file_name != "", ERROR_FILENAME_EMPTY + assert do_file_name == file_name, "The file name received wrong file name." + + +def test_obj_not_pending_to_synchronize(): + object_tracker = ObjectTracker() + do = DummyObject() + # The object is being tracked + pending = object_tracker.is_obj_pending_to_synchronize(do) + assert pending is False, "The object should not be pending to synchronize." + + +def test_not_pending_to_synchronize(): + object_tracker = ObjectTracker() + # The object is being tracked + pending = object_tracker.is_pending_to_synchronize("IMPOSSIBLE_ID") + assert pending is False, "The object should not be pending to synchronize." + + +def test_obj_pending_to_synchronize(): + object_tracker = ObjectTracker() + do = DummyObject() + _, _ = object_tracker.track(do) + # The object is being tracked + pending = object_tracker.is_obj_pending_to_synchronize(do) + assert ( + pending is True + ), "The object must be pending to synchronize after tracking." # noqa: E501 + + +def test_pending_to_synchronize(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, _ = object_tracker.track(do) + # The object is being tracked + pending = object_tracker.is_pending_to_synchronize(do_id) + assert ( + pending is True + ), "The object must be pending to synchronize after tracking." # noqa: E501 + + +def test_update_mapping(): + object_tracker = ObjectTracker() + do = DummyObject() + do_id, _ = object_tracker.track(do) + # The object is being tracked + written = object_tracker.has_been_written(do_id) + assert ( + written is False + ), "The object identifier must not be in written_objects after tracking." # noqa: E501 + object_tracker.update_mapping(do_id, do) + new_id = object_tracker.get_object_id(do) + assert ( + do_id != new_id + ), "The identifiers must not be equal after updating the mapping." # noqa: E501 + written = object_tracker.has_been_written(new_id) + assert ( + written is True + ), "The object's new identifier must be in written_objects after updating its mapping." # noqa: E501 + file_name = object_tracker.pop_written_obj(new_id) + assert file_name is not None, "The object file name must not be None." + assert isinstance(file_name, str), "The object file name must be string." + assert file_name != "", "The object file name must not be empty." + written = object_tracker.has_been_written(new_id) + assert ( + written is False + ), "The object's new identifier must not be in written_objects after popping." # noqa: E501 + + +def test_clean_object_tracker(): + object_tracker = ObjectTracker() + do = DummyObject() + _, _ = object_tracker.track(do) + object_tracker.clean_object_tracker() + assert len(object_tracker.pending_to_synchronize) == 0 + assert len(object_tracker.file_names) == 0 + assert len(object_tracker.written_objects) == 0 + assert len(object_tracker.obj_id_to_obj) == 0 + assert len(object_tracker.address_to_obj_id) == 0 + + +def test_report(): + object_tracker = ObjectTracker() + object_tracker.enable_report() + assert ( + object_tracker.is_report_enabled() is True + ), "Reporting must be enabled." # noqa: E501 + do = DummyObject() + _, _ = object_tracker.track(do) + object_tracker.stop_tracking(do) + object_tracker.generate_report(".") + report = "object_tracker.png" + generated = os.path.exists(report) and os.path.isfile(report) + assert ( + generated is True + ), "Report result image has not been generated." # noqa: E501 + if generated: + os.remove(report) + object_tracker.clean_report() + assert ( + len(object_tracker.reporting_info) == 0 + ), "The reporting info list has not been cleared!" # noqa: E501 diff --git a/examples/dds/pycompss/tests/unittests/streams/__init__.py b/examples/dds/pycompss/tests/unittests/streams/__init__.py new file mode 100644 index 00000000..c4a544d2 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/streams/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the Stream unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/streams/test_distro_stream_client.py b/examples/dds/pycompss/tests/unittests/streams/test_distro_stream_client.py new file mode 100644 index 00000000..a376f04d --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/streams/test_distro_stream_client.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.streams.distro_stream import DistroStreamClientHandler +from pycompss.util.process.manager import create_process + + +def test_client_handler(): + """ + Tests the client handler with two different processes. + + :return: + """ + + def runner(): + print("Starting process") + print("Init Client Handler") + DistroStreamClientHandler.init_and_start("localhost", "49049") + print("Stop Client Handler") + DistroStreamClientHandler.set_stop() + print("End process") + + p1 = create_process(target=runner) + p2 = create_process(target=runner) + + p1.start() + p2.start() + + p1.join() + p2.join() diff --git a/examples/dds/pycompss/tests/unittests/util/__init__.py b/examples/dds/pycompss/tests/unittests/util/__init__.py new file mode 100644 index 00000000..40b0c52b --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the Utils unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/util/test_arguments.py b/examples/dds/pycompss/tests/unittests/util/test_arguments.py new file mode 100644 index 00000000..b4795c20 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_arguments.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import sys + +from pycompss.util.exceptions import PyCOMPSsException + +if sys.version_info <= (3, 0): + from cStringIO import StringIO +else: + from io import StringIO + +ERROR_UNEXPECTED_WARNING = "ERROR: Unexpected warning message received." +ERROR_UNEXPECTED_ERROR = "ERROR: Unexpected error message received." +ERROR_MISSING_WARNING = "ERROR: Could not find warning message." +ERROR_MISSING_ERROR = "ERROR: Could not find error message." +ERROR_EXCEPTION = "ERROR: Exception has not been raised" + + +def test_check_arguments_fine(): + from pycompss.util.arguments import check_arguments + + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = [ + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + ] + decorator = "Unittest" + + old_stderr = sys.stderr + sys.stderr = my_stderr = StringIO() + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + sys.stderr = old_stderr + assert "WARNING" not in my_stderr.getvalue(), ERROR_UNEXPECTED_WARNING + assert "ERROR" not in my_stderr.getvalue(), ERROR_UNEXPECTED_ERROR + + +def test_check_arguments_using_deprecated(): + from pycompss.util.arguments import check_arguments + + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = [ + "mandatory_argument_1", + "mandatoryArgument2", + "deprecated_argument_1", + ] + decorator = "Unittest" + + old_stderr = sys.stderr + sys.stderr = my_stderr = StringIO() + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + sys.stderr = old_stderr + assert "WARNING" in my_stderr.getvalue(), ERROR_MISSING_WARNING + assert "ERROR" not in my_stderr.getvalue(), ERROR_UNEXPECTED_ERROR + + +def test_check_arguments_missing_mandatory(): + from pycompss.util.arguments import check_arguments + + result = False + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = ["mandatoryArgument2"] + decorator = "Unittest" + + try: + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + except PyCOMPSsException: + # This is ok + result = True + assert result, ERROR_EXCEPTION + + +def test_check_arguments_missing_mandatory_no_underscore(): + from pycompss.util.arguments import check_arguments + + result = False + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = ["mandatory_argument_1"] + decorator = "Unittest" + + try: + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + except PyCOMPSsException: + # This is ok + result = True + assert result, ERROR_EXCEPTION + + +def test_check_arguments_unexpected(): + from pycompss.util.arguments import check_arguments + + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = [ + "mandatory_argument_1", + "mandatoryArgument2", + "unexpected_argument", + ] + decorator = "Unittest" + + old_stderr = sys.stderr + sys.stderr = my_stderr = StringIO() + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + sys.stderr = old_stderr + assert "WARNING" in my_stderr.getvalue(), ERROR_MISSING_WARNING + assert "ERROR" not in my_stderr.getvalue(), ERROR_UNEXPECTED_ERROR + + +def test_check_arguments_using_old_is_modifier(): + from pycompss.util.arguments import check_arguments + + result = False + mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} + deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} + supported_arguments = { + "mandatory_argument_1", + "mandatoryArgument2", + "optional_argument", + } + argument_names = [ + "mandatory_argument_1", + "mandatoryArgument2", + "isModifier", + ] + decorator = "Unittest" + + old_stderr = sys.stderr + sys.stderr = my_stderr = StringIO() + try: + check_arguments( + mandatory_arguments, + deprecated_arguments, + supported_arguments, + argument_names, + decorator, + ) + except PyCOMPSsException: + sys.stderr = old_stderr + assert "WARNING" not in my_stderr.getvalue(), ERROR_UNEXPECTED_WARNING + assert "ERROR" in my_stderr.getvalue(), ERROR_MISSING_ERROR + result = True + assert result, ERROR_EXCEPTION diff --git a/examples/dds/pycompss/tests/unittests/util/test_context.py b/examples/dds/pycompss/tests/unittests/util/test_context.py new file mode 100644 index 00000000..cd333172 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_context.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.context import CONTEXT + +PRE_CONTEXT_ERROR = "ERROR: The context was in pycompss (OUT_OF_SCOPE)." +MASTER_CONTEXT_ERROR = "ERROR: The context was not in pycompss (MASTER)." +WORKER_CONTEXT_ERROR = "ERROR: The context was not in pycompss (WORKER)." + + +def test_inmaster_context(): + CONTEXT.set_master() + master_context = CONTEXT.in_master() + assert master_context is True, MASTER_CONTEXT_ERROR + CONTEXT.set_out_of_scope() + + +def test_inworker_context(): + CONTEXT.set_worker() + worker_context = CONTEXT.in_worker() + assert worker_context is True, WORKER_CONTEXT_ERROR + CONTEXT.set_out_of_scope() + + +def test_in_pycompss_context(): + CONTEXT.set_master() + master_context = CONTEXT.in_pycompss() + CONTEXT.set_worker() + worker_context = CONTEXT.in_pycompss() + assert master_context is True, MASTER_CONTEXT_ERROR + assert worker_context is True, WORKER_CONTEXT_ERROR + CONTEXT.set_out_of_scope() + + +def test_who_contextualized(): + CONTEXT.set_master() + who = CONTEXT.get_who_contextualized() + assert ( + __name__ in who + or "None" in who + or "_callers" # callers when using mypy + ), "ERROR: Wrong who (%s) contextualized." % str(who) + CONTEXT.set_out_of_scope() + + +def test_get_context(): + CONTEXT.set_out_of_scope() + pre_context = CONTEXT.get_pycompss_context() + CONTEXT.set_master() + master_context = CONTEXT.get_pycompss_context() + CONTEXT.set_worker() + worker_context = CONTEXT.get_pycompss_context() + assert ( + pre_context == CONTEXT.out_of_scope + ), "ERROR: The context was not OUT_OF_SCOPE before setting" # noqa: E501 + assert ( + master_context == CONTEXT.master + ), "ERROR: The context was not in MASTER." + assert ( + worker_context == CONTEXT.worker + ), "ERROR: The context was not in WORKER." + CONTEXT.set_out_of_scope() + + +def test_enable_nesting(): + not_enabled = CONTEXT.is_nesting_enabled() + CONTEXT.enable_nesting() + is_enabled = CONTEXT.is_nesting_enabled() + CONTEXT.disable_nesting() + is_disabled = CONTEXT.is_nesting_enabled() + assert ( + not_enabled is False + ), "ERROR: Nesting must not be enabled by default." + assert is_enabled is True, "ERROR: Nesting has not been enabled." + assert is_disabled is False, "ERROR: Nesting has not been disabled." diff --git a/examples/dds/pycompss/tests/unittests/util/test_exceptions.py b/examples/dds/pycompss/tests/unittests/util/test_exceptions.py new file mode 100644 index 00000000..8b792751 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_exceptions.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +from pycompss.util.exceptions import DDSException +from pycompss.util.exceptions import MissingImplementedException +from pycompss.util.exceptions import NotImplementedException +from pycompss.util.exceptions import NotInPyCOMPSsException +from pycompss.util.exceptions import PyCOMPSsException + +GENERIC_MESSAGE = "Message to show" +GENERIC_MESSAGE_ERROR = "ERROR: Received unexpected exception message." + + +def test_pycompss_exception(): + try: + raise PyCOMPSsException(GENERIC_MESSAGE) + except Exception as e: # NOSONAR + is_ok = True + assert ( + str(e) == f"PyCOMPSs Exception: {GENERIC_MESSAGE}" + ), GENERIC_MESSAGE_ERROR + else: + is_ok = False + assert is_ok, "ERROR: The PyCOMPSsException has not been correctly raised" + + +def test_not_in_pycompss_exception(): + try: + raise NotInPyCOMPSsException(GENERIC_MESSAGE) + except Exception as e: # NOSONAR + is_ok = True + assert ( + str(e) == "Outside PyCOMPSs scope: " + GENERIC_MESSAGE + ), GENERIC_MESSAGE_ERROR + else: + is_ok = False + assert ( + is_ok + ), "ERROR: The NotInPyCOMPSsException has not been correctly raised" + + +def test_not_implemented_exception(): + try: + raise NotImplementedException(GENERIC_MESSAGE) + except Exception as e: # NOSONAR + is_ok = True + assert ( + str(e) + == "Functionality " + + GENERIC_MESSAGE + + " not implemented yet." # noqa: E501 + ), GENERIC_MESSAGE_ERROR + else: + is_ok = False + assert ( + is_ok + ), "ERROR: The NotImplementedException has not been correctly raised" + + +def test_missing_implemented_exception(): + try: + raise MissingImplementedException(GENERIC_MESSAGE) + except Exception as e: # NOSONAR + is_ok = True + assert ( + str(e) + == "Missing " + + GENERIC_MESSAGE + + ". Needs to be overridden." # noqa: E501 + ), GENERIC_MESSAGE_ERROR + else: + is_ok = False + assert ( + is_ok + ), "ERROR: The MissingImplementedException has not been correctly raised" + + +def test_dds_exception(): + try: + raise DDSException(GENERIC_MESSAGE) + except Exception as e: # NOSONAR + is_ok = True + assert ( + str(e) == f"DDS Exception: {GENERIC_MESSAGE}" + ), GENERIC_MESSAGE_ERROR + else: + is_ok = False + assert is_ok, "ERROR: The DDSException has not been correctly raised" diff --git a/examples/dds/pycompss/tests/unittests/util/test_jvm_parser.py b/examples/dds/pycompss/tests/unittests/util/test_jvm_parser.py new file mode 100644 index 00000000..e5ea9e48 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_jvm_parser.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import shutil +import tempfile + +from pycompss.util.exceptions import PyCOMPSsException + + +def test_jvm_parser(): + from pycompss.util.jvm.parser import convert_to_dict + + jvm_opt_file = tempfile.NamedTemporaryFile(delete=False).name + temp_folder = tempfile.mkdtemp() + jvm_expected_result = { + "+PerfDisableSharedMem": True, + "-UsePerfData": True, + "+UseG1GC": True, + "+UseThreadPriorities": True, + "ThreadPriorityPolicy=42": True, + "-Dlog4j.configurationFile": "/opt/COMPSs/Runtime/configuration/log/COMPSsMaster-log4j.debug", # noqa: E501 + "-Dcompss.to.file": "false", + "-Dcompss.project.file": "/opt/COMPSs/Runtime/configuration/xml/projects/default_project.xml", # noqa: E501 + "-Dcompss.resources.file": "/opt/COMPSs/Runtime/configuration/xml/resources/default_resources.xml", # noqa: E501 + "-Dcompss.project.schema": "/opt/COMPSs/Runtime/configuration/xml/projects/project_schema.xsd", # noqa: E501 + "-Dcompss.resources.schema": "/opt/COMPSs/Runtime/configuration/xml/resources/resources_schema.xsd", # noqa: E501 + "-Dcompss.lang": "python", + "-Dcompss.summary": "false", + "-Dcompss.task.execution": "compss", + "-Dcompss.storage.conf": "null", + "-Dcompss.streaming": "null", + "-Dcompss.streaming.masterName": "null", + "-Dcompss.streaming.masterPort": "null", + "-Dcompss.core.count": "50", + "-Dcompss.appName": "increment", + "-Dcompss.uuid": "dc126fe7-1b0a-4360-80f2-55c815e2e604", + "-Dcompss.baseLogDir": "", + "-Dcompss.specificLogDir": "", + "-Dcompss.appLogDir": temp_folder, + "-Dcompss.graph": "false", + "-Dcompss.monitor": "0", + "-Dcompss.tracing": "0", + "-Dcompss.extrae.file": "null", + "-Dcompss.comm": "es.bsc.compss.nio.master.NIOAdaptor", + "-Dcompss.conn": "es.bsc.compss.connectors.DefaultSSHConnector", + "-Dcompss.masterName": "", + "-Dcompss.masterPort": "", + "-Dcompss.scheduler": "es.bsc.compss.scheduler.lookahead.locality.LocalityTS", # noqa: E501 + "-Dgat.adaptor.path": "/opt/COMPSs/Dependencies/JAVA_GAT/lib/adaptors", + "-Dgat.debug": "true", + "-Dgat.broker.adaptor": "sshtrilead", + "-Dgat.file.adaptor": "sshtrilead", + "-Dcompss.worker.cp": "/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/opt/COMPSs/Runtime/compss-engine.jar::/opt/COMPSs/Runtime/compss-engine.jar", # noqa: E501 + "-Dcompss.worker.jvm_opts": "-Xms1024m,-Xmx1024m,-Xmn400m", + "-Dcompss.worker.cpu_affinity": "automatic", + "-Dcompss.worker.gpu_affinity": "automatic", + "-Dcompss.worker.fpga_affinity": "automatic", + "-Dcompss.worker.fpga_reprogram": "", + "-Dcompss.profile.input": "", + "-Dcompss.profile.output": "", + "-Dcompss.scheduler.config": "", + "-Dcompss.external.adaptation": "false", + "-Djava.class.path": "/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/opt/COMPSs/Runtime/compss-engine.jar::/opt/COMPSs/Runtime/compss-engine.jar", # noqa: E501 + "-Djava.library.path": "/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Runtime/compss-engine.jar:/usr/lib64/jvm/java-1.8.0/jre/lib/amd64/server/:/usr/lib64/mpi/gcc/openmpi/lib64/:/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Runtime/compss-engine.jar:/usr/lib64/jvm/java-1.8.0/jre/lib/amd64/server/:/usr/lib64/mpi/gcc/openmpi/lib64/:/usr/lib64/mpi/gcc/openmpi/lib64::/opt/COMPSs/Bindings/bindings-common/lib:/usr/lib64/jvm/java/jre/lib/amd64/server", # noqa: E501 + "-Dcompss.worker.pythonpath": "/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/home/user/gitlab/framework/compss/programming_model/bindings/python:.:/opt/COMPSs/Bindings/python/:/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Bindings/python/:/opt/COMPSs/Bindings/bindings-common/lib/:", # noqa: E501 + "-Dcompss.python.interpreter": "python3", + "-Dcompss.python.version": "3", + "-Dcompss.python.virtualenvironment": "null", + "-Dcompss.python.propagate_virtualenvironment": "true", + "-Dcompss.python.mpi_worker": "false", + "other": True, + } + with open(jvm_opt_file, "w") as f_jvm: + f_jvm.write( + """-XX:+PerfDisableSharedMem +-XX:-UsePerfData +-XX:+UseG1GC +-XX:+UseThreadPriorities +-XX:ThreadPriorityPolicy=42 +-Dlog4j.configurationFile=/opt/COMPSs/Runtime/configuration/log/COMPSsMaster-log4j.debug +-Dcompss.to.file=false +-Dcompss.project.file=/opt/COMPSs/Runtime/configuration/xml/projects/default_project.xml +-Dcompss.resources.file=/opt/COMPSs/Runtime/configuration/xml/resources/default_resources.xml +-Dcompss.project.schema=/opt/COMPSs/Runtime/configuration/xml/projects/project_schema.xsd +-Dcompss.resources.schema=/opt/COMPSs/Runtime/configuration/xml/resources/resources_schema.xsd +-Dcompss.lang=python +-Dcompss.summary=false +-Dcompss.task.execution=compss +-Dcompss.storage.conf=null +-Dcompss.streaming=null +-Dcompss.streaming.masterName=null +-Dcompss.streaming.masterPort=null +-Dcompss.core.count=50 +-Dcompss.appName=increment +-Dcompss.uuid=dc126fe7-1b0a-4360-80f2-55c815e2e604 +-Dcompss.baseLogDir= +-Dcompss.specificLogDir= +-Dcompss.appLogDir={0} +-Dcompss.graph=false +-Dcompss.monitor=0 +-Dcompss.tracing=0 +-Dcompss.extrae.file=null +-Dcompss.comm=es.bsc.compss.nio.master.NIOAdaptor +-Dcompss.conn=es.bsc.compss.connectors.DefaultSSHConnector +-Dcompss.masterName= +-Dcompss.masterPort= +-Dcompss.scheduler=es.bsc.compss.scheduler.lookahead.locality.LocalityTS +-Dgat.adaptor.path=/opt/COMPSs/Dependencies/JAVA_GAT/lib/adaptors +-Dgat.debug=true +-Dgat.broker.adaptor=sshtrilead +-Dgat.file.adaptor=sshtrilead +-Dcompss.worker.cp=/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/opt/COMPSs/Runtime/compss-engine.jar::/opt/COMPSs/Runtime/compss-engine.jar +-Dcompss.worker.jvm_opts=-Xms1024m,-Xmx1024m,-Xmn400m +-Dcompss.worker.cpu_affinity=automatic +-Dcompss.worker.gpu_affinity=automatic +-Dcompss.worker.fpga_affinity=automatic +-Dcompss.worker.fpga_reprogram= +-Dcompss.profile.input= +-Dcompss.profile.output= +-Dcompss.scheduler.config= +-Dcompss.external.adaptation=false +-Djava.class.path=/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/opt/COMPSs/Runtime/compss-engine.jar::/opt/COMPSs/Runtime/compss-engine.jar +-Djava.library.path=/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Runtime/compss-engine.jar:/usr/lib64/jvm/java-1.8.0/jre/lib/amd64/server/:/usr/lib64/mpi/gcc/openmpi/lib64/:/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Runtime/compss-engine.jar:/usr/lib64/jvm/java-1.8.0/jre/lib/amd64/server/:/usr/lib64/mpi/gcc/openmpi/lib64/:/usr/lib64/mpi/gcc/openmpi/lib64::/opt/COMPSs/Bindings/bindings-common/lib:/usr/lib64/jvm/java/jre/lib/amd64/server +-Dcompss.worker.pythonpath=/home/user/gitlab/framework/compss/programming_model/bindings/python/src/pycompss/tests/runtime/../resources:/home/user/gitlab/framework/compss/programming_model/bindings/python:.:/opt/COMPSs/Bindings/python/:/opt/COMPSs/Bindings/bindings-common/lib/:/opt/COMPSs/Bindings/python/:/opt/COMPSs/Bindings/bindings-common/lib/: +-Dcompss.python.interpreter=python3 +-Dcompss.python.version=3 +-Dcompss.python.virtualenvironment=null +-Dcompss.python.propagate_virtualenvironment=true +-Dcompss.python.mpi_worker=false +other +""".format( + temp_folder + ) # noqa + ) + result = convert_to_dict(jvm_opt_file) + assert len(result) == len( + jvm_expected_result + ), "The sizes of the dictionaries does not match" + for k, v in jvm_expected_result.items(): + if k not in result: + raise PyCOMPSsException( + "Key: %s is not in the result dictionary" % k + ) + assert ( + v == result[k] + ), "The value of key: %s does not match the expected value: %s" % ( + k, + str(v), + ) + assert ( + result == jvm_expected_result + ), "The jvm opts file has not been parsed as expected" + os.remove(jvm_opt_file) + shutil.rmtree(temp_folder) diff --git a/examples/dds/pycompss/tests/unittests/util/test_mpi_helper.py b/examples/dds/pycompss/tests/unittests/util/test_mpi_helper.py new file mode 100644 index 00000000..6a682434 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_mpi_helper.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_mpi_helper(): + try: + from pycompss.util.mpi.helper import rank_distributor + except AttributeError: + raise Exception("UNSUPPORTED WITH MYPY") + + result = rank_distributor((2, 3, 4)) + expected_result = [0, 1, 2, 4, 5, 6] + assert result == expected_result, "Unexpected rank distributor result" diff --git a/examples/dds/pycompss/tests/unittests/util/test_object_replace.py b/examples/dds/pycompss/tests/unittests/util/test_object_replace.py new file mode 100644 index 00000000..fb77fd40 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_object_replace.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +class MyClass(object): + def __init__(self, value, content, message): + self.value = value + self.content = content + self.message = message + + +def test_object_replace(): + try: + from pycompss.util.objects.replace import replace + except ImportError: + raise Exception("UNSUPPORTED WITH MYPY") + + o = MyClass(1, [1, 2, 3, 4], "hello world!") + p = MyClass(100, [40, 30, 20, 10], "goodbye world!") + + assert id(o) != id(p), "ERROR: The objects have the same identifier." + + replace(o, p) + + assert id(o) == id( + p + ), "ERROR: The objects do not have the same identifier." + + +# # Commented out due to incompatibility with mypy +# def test_replace_main(): +# from pycompss.util.objects.replace import examine_vars +# from pycompss.util.objects.replace import a +# from pycompss.util.objects.replace import U +# from pycompss.util.objects.replace import S +# from pycompss.util.objects.replace import replace +# from pycompss.util.objects.replace import b +# from pycompss.util.objects.replace import V +# from pycompss.util.objects.replace import T +# # Does the same as __main__ +# examine_vars(id(a), id(U), id(S)) +# print("-" * 35) +# replace(a, b) +# replace(U, V) +# replace(S, T) +# print("-" * 35) diff --git a/examples/dds/pycompss/tests/unittests/util/test_object_sizer.py b/examples/dds/pycompss/tests/unittests/util/test_object_sizer.py new file mode 100644 index 00000000..f6fb24a7 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_object_sizer.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import sys + + +class MyClass(object): + def __init__(self): + self.value = 1 + self.content = [1, 2, 3, 4] + self.message = "message" + self.more = [ + {"a": 12345, "b": 54321, "c": 10000}, + 1, + True, + [1, 2, 3, 4], + "test", + ] + + +def test_object_sizer(): + from pycompss.util.objects.sizer import total_sizeof + + o = MyClass() + system_size = sys.getsizeof(o) + real_size = total_sizeof(o, handlers={list: iter}, verbose=True) + assert real_size > system_size, "Failed checking the object size" diff --git a/examples/dds/pycompss/tests/unittests/util/test_objects.py b/examples/dds/pycompss/tests/unittests/util/test_objects.py new file mode 100644 index 00000000..a112c749 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_objects.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + + +def test_group_iterable(): + from pycompss.util.objects.util import group_iterable + + iterable = list(range(10)) + n = 2 + expected = [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)] + result = list(group_iterable(iterable, n)) + assert result == expected, "ERROR: Wrong grouping." diff --git a/examples/dds/pycompss/tests/unittests/util/test_serializer.py b/examples/dds/pycompss/tests/unittests/util/test_serializer.py new file mode 100644 index 00000000..b46d4170 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_serializer.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys + +import dill +import numpy +import numpy as np + +from pycompss.tests.outlog import create_logger + +if sys.version_info >= (3, 0): + import pickle as pickle # Uses _pickle if available +else: + import cPickle as pickle # noqa + +LOGGER = create_logger() + + +def test_get_serializer_priority(): + from pycompss.util.serialization.serializer import get_serializer_priority + + priority = get_serializer_priority([1, 2, 3], logger=LOGGER) + assert priority == [pickle, dill], "ERROR: Received wrong priority." + + priority = get_serializer_priority(np.random.rand(2, 2), LOGGER) + assert priority == [ + numpy, + pickle, + dill, + ], "ERROR: Received wrong priority with numpy object." + + +def test_get_serializers(): + from pycompss.util.serialization.serializer import get_serializer_priority + + serializers = get_serializer_priority(LOGGER, logger=LOGGER) + assert serializers == [ + pickle, + dill, + ], "ERROR: Received wrong serializers. " + str(serializers) + + +def test_serialize_deserialize_obj_to_file(): + # Uses serialize to handler underneath. + from pycompss.util.serialization.serializer import serialize_to_file + from pycompss.util.serialization.serializer import deserialize_from_file + + target_file = "target.pkl" + obj = [1, 3, 2, "hello", "world"] + serialize_to_file(obj, target_file, logger=LOGGER) + result = deserialize_from_file(target_file, logger=LOGGER) + os.remove(target_file) + assert ( + obj == result + ), "ERROR: Object serialization and deserialization retrieved wrong object." # noqa: E501 + + +def test_serialize_deserialize_np_to_file(): + # Uses serialize to handler underneath. + from pycompss.util.serialization.serializer import serialize_to_file + from pycompss.util.serialization.serializer import deserialize_from_file + + target_file_np = "target_np.pkl" + obj_np = np.random.rand(4, 4) + serialize_to_file(obj_np, target_file_np, logger=LOGGER) + result_np = deserialize_from_file(target_file_np, logger=LOGGER) + os.remove(target_file_np) + assert np.array_equal( + obj_np, result_np + ), "ERROR: Numpy object serialization and deserialization retrieved wrong object." # noqa: E501 + + +def test_serialize_deserialize_obj_to_file_no_gc(): + # Uses serialize to handler underneath. + import pycompss.util.serialization.serializer as serializer + from pycompss.util.serialization.serializer import serialize_to_file + from pycompss.util.serialization.serializer import deserialize_from_file + + serializer.DISABLE_GC = True + target_file = "target.pkl" + obj = [1, 3, 2, "hello", "world"] + serialize_to_file(obj, target_file, logger=LOGGER) + result = deserialize_from_file(target_file, logger=LOGGER) + os.remove(target_file) + assert ( + obj == result + ), "ERROR: Object serialization and deserialization (without garbage collector) retrieved wrong object." # noqa: E501 + + +def test_serialize_deserialize_np_to_file_no_gc(): + # Uses serialize to handler underneath. + import pycompss.util.serialization.serializer as serializer + from pycompss.util.serialization.serializer import serialize_to_file + from pycompss.util.serialization.serializer import deserialize_from_file + + serializer.DISABLE_GC = True + target_file_np = "target_np.pkl" + obj_np = np.random.rand(4, 4) + serialize_to_file(obj_np, target_file_np, logger=LOGGER) + result_np = deserialize_from_file(target_file_np, logger=LOGGER) + os.remove(target_file_np) + assert np.array_equal( + obj_np, result_np + ), "ERROR: Numpy object serialization and deserialization (without garbage collector) retrieved wrong object." # noqa: E501 + + +# def test_serialize_deserialize_string(): +# obj = ["hello", 1, "world", 2, [5, 4, 3, 2, 1], None] +# from pycompss.util.serialization.serializer import serialize_to_bytes +# from pycompss.util.serialization.serializer import deserialize_from_bytes +# +# serialized = serialize_to_bytes(obj) +# result = deserialize_from_bytes(serialized) +# assert ( +# result == obj +# ), "ERROR: Serialization and deserialization to/from string retrieved wrong object." # noqa: E501 + + +def test_serialize_objects(): + from pycompss.util.serialization.serializer import serialize_objects + from pycompss.util.serialization.serializer import deserialize_from_file + + obj1 = ([1, 2, 3, 4], "obj1.pkl") + obj2 = ({"hello": "mars", "goodbye": "world"}, "obj2.pkl") + obj3 = (np.random.rand(3, 3), "obj3.pkl") + objects = [obj1, obj2, obj3] + serialize_objects(objects, logger=LOGGER) + result = [] + for obj in objects: + result.append(deserialize_from_file(obj[1], logger=LOGGER)) + os.remove(obj1[1]) + os.remove(obj2[1]) + os.remove(obj3[1]) + assert len(result) == len( + objects + ), "ERROR: Wrong number of objects retrieved." + assert result[0] == obj1[0], "ERROR: Wrong first object." + assert result[1] == obj2[0], "ERROR: Wrong second object." + assert np.array_equal(result[2], obj3[0]), "ERROR: Wrong third object." diff --git a/examples/dds/pycompss/tests/unittests/util/test_supercomputer.py b/examples/dds/pycompss/tests/unittests/util/test_supercomputer.py new file mode 100644 index 00000000..9f09eb81 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_supercomputer.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os + + +def test_get_master_node(): + from pycompss.util.supercomputer.scs import get_master_node + + master_node = "my_master_node" + os.environ["COMPSS_MASTER_NODE"] = master_node + result = get_master_node() + del os.environ["COMPSS_MASTER_NODE"] + assert result == master_node, "ERROR: Wrong master node." + + +def test_get_master_port(): + from pycompss.util.supercomputer.scs import get_master_port + + master_port = "1234" + os.environ["COMPSS_MASTER_PORT"] = master_port + result = get_master_port() + del os.environ["COMPSS_MASTER_PORT"] + assert result == master_port, "ERROR: Wrong master port." + + +def test_get_worker_nodes(): + from pycompss.util.supercomputer.scs import get_worker_nodes + + worker_nodes = "my_worker_nodes" + os.environ["COMPSS_WORKER_NODES"] = worker_nodes + result = get_worker_nodes() + del os.environ["COMPSS_WORKER_NODES"] + assert result == worker_nodes, "ERROR: Wrong worker nodes." + + +def test_get_xmls(): + from pycompss.util.supercomputer.scs import get_xmls + + project = "my_project.xml" + resources = "my_resources.xml" + os.environ["COMPSS_PROJECT_XML"] = project + os.environ["COMPSS_RESOURCES_XML"] = resources + result_project, result_resources = get_xmls() + del os.environ["COMPSS_PROJECT_XML"] + del os.environ["COMPSS_RESOURCES_XML"] + assert result_project == project, "ERROR: Wrong project XML." + assert result_resources == resources, "ERROR: Wrong resources XML." + + +def test_get_uuid(): + from pycompss.util.supercomputer.scs import get_uuid + + uuid = "my_uuid" + os.environ["COMPSS_UUID"] = uuid + result = get_uuid() + del os.environ["COMPSS_UUID"] + assert result == uuid, "ERROR: Wrong UUID." + + +def test_get_log_dir(): + from pycompss.util.supercomputer.scs import get_log_dir + + log_dir = "my_log_dir" + os.environ["COMPSS_LOG_DIR"] = log_dir + result = get_log_dir() + del os.environ["COMPSS_LOG_DIR"] + assert result == log_dir, "ERROR: Wrong log directory." + + +def test_get_master_working_dir(): + from pycompss.util.supercomputer.scs import get_master_working_dir + + master_working_dir = "my_master_working_dir" + os.environ["COMPSS_MASTER_WORKING_DIR"] = master_working_dir + result = get_master_working_dir() + del os.environ["COMPSS_MASTER_WORKING_DIR"] + assert ( + result == master_working_dir + ), "ERROR: Wrong master working directory." + + +def test_get_log_level(): + from pycompss.util.supercomputer.scs import get_log_level + + log_level = "my_log_level" + os.environ["COMPSS_LOG_LEVEL"] = log_level + result = get_log_level() + del os.environ["COMPSS_LOG_LEVEL"] + assert result == log_level, "ERROR: Wrong log level." + + +def test_get_tracing(): + from pycompss.util.supercomputer.scs import get_tracing + + tracing = "true" + os.environ["COMPSS_TRACING"] = tracing + result = get_tracing() + assert result, "ERROR: Expected tracing enabled." + tracing = "fakse" + os.environ["COMPSS_TRACING"] = tracing + result = get_tracing() + del os.environ["COMPSS_TRACING"] + assert not result, "ERROR: Expected tracing disabled." + + +def test_get_storage_conf(): + from pycompss.util.supercomputer.scs import get_storage_conf + + storage_conf = "my_storage_conf" + os.environ["COMPSS_STORAGE_CONF"] = storage_conf + result = get_storage_conf() + del os.environ["COMPSS_STORAGE_CONF"] + assert result == storage_conf, "ERROR: Wrong storage conf." diff --git a/examples/dds/pycompss/tests/unittests/util/test_warning_modules.py b/examples/dds/pycompss/tests/unittests/util/test_warning_modules.py new file mode 100644 index 00000000..eef425c8 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/util/test_warning_modules.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys + +from pycompss.util.exceptions import PyCOMPSsException + + +def test_get_optional_module_warning(): + from pycompss.util.warnings.modules import get_optional_module_warning + + warning = get_optional_module_warning( + "UNITTEST_NAME", "UNITTEST_DESCRIPTION" + ) + assert isinstance( + warning, str + ), "Optional module warning does NOT return a string" + assert warning != "", "Optional module warning can not be empty" + assert ( + "UNITTEST_NAME" in warning + ), "Module name not in optional module warning" + assert ( + "UNITTEST_DESCRIPTION" in warning + ), "Module description not in optional module warning" + + +def test_show_optional_module_warning(): + import pycompss.util.warnings.modules as warn + + # Hack - Add non existing package + warn.OPTIONAL_MODULES["non_existing_package"] = "this is the description" + stdout_backup = sys.stdout + out_file = "warning.out" + fd = open(out_file, "w") + sys.stdout = fd + warn.show_optional_module_warnings() + # Cleanup + sys.stdout = stdout_backup + fd.close() + del warn.OPTIONAL_MODULES["non_existing_package"] + # Result check + if os.path.exists(out_file) and os.path.getsize(out_file) > 0: + # Non empty file exists - this is ok. + os.remove(out_file) + else: + raise PyCOMPSsException("The warning has not been shown") diff --git a/examples/dds/pycompss/tests/unittests/worker/__init__.py b/examples/dds/pycompss/tests/unittests/worker/__init__.py new file mode 100644 index 00000000..d2c91554 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/worker/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the Workers unittests.""" diff --git a/examples/dds/pycompss/tests/unittests/worker/test_container_worker.py b/examples/dds/pycompss/tests/unittests/worker/test_container_worker.py new file mode 100644 index 00000000..410eb74b --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/worker/test_container_worker.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys +import tempfile + +from pycompss.api.exceptions import COMPSsException +from pycompss.api.task import task +from pycompss.tests.outlog import create_logger +from pycompss.util.serialization.serializer import serialize_to_file +from pycompss.worker.container.container_worker import main + +CONTAINER_WORKER = "container_worker.py" +LOGGER = create_logger() + + +@task() +def simple(): + # Do nothing task + pass + + +@task() +def simple_compss_exception(): + raise COMPSsException("On purpose exception") + + +@task() +def simple_exception(): + nom = 10 + denom = 0 + _ = nom / denom # throws exception + + +@task(returns=1) +def increment(value): + return value + 1 + + +def test_container_worker_simple_task(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + sys.argv = [ + CONTAINER_WORKER, + "test_container_worker", + "simple", + "debug", + "false", + "null", + "0", + "0", + "0", + "0", + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + + +def test_container_worker_increment_task(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + temp_file = tempfile.NamedTemporaryFile(delete=False).name + temp_file_name = os.path.basename(temp_file) + serialize_to_file(1, temp_file, logger=LOGGER) + sys.argv = [ + CONTAINER_WORKER, + "test_container_worker", + "increment", + "true", + "false", + "false", + "10", + "1", + "2", + "10", + "3", + "#", + "$return_0", + "FILE", + "null:" + temp_file_name + ":false:true:" + temp_file, + "4", + "3", + "null", + "value", + "#UNDEFINED#:#UNDEFINED#", + "1", + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + os.remove(temp_file) + + +def test_container_worker_simple_task_compss_exception(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + sys.argv = [ + CONTAINER_WORKER, + "test_container_worker", + "simple_compss_exception", + "debug", + "false", + "null", + "0", + "0", + "0", + "0", + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + exit_code = main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + assert ( + exit_code == 2 + ), "Wrong exit code received (expected 2, received %s)" % str(exit_code) + + +def test_container_worker_simple_task_exception(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + sys.argv = [ + CONTAINER_WORKER, + "test_container_worker", + "simple_exception", + "debug", + "false", + "null", + "0", + "0", + "0", + "0", + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + exit_code = main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + assert ( + exit_code == 1 + ), "Wrong exit code received (expected 1, received %s)" % str(exit_code) diff --git a/examples/dds/pycompss/tests/unittests/worker/test_external_mpi.py b/examples/dds/pycompss/tests/unittests/worker/test_external_mpi.py new file mode 100644 index 00000000..6dcb093b --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/worker/test_external_mpi.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys +import tempfile + +from pycompss.api.task import task +from pycompss.util.exceptions import PyCOMPSsException + +using_mypy = False +try: + from pycompss.worker.external.mpi_executor import main +except AttributeError: + using_mypy = True + + +@task() +def simple(): + # Do nothing task + pass + + +@task(returns=1) +def increment(value): + return value + 1 + + +def test_external_mpi_worker_simple_task(): + if using_mypy: + raise Exception("UNSUPPORTED WITH MYPY") + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + job1_out = tempfile.NamedTemporaryFile(delete=False).name + job1_err = tempfile.NamedTemporaryFile(delete=False).name + working_dir = os.getcwd() + sys.argv = [ + "test_external_mpi.py", + " ".join( + [ + "EXECUTE_TASK", + "1", + working_dir, + job1_out, + job1_err, + "0", + "1", + "true", + "null", + "METHOD", + "test_external_mpi", + "simple", + "0", + "1", + "1", + "localhost", + "1", + "false", + "None", + "0", + "0", + "-", + "0", + "0", + ] + ), + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + check_task(job1_out, job1_err) + os.remove(job1_out) + os.remove(job1_err) + + +def test_external_mpi_worker_increment_task(): + if using_mypy: + raise Exception("UNSUPPORTED WITH MYPY") + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + job2_out = tempfile.NamedTemporaryFile(delete=False).name + job2_err = tempfile.NamedTemporaryFile(delete=False).name + job2_result = tempfile.NamedTemporaryFile(delete=False).name + working_dir = os.getcwd() + sys.argv = [ + "test_external_mpi.py", + " ".join( + [ + "EXECUTE_TASK", + "2", + working_dir, + job2_out, + job2_err, + "0", + "1", + "true", + "null", + "METHOD", + "test_external_mpi", + "increment", + "0", + "1", + "1", + "localhost", + "1", + "false", + "10", + "1", + "2", + "4", + "3", + "null", + "value", + "null", + "1", + "10", + "3", + "#", + "$return_0", + "null", + job2_result + + ":d1v2_1599560599402.IT:false:true:" + + job2_result, # noqa: E501 + "-", + "0", + "0", + ] + ), + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + check_task(job2_out, job2_err) + os.remove(job2_out) + os.remove(job2_err) + os.remove(job2_result) + + +def check_task(job_out, job_err): + if os.path.exists(job_err) and os.path.getsize(job_err) > 0: # noqa + # Non empty file exists + raise PyCOMPSsException( + "An error happened in the task. Please check " + job_err + ) + with open(job_out, "r") as f: + content = f.read() + if "ERROR" in content: + raise PyCOMPSsException( + "An error happened in the task. Please check " + job_out + ) + if "EXCEPTION" in content or "Exception" in content: + raise PyCOMPSsException( + "An exception happened in the task. Please check " + job_out + ) + if "END TASK execution. Status: Ok" not in content: + raise PyCOMPSsException( + "The task was supposed to be OK. Please check " + job_out + ) diff --git a/examples/dds/pycompss/tests/unittests/worker/test_gat.py b/examples/dds/pycompss/tests/unittests/worker/test_gat.py new file mode 100644 index 00000000..91a5a0f1 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/worker/test_gat.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import os +import sys +import tempfile + +from pycompss.api.task import task +from pycompss.worker.gat.worker import main + + +@task() +def simple(): + # Do nothing task + pass + + +@task(returns=1) +def increment(value): + return value + 1 + + +def test_gat_worker_simple_task(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + sys.argv = [ + "worker.py", + "false", + "1", + "true", + "null", + "NONE", + "localhost", + "49049", + "METHOD", + "test_gat", + "simple", + "1", + "0", + "0", + "1", + "false", + "null", + "0", + "0", + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + + +def test_gat_worker_increment_task(): + # Override sys.argv to mimic runtime call + sys_argv_backup = list(sys.argv) + sys_path_backup = list(sys.path) + temp_file = tempfile.NamedTemporaryFile(delete=False).name + sys.argv = [ + "worker.py", + "false", + "1", + "true", + "null", + "NONE", + "localhost", + "49049", + "METHOD", + "test_gat", + "increment", + "1", + "0", + "0", + "1", + "false", + "null", + "1", + "2", + "4", + "3", + "null", + "value", + "null", + "1", + "10", + "3", + "#", + "$return_0", + "null", + temp_file, + ] + current_path = os.path.dirname(os.path.abspath(__file__)) + sys.path.append(current_path) + main() + sys.argv = sys_argv_backup + sys.path = sys_path_backup + os.remove(temp_file) diff --git a/examples/dds/pycompss/tests/unittests/worker/test_piper_worker_cache.py b/examples/dds/pycompss/tests/unittests/worker/test_piper_worker_cache.py new file mode 100644 index 00000000..ad3e15f7 --- /dev/null +++ b/examples/dds/pycompss/tests/unittests/worker/test_piper_worker_cache.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +import sys + +from pycompss.worker.piper.cache.setup import is_cache_enabled + +NOT_PYTHON_3_8 = ( + "WARNING: Could not perform cache test since python " + "version is lower than 3.8" +) + + +def test_is_cache_enabled(): + if sys.version_info >= (3, 8): + case1 = is_cache_enabled("true") + assert case1, "Unexpected return. Expected: True" + case2 = is_cache_enabled("True") + assert case2, "Unexpected return. Expected: True" + case3 = is_cache_enabled("true:1000") + assert case3, "Unexpected return. Expected: True" + case4 = is_cache_enabled("True:1000") + assert case4, "Unexpected return. Expected: True" + else: + print(NOT_PYTHON_3_8) + + +# def test_piper_worker_cache(): +# if sys.version_info >= (3, 8): +# # Initiate cache +# smm, cache_process, cache_queue, cache_ids = start_cache(logging, +# "Default", +# False, +# "") +# load_shared_memory_manager() +# # Supported types: +# np_obj = np.random.rand(4) +# np_obj_name = "np_obj" +# list_obj = [1, 2, 3, 4, 5, "hi"] +# list_obj_name = "list_obj_name" +# tuple_obj = ("1", 2, 3, "4", "hi") +# tuple_obj_name = "tuple_obj_name" +# # Check insertions +# insert_object_into_cache_wrapper(logging, cache_queue, np_obj, np_obj_name, np_obj_name, None) # noqa: E501 +# insert_object_into_cache_wrapper(logging, cache_queue, list_obj, list_obj_name, list_obj_name, None) # noqa: E501 +# insert_object_into_cache_wrapper(logging, cache_queue, tuple_obj, tuple_obj_name, tuple_obj_name, None) # noqa: E501 +# # Check retrieves +# np_obj_new, np_obj_shm = retrieve_object_from_cache(logging, cache_ids, cache_queue, np_obj_name, np_obj_name, None, False) # noqa: E501 +# list_obj_new, list_obj_shm = retrieve_object_from_cache(logging, cache_ids, cache_queue, list_obj_name, list_obj_name, None, False) # noqa: E501 +# tuple_obj_new, tuple_obj_shm = retrieve_object_from_cache(logging, cache_ids, cache_queue, tuple_obj_name, tuple_obj_name, None, False) # noqa: E501 +# assert ( +# set(np_obj_new).intersection(np_obj) +# ), "ERROR: Numpy object retrieved from cache differs from inserted" +# assert ( +# set(list_obj_new).intersection(list_obj) +# ), "ERROR: List retrieved from cache differs from inserted" +# assert ( +# set(tuple_obj_new).intersection(tuple_obj) +# ), "ERROR: Tuple retrieved from cache differs from inserted" +# # Check replace +# new_list_obj = ["hello", "world", 6] +# replace_object_into_cache(logging, cache_queue, new_list_obj, list_obj_name, list_obj_name, False) # noqa: E501 +# time.sleep(0.5) +# list_obj_new2, list_obj_shm2 = retrieve_object_from_cache(logging, cache_ids, cache_queue, list_obj_name, list_obj_name, None, False) # noqa: E501 +# assert ( +# set(list_obj_new2).intersection(new_list_obj) +# ), "ERROR: List retrieved from cache differs from inserted" +# # Remove object +# remove_object_from_cache(logging, cache_queue, list_obj_name) +# time.sleep(0.5) +# is_ok = False +# try: +# _, _ = retrieve_object_from_cache(logging, cache_ids, cache_queue, list_obj_name, list_obj_name, None, False) # noqa: E501 +# except Exception: # NOSONAR +# is_ok = True +# assert ( +# is_ok +# ), "ERROR: List has not been removed." +# # Check if in cache +# assert ( +# in_cache(np_obj_name, cache_ids) +# ), "ERROR: numpy object not in cache. And it should be." +# assert ( +# not in_cache(list_obj_name, cache_ids) +# ), "ERROR: list object should not be in cache." +# assert ( +# not in_cache(list_obj_name, {}) +# ), "ERROR: in cache should return False if dict is empty." +# # Stop cache +# stop_cache(smm, cache_queue, False, cache_process) +# else: +# print(NOT_PYTHON_3_8) +# +# +# def test_piper_worker_cache_stress(): +# if sys.version_info >= (3, 8): +# # Initiate cache +# smm, cache_process, cache_queue, cache_ids = start_cache(logging, +# "true:100", +# False, +# "") +# load_shared_memory_manager() +# # Create multiple objects: +# amount = 40 +# np_objs = [np.random.rand(4) for _ in range(amount)] +# np_objs_names = ["name" + str(i) for i in range(amount)] +# # Check insertions +# for i in range(amount): +# insert_object_into_cache_wrapper(logging, cache_queue, +# np_objs[i], np_objs_names[i], +# np_objs_names[i], None) +# # Stop cache +# stop_cache(smm, cache_queue, False, cache_process) +# else: +# print(NOT_PYTHON_3_8) +# +# +# def test_piper_worker_cache_reuse(): +# if sys.version_info >= (3, 8): +# # Initiate cache +# smm, cache_process, cache_queue, cache_ids = start_cache(logging, +# "true:100000", +# False, +# "") +# load_shared_memory_manager() +# # Create multiple objects and store with the same name: +# amount = 10 +# np_objs = [np.random.rand(4) for _ in range(amount)] +# obj_name = "name" +# np_objs_names = [obj_name for _ in range(amount)] +# # Check insertions +# for i in range(amount): +# insert_object_into_cache_wrapper(logging, cache_queue, +# np_objs[i], np_objs_names[i], +# np_objs_names[i], None) +# if obj_name not in cache_ids: +# raise Exception("Object " + obj_name + +# " not found in cache_ids.") +# else: +# if cache_ids[obj_name][4] != 9: +# raise Exception("Wrong number of hits!!!") +# # Stop cache +# stop_cache(smm, cache_queue, False, cache_process) +# else: +# print(NOT_PYTHON_3_8) diff --git a/examples/dds/pycompss/util/__init__.py b/examples/dds/pycompss/util/__init__.py new file mode 100644 index 00000000..d6ce2401 --- /dev/null +++ b/examples/dds/pycompss/util/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/arguments.py b/examples/dds/pycompss/util/arguments.py new file mode 100644 index 00000000..7c13a72c --- /dev/null +++ b/examples/dds/pycompss/util/arguments.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Arguments. + +This file contains the common methods to do any argument (used in a decorator) +check or management. +""" + +from __future__ import print_function + +import re +import sys + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + + +def check_arguments( + mandatory_arguments: typing.Set[str], + deprecated_arguments: typing.Set[str], + supported_arguments: typing.Set[str], + argument_names: typing.List[str], + decorator: str, +) -> None: + """Perform all needed checks to the decorator definition. + + The checks are: + 1.- Checks that the mandatory arguments are present (otherwise, raises + an exception). + 2.- Looks for unexpected arguments (displays a warning through stderr). + + :param mandatory_arguments: Set of mandatory argument names. + :param deprecated_arguments: Set of deprecated argument names. + :param supported_arguments: Set of supported argument names. + :param argument_names: List of argument names to check. + :param decorator: String - Decorator name. + :return: None. + """ + decorator_str = f"{decorator} decorator" + # Look for mandatory arguments + check_mandatory_arguments( + mandatory_arguments, argument_names, decorator_str + ) + # Look for deprecated arguments + __check_deprecated_arguments( + deprecated_arguments, argument_names, decorator_str + ) + # Look for unexpected arguments + __check_unexpected_arguments( + supported_arguments, argument_names, decorator_str + ) + + +def check_mandatory_arguments( + mandatory_arguments: typing.Set[str], + argument_names: typing.List[str], + where: str, +) -> None: + """Check that all mandatory arguments are in arguments. + + :param mandatory_arguments: Set of supported arguments. + :param argument_names: List of arguments to check. + :param where: Location of the argument. + :return: None. + """ + for argument in mandatory_arguments: + if "_" in argument: + if ( + argument not in argument_names + and __to_camel_case(argument) not in argument_names + ): + # The mandatory argument or it converted to camel case is + # not in the arguments + __error_mandatory_argument(where, argument) + else: + if argument not in argument_names: + # The mandatory argument is not in the arguments + __error_mandatory_argument(where, argument) + + +def __to_camel_case(argument: str) -> str: + """Convert the given argument to camel case. + + :param argument: String to convert to camel case. + :return: Camel case string. + """ + components = argument.split("_") + return components[0] + "".join(x.title() for x in components[1:]) + + +def __error_mandatory_argument(decorator: str, argument: str) -> None: + """Raise an exception when the argument is mandatory in the decorator. + + :param argument: Argument name. + :param decorator: Decorator name. + :return: None. + :raise PyCOMPSsException: With the decorator and argument that produced + the error. + """ + raise PyCOMPSsException( + f"The argument {str(argument)} is mandatory " + f"in the {str(decorator)} decorator." + ) + + +def __check_deprecated_arguments( + deprecated_arguments: typing.Set[str], + argument_names: typing.List[str], + where: str, +) -> None: + """Look for deprecated arguments and displays a warning if found. + + :param deprecated_arguments: Set of deprecated arguments. + :param argument_names: List of arguments to check. + :param where: Location of the argument. + :return: None. + :raise PyCOMPSsException: With the unsupported argument. + """ + for argument in argument_names: + if argument == "isModifier": + message = ( + f"ERROR: Unsupported argument: isModifier Found " + f"in {str(where)}.\n" + " Please, use: target_direction" + ) + print(message, file=sys.stderr) # also show the warn in stderr + raise PyCOMPSsException(f"Unsupported argument: {str(argument)}") + + if argument in deprecated_arguments: + current_argument = re.sub("([A-Z]+)", r"_\1", argument).lower() + message = ( + f"WARNING: Deprecated argument: {str(argument)} Found " + f"in {str(where)}.\n" + f" Please, use: {current_argument}" + ) + + # The print through stdout is disabled to prevent the message to + # appear twice in the console. So the warning message will only + # appear in STDERR. + # print(message) # show the warn through stdout + print(message, file=sys.stderr) # also show the warn in stderr + + +def __check_unexpected_arguments( + supported_arguments: typing.Set[str], + argument_names: typing.List[str], + where: str, +) -> None: + """Look for unexpected arguments and displays a warning if found. + + :param supported_arguments: Set of supported arguments. + :param argument_names: List of arguments to check. + :param where: Location of the argument. + :return: None. + """ + for argument in argument_names: + if argument not in supported_arguments: + message = ( + f"WARNING: Unexpected argument: {str(argument)} Found " + f"in {str(where)}." + ) + # The print through stdout is disabled to prevent the message to + # appear twice in the console. So the warning message will only + # appear in STDERR. + # print(message) # show the warn through stdout + print(message, file=sys.stderr) # also show the warn in stderr diff --git a/examples/dds/pycompss/util/context.py b/examples/dds/pycompss/util/context.py new file mode 100644 index 00000000..cfd533e5 --- /dev/null +++ b/examples/dds/pycompss/util/context.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Context. + +This file contains the methods to detect the origin of the call stack. +Useful to detect if we are in the master or in the worker. +""" +import inspect +from contextlib import contextmanager + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +################# +# CONTEXT CLASS # +################# + + +class Context: + """Keep the context variables.""" + + __slots__ = [ + "master", + "worker", + "out_of_scope", + "who", + "where", + "nesting", + "loading", + "to_register", + ] + + def __init__(self) -> None: + """Instantiate the context class.""" + # Final variables + self.master = "MASTER" + self.worker = "WORKER" + self.out_of_scope = "OUT_OF_SCOPE" + # Context definition variables + self.who = self.out_of_scope + self.where = self.out_of_scope + # Context status or features + self.nesting = False + self.loading = False + self.to_register = [] # type: typing.List[typing.Any] + + def in_master(self) -> bool: + """Determine if the execution is performed in the master node. + + :return: True if in master. False on the contrary. + """ + return self.where == self.master + + def in_worker(self) -> bool: + """Determine if the execution is performed in a worker node. + + :return: True if in worker. False on the contrary. + """ + return self.where == self.worker + + def in_pycompss(self) -> bool: + """Determine if the execution is performed within the PyCOMPSs scope. + + :return: True if under scope. False on the contrary. + """ + return self.where != self.out_of_scope + + def __set_pycompss_context__(self, where: str) -> None: + """Set the Python Binding context (MASTER or WORKER or OUT_OF_SCOPE). + + :param where: New context (MASTER or WORKER or OUT_OF_SCOPE). + :return: None. + """ + self.where = where + if __debug__: + # Only check who contextualized if debugging + # 2 since it is called from set_master, + # set_worker or set_out_of_scope. + try: + caller_stack = inspect.stack()[2] + except IndexError: + # Within mypy + caller_stack = inspect.stack()[1] + caller_module = inspect.getmodule(caller_stack[0]) + self.who = str(caller_module) + + def set_master(self) -> None: + """Set the context to master. + + :return: None. + """ + self.__set_pycompss_context__(self.master) + + def set_worker(self) -> None: + """Set the context to worker. + + :return: None. + """ + self.__set_pycompss_context__(self.worker) + + def set_out_of_scope(self) -> None: + """Set the context to out of scope (not master nor worker). + + Usually for initialization or other tasks that are shared between + master and worker operation. + + :return: None. + """ + self.__set_pycompss_context__(self.out_of_scope) + + def get_pycompss_context(self) -> str: + """Return the PyCOMPSs context name. + + * For debugging purposes. + + :return: PyCOMPSs context name. + """ + return self.where + + def get_who_contextualized(self) -> str: + """Return the PyCOMPSs contextualization caller. + + * For debugging purposes. + + :return: PyCOMPSs contextualization caller name + :raises PyCOMPSsException: Unsupported function if not debugging. + """ + if __debug__: + return self.who + raise PyCOMPSsException( + "Get who contextualized only works in debug mode." + ) + + def is_nesting_enabled(self) -> bool: + """Check if nesting is enabled. + + :returns: None. + """ + return self.nesting is True + + def enable_nesting(self) -> None: + """Enable nesting. + + :returns: None. + """ + self.nesting = True + + def disable_nesting(self) -> None: + """Disable nesting. + + :returns: None. + """ + self.nesting = False + + def is_loading(self) -> bool: + """Check if is loading. + + :returns: None + """ + return self.loading is True + + def enable_loading(self) -> None: + """Enable loading. + + :returns: None. + """ + self.loading = True + + def disable_loading(self) -> None: + """Enable loading. + + :returns: None. + """ + self.loading = False + + def add_to_register_later( + self, core_element: typing.Tuple[typing.Any, str] + ) -> None: + """Accumulate core elements to be registered later. + + :param core_element: Core element to be registered. + :return: None. + """ + self.to_register.append(core_element) + + def get_to_register(self) -> typing.List[typing.Tuple[typing.Any, str]]: + """Retrieve the to register list. + + :return: To register list. + """ + return self.to_register + + +CONTEXT = Context() + + +############# +# FUNCTIONS # +############# + + +@contextmanager +def loading_context() -> typing.Iterator[None]: + """Context which sets the loading mode. + + Intended to be used only with the @implements decorators, since they + try to register on loading. + + :return: None. + """ + CONTEXT.enable_loading() + yield + CONTEXT.disable_loading() diff --git a/examples/dds/pycompss/util/environment/__init__.py b/examples/dds/pycompss/util/environment/__init__.py new file mode 100644 index 00000000..02077fea --- /dev/null +++ b/examples/dds/pycompss/util/environment/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the environment utils helpers. + +Where the helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/util/environment/configuration.py b/examples/dds/pycompss/util/environment/configuration.py new file mode 100644 index 00000000..66579945 --- /dev/null +++ b/examples/dds/pycompss/util/environment/configuration.py @@ -0,0 +1,916 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Environment - Configuration. + +This file contains the configurator methods. +Currently, it is used by interactive.py and launch.py. +""" + +import base64 +import json +import os +import pathlib +import sys +import uuid as _uuid +import re +from tempfile import mkstemp + +from pycompss.runtime.task.features import TASK_FEATURES +from pycompss.util.supercomputer.scs import get_log_dir +from pycompss.util.supercomputer.scs import get_log_level +from pycompss.util.supercomputer.scs import get_master_node +from pycompss.util.supercomputer.scs import get_master_port +from pycompss.util.supercomputer.scs import get_master_working_dir +from pycompss.util.supercomputer.scs import get_storage_conf +from pycompss.util.supercomputer.scs import get_tracing +from pycompss.util.supercomputer.scs import get_uuid +from pycompss.util.supercomputer.scs import get_xmls +from pycompss.util.typing_helper import typing + +DEFAULT_PROJECT_PATH = "/Runtime/configuration/xml/projects/" +DEFAULT_RESOURCES_PATH = "/Runtime/configuration/xml/resources/" +DEFAULT_LOG_PATH = "/Runtime/configuration/log/" +DEFAULT_TRACING_PATH = "/Runtime/configuration/xml/tracing/" + + +def preload_user_code() -> bool: + """Check if the user code has to be preloaded or not. + + Check if the user code has to be preloaded before starting the runtime + or has been disabled by the user through environment variable. + + :return: True if preload. False otherwise. + """ + environment_variable_load = "COMPSS_LOAD_SOURCE" + if environment_variable_load not in os.environ: + return True + if ( + environment_variable_load in os.environ + and os.environ[environment_variable_load] != "false" + ): + return True + return False + + +def export_current_flags(all_vars: dict) -> None: + """Export current flags to PYCOMPSS_CURRENT_FLAGS environment variable. + + Save all vars in global environment variable current flags so that + events.py can restart the notebook with the same flags. + Removes b" and " to avoid issues with javascript. + + :param all_vars: Dictionary containing all flags. + :return: None. + """ + environment_variable_current_flags = "PYCOMPSS_CURRENT_FLAGS" + all_flags = str(base64.b64encode(json.dumps(all_vars).encode()))[2:-1] + os.environ[environment_variable_current_flags] = all_flags + + +def prepare_environment( + interactive: bool, + o_c: bool, + storage_impl: str, + app: str, + debug: bool, + mpi_worker: bool, +) -> dict: + """Set up the environment variable and retrieve their content. + + :param interactive: True | False If the environment is interactive or not. + :param o_c: Object conversion to string. + :param storage_impl: Storage implementation. + :param app: Application name. + :param debug: True | False If debug is enabled. + :param mpi_worker: True | False if mpi worker is enabled. + :return: Dictionary which contains the compss_home, pythonpath, classpath, + ld_library_path, cp, extrae_home, extrae_lib and file_name values. + """ + launch_path = os.path.dirname(os.path.realpath(__file__)) + + if "COMPSS_HOME" in os.environ: + compss_home = os.environ["COMPSS_HOME"] + else: + compss_home = "" + if interactive: + # compss_home = launch_path without the last 6 folders: + # (Bindings/python/version/pycompss/util/environment) + compss_home = os.path.sep.join(launch_path.split(os.path.sep)[:-6]) + os.environ["COMPSS_HOME"] = compss_home + + # Grab the existing PYTHONPATH, CLASSPATH and LD_LIBRARY_PATH environment + # variables values + pythonpath = os.getcwd() + ":." + if "PYTHONPATH" in os.environ: + pythonpath += ":" + os.environ["PYTHONPATH"] + classpath = "" + if "CLASSPATH" in os.environ: + classpath = os.environ["CLASSPATH"] + ld_library_path = "" + if "LD_LIBRARY_PATH" in os.environ: + ld_library_path = os.environ["LD_LIBRARY_PATH"] + + # Enable/Disable object to string conversion + # set cross-module variable + TASK_FEATURES.set_object_conversion(o_c) + + # Get the filename and its path. + file_name = os.path.splitext(os.path.basename(app))[0] + cp = os.path.dirname(app) + if interactive: + # Rename file_name and cp + file_name = "Interactive" + cp = os.getcwd() + "/" + + # Set storage classpath + if storage_impl != "": + if storage_impl == "redis": + cp = ( + cp + + ":" + + compss_home + + "/Tools/storage/redis/compss-redisPSCO.jar" + ) + else: + cp = cp + ":" + storage_impl + + # Set extrae dependencies + if "EXTRAE_HOME" not in os.environ: + # It can be defined by the user or by launch_compss when running + # in Supercomputer + extrae_home = compss_home + "/Dependencies/extrae" + os.environ["EXTRAE_HOME"] = extrae_home + else: + extrae_home = os.environ["EXTRAE_HOME"] + + extrae_lib = extrae_home + "/lib" + os.environ["LD_LIBRARY_PATH"] = extrae_lib + ":" + ld_library_path + os.environ["EXTRAE_USE_POSIX_CLOCK"] = "0" + + control_binding_commons_debug(debug) + + env_vars = { + "compss_home": compss_home, + "pythonpath": pythonpath, + "classpath": classpath, + "ld_library_path": ld_library_path, + "cp": cp, + "extrae_home": extrae_home, + "extrae_lib": extrae_lib, + "file_name": file_name, + "mpi_worker": mpi_worker, + } + return env_vars + + +def control_binding_commons_debug(debug: bool) -> None: + """Enable the binding-commons debug mode. + + :returns: None. + """ + if debug: + # Add environment variable to get binding-commons debug information + os.environ["COMPSS_BINDINGS_DEBUG"] = "1" + + +def prepare_loglevel_graph_for_monitoring( + monitor: typing.Union[None, int], graph: bool, debug: bool, log_level: str +) -> dict: + """Check if monitor is enabled and updates graph and log level. + + If monitor is True, then the log_level and graph are set to debug. + + :param monitor: Monitor refresh frequency. None if disabled. + :param graph: True | False If graph is enabled or disabled. + :param debug: True | False If debug is enabled or disabled. + :param log_level: Defined log level. + :return: Dictionary containing the updated monitor, graph and log_level + values. + """ + if monitor != -1: + # Enable the graph if the monitoring is enabled + graph = True + # Set log level info + log_level = "info" + + if debug: + # If debug is enabled, the output is more verbose + log_level = "debug" + + monitoring_vars = { + "monitor": monitor, + "graph": graph, + "log_level": log_level, + } + return monitoring_vars + + +def updated_variables_in_sc() -> dict: + """Retrieve the updated variable values within SCs. + + :return: Dictionary containing the updated variables (project_xml, + resources_xml, master_name, master_port, uuid, log_dir, + master_working_dir, storage_conf, log_level, debug and trace). + """ + # Since the deployment in supercomputers is done through the use of + # enqueue_compss and consequently launch_compss - the project and resources + # xmls are already created + project_xml, resources_xml = get_xmls() + # It also exported some environment variables that we need here + master_name = get_master_node() + master_port = get_master_port() + uuid = get_uuid() + log_dir = get_log_dir() + master_working_dir = get_master_working_dir() + storage_conf = get_storage_conf() + # Override debug considering the parameter defined in + # pycompss_interactive_sc script and exported by launch_compss + log_level = get_log_level() + debug = log_level == "debug" + # Override tracing considering the parameter defined in + # pycompss_interactive_sc script and exported by launch_compss + trace = get_tracing() + updated_vars = { + "project_xml": project_xml, + "resources_xml": resources_xml, + "master_name": master_name, + "master_port": master_port, + "uuid": uuid, + "log_dir": log_dir, + "master_working_dir": master_working_dir, + "storage_conf": storage_conf, + "log_level": log_level, + "debug": debug, + "trace": trace, + } + return updated_vars + + +def prepare_tracing_environment( + trace: bool, extrae_lib: str, ld_library_path: str +) -> str: + """Prepare the environment for tracing. + + Also retrieves the appropriate trace value for the initial configuration + file (which is an integer). + + :param trace: [ True | False ] Tracing mode. + :param extrae_lib: Extrae lib path. + :param ld_library_path: LD_LIBRARY_PATH environment content + :return: Trace mode (as integer) + """ + if trace is True: + ld_library_path = ld_library_path + ":" + extrae_lib + return ld_library_path + + +def check_infrastructure_variables( + project_xml: str, + resources_xml: str, + compss_home: str, + app_name: str, + file_name: str, + external_adaptation: bool, +) -> dict: + """Check the infrastructure variables and updates them if None. + + :param project_xml: Project xml file path (None if not defined). + :param resources_xml: Resources xml file path (None if not defined). + :param compss_home: Compss home path. + :param app_name: Application name (if None, it changes it with filename). + :param file_name: Application file name. + :param external_adaptation: External adaptation. + :return: Updated variables (project_xml, resources_xml, app_name, + external_adaptation, major_version, + python_interpreter, python_version and + python_virtual_environment). + """ + if project_xml == "": + project_xml = ( + compss_home + DEFAULT_PROJECT_PATH + "default_project.xml" + ) + if resources_xml == "": + resources_xml = ( + compss_home + DEFAULT_RESOURCES_PATH + "default_resources.xml" + ) + app_name = file_name if app_name == "" else app_name + external_adaptation_str = "true" if external_adaptation else "false" + major_version = str(sys.version_info[0]) + python_interpreter = "python" + major_version + python_version = major_version + # Check if running within a virtual environment + if "VIRTUAL_ENV" in os.environ: + python_virtual_environment = os.environ["VIRTUAL_ENV"] + elif "CONDA_DEFAULT_ENV" in os.environ: + python_virtual_environment = os.environ["CONDA_DEFAULT_ENV"] + else: + python_virtual_environment = "null" + inf_vars = { + "project_xml": project_xml, + "resources_xml": resources_xml, + "app_name": app_name, + "external_adaptation": external_adaptation_str, + "major_version": major_version, + "python_interpreter": python_interpreter, + "python_version": python_version, + "python_virtual_environment": python_virtual_environment, + } + return inf_vars + + +def create_init_config_file( + compss_home: str, + debug: bool, + log_level: str, + project_xml: str, + resources_xml: str, + summary: bool, + task_execution: str, + storage_conf: str, + streaming_backend: str, + streaming_master_name: str, + streaming_master_port: str, + task_count: int, + app_name: str, + uuid: str, + log_dir: str, + master_working_dir: str, + graph: bool, + monitor: int, + trace: int, + extrae_cfg: str, + extrae_final_directory: str, + comm: str, + conn: str, + master_name: str, + master_port: str, + scheduler: str, + cp: str, + classpath: str, + ld_library_path: str, + pythonpath: str, + jvm_workers: str, + cpu_affinity: str, + gpu_affinity: str, + fpga_affinity: str, + fpga_reprogram: str, + profile_input: str, + profile_output: str, + scheduler_config: str, + external_adaptation: str, + python_interpreter: str, + python_version: str, + python_virtual_environment: str, + propagate_virtual_environment: bool, + mpi_worker: bool, + worker_cache: typing.Union[bool, str], + shutdown_in_node_failure: bool, + io_executors: int, + env_script: str, + reuse_on_block: bool, + nested_enabled: bool, + tracing_task_dependencies: bool, + trace_label: str, + extrae_cfg_python: str, + wcl: int, + cache_profiler: bool, + ear: bool, + data_provenance: bool, + checkpoint_policy: str, + checkpoint_params: str, + checkpoint_folder: str, + **kwargs: typing.Any, +) -> None: + """Create the initialization file for the runtime start (java opts file). + + :param compss_home: COMPSs installation path. + :param debug: Enable/Disable debugging + (True|False) (overrides log_level). + :param log_level: Define the log level + ("off" (default) | "info" | "debug"). + :param project_xml: Specific project.xml path. + :param resources_xml: Specific resources.xml path. + :param summary: Enable/Disable summary (True|False). + :param task_execution: Who performs the task execution + (normally "compss"). + :param storage_conf: None| Storage configuration file path. + :param streaming_backend: Streaming backend (default: None => "null"). + :param streaming_master_name: Streaming master name + (default: None => "null"). + :param streaming_master_port: Streaming master port + (default: None => "null"). + :param task_count: Number of tasks + (for structure initialization purposes). + :param app_name: Application name. + :param uuid: None| Application UUID. + :param log_dir: None| Log path. + :param master_working_dir: None| Master working path. + :param graph: Enable/Disable graph generation. + :param monitor: None| Disable/Frequency of the monitor. + :param trace: Enable/Disable trace generation. + :param extrae_cfg: None| Default extrae configuration/user + specific extrae configuration. + :param extrae_final_directory: None| + Default extrae final directory. + :param comm: GAT/NIO. + :param conn: Connector + (normally: es.bsc.compss.connectors.DefaultSSHConnector). + :param master_name: Master node name. + :param master_port: Master node port. + :param scheduler: Scheduler (normally: + es.bsc.compss.scheduler.lookahead.locality.LocalityTS). + :param cp: Application path. + :param classpath: CLASSPATH environment variable contents. + :param ld_library_path: LD_LIBRARY_PATH environment + variable contents. + :param pythonpath: PYTHONPATH environment variable contents. + :param jvm_workers: Worker"s jvm configuration + (example: "-Xms1024m,-Xmx1024m,-Xmn400m"). + :param cpu_affinity: CPU affinity (default: automatic). + :param gpu_affinity: GPU affinity (default: automatic). + :param fpga_affinity: FPGA affinity (default: automatic). + :param fpga_reprogram: FPGA reprogram command (default: ""). + :param profile_input: profiling input. + :param profile_output: profiling output. + :param scheduler_config: Path to the file which contains the + scheduler configuration.. + :param external_adaptation: Enable external adaptation. + This option disables the Resource Optimizer. + :param python_interpreter: Python interpreter. + :param python_version: Python interpreter version. + :param python_virtual_environment: Python virtual env path. + :param propagate_virtual_environment: Propagate python virtual + environment to workers. + :param mpi_worker: Use the MPI worker [ True | False ] (default: False). + :param worker_cache: Use the worker cache [ True | int(size) | False ] + (default: False). + :param shutdown_in_node_failure: Shutdown in node failure [ True | False] + (default: False). + :param io_executors: Number of IO executors. + :param env_script: Environment script to be sourced in workers. + :param reuse_on_block: Reuse on block [ True | False] (default: True). + :param nested_enabled: Nested enabled [ True | False] (default: False). + :param tracing_task_dependencies: Include task dependencies in trace + [ True | False] (default: False). + :param trace_label: Add trace label. + :param extrae_cfg_python: Extrae configuration file for the + workers. + :param wcl: Wall clock limit. Stops the runtime if reached. + 0 means forever. + :param cache_profiler: Use the cache profiler [ True | False ] + (default: False). + :param ear: Use the EAR [ True | False ] (default: False). + :param kwargs: Any other parameter. + :return: None. + """ + temp_fd, temp_path = mkstemp() + with open(temp_path, "w") as jvm_options_file: + # JVM GENERAL OPTIONS + jvm_options_file.write("-XX:+PerfDisableSharedMem\n") + jvm_options_file.write("-XX:-UsePerfData\n") + jvm_options_file.write("-XX:+UseG1GC\n") + jvm_options_file.write("-XX:+UseThreadPriorities\n") + jvm_options_file.write("-XX:ThreadPriorityPolicy=0\n") + jvm_options_file.write( + "-javaagent:" + compss_home + "/Runtime/compss-engine.jar\n" + ) + jvm_options_file.write("-Dcompss.to.file=false\n") + jvm_options_file.write("-Dcompss.appName=" + app_name + "\n") + + if data_provenance: + jvm_options_file.write("-Dcompss.data_provenance=true\n") + else: + jvm_options_file.write("-Dcompss.data_provenance=false\n") + + if uuid == "": + my_uuid = str(_uuid.uuid4()) + else: + my_uuid = uuid + jvm_options_file.write("-Dcompss.uuid=" + my_uuid + "\n") + + if shutdown_in_node_failure: + jvm_options_file.write("-Dcompss.shutdown_in_node_failure=true\n") + else: + jvm_options_file.write("-Dcompss.shutdown_in_node_failure=false\n") + + if log_dir == "": + log_dir = __create_log_dir() + if master_working_dir == "": + master_working_dir = __create_master_working_dir(log_dir) + jvm_options_file.write( + "-Dcompss.master.workingDir=" + master_working_dir + "\n" + ) + jvm_options_file.write("-Dcompss.log.dir=" + log_dir + "\n") + + conf_file_key = "-Dlog4j.configurationFile=" + log_off = True + if debug or log_level == "debug": + jvm_options_file.write( + conf_file_key + + compss_home + + DEFAULT_LOG_PATH + + "COMPSsMaster-log4j.debug\n" + ) # DEBUG + log_off = False + elif monitor is not None or log_level == "info": + jvm_options_file.write( + conf_file_key + + compss_home + + DEFAULT_LOG_PATH + + "COMPSsMaster-log4j.info\n" + ) # INFO + log_off = False + if log_off: + jvm_options_file.write( + conf_file_key + + compss_home + + DEFAULT_LOG_PATH + + "COMPSsMaster-log4j\n" + ) # NO DEBUG + + if graph: + jvm_options_file.write("-Dcompss.graph=true\n") + else: + jvm_options_file.write("-Dcompss.graph=false\n") + + if monitor == -1: + jvm_options_file.write("-Dcompss.monitor=0\n") + else: + jvm_options_file.write("-Dcompss.monitor=" + str(monitor) + "\n") + + if summary: + jvm_options_file.write("-Dcompss.summary=true\n") + else: + jvm_options_file.write("-Dcompss.summary=false\n") + + jvm_options_file.write( + "-Dcompss.worker.cp=" + + cp + + ":" + + compss_home + + "/Runtime/compss-engine.jar:" + + classpath + + "\n" + ) + jvm_options_file.write("-Dcompss.worker.appdir=" + cp + "\n") + jvm_options_file.write( + "-Dcompss.worker.jvm_opts=" + jvm_workers + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.cpu_affinity=" + cpu_affinity + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.gpu_affinity=" + gpu_affinity + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.fpga_affinity=" + fpga_affinity + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.fpga_reprogram=" + fpga_reprogram + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.io_executors=" + str(io_executors) + "\n" + ) + jvm_options_file.write( + "-Dcompss.worker.env_script=" + env_script + "\n" + ) + + if comm == "GAT": + gat = "-Dcompss.comm=es.bsc.compss.gat.master.GATAdaptor" + jvm_options_file.write(gat + "\n") + elif comm == "NIO": + nio = "-Dcompss.comm=es.bsc.compss.nio.master.NIOAdaptor" + jvm_options_file.write(nio + "\n") + elif comm == "GOS": + gos = "-Dcompss.comm=es.bsc.compss.gos.master.GOSAdaptor" + jvm_options_file.write(gos + "\n") + else: + jvm_options_file.write("-Dcompss.comm=" + comm + "\n") + + jvm_options_file.write("-Dcompss.masterName=" + master_name + "\n") + jvm_options_file.write("-Dcompss.masterPort=" + master_port + "\n") + + jvm_options_file.write( + "-Dgat.adaptor.path=" + + compss_home + + "/Dependencies/JAVA_GAT/lib/adaptors\n" + ) + if debug: + jvm_options_file.write("-Dgat.debug=true\n") + else: + jvm_options_file.write("-Dgat.debug=false\n") + jvm_options_file.write("-Dgat.broker.adaptor=sshtrilead\n") + jvm_options_file.write("-Dgat.file.adaptor=sshtrilead\n") + + if reuse_on_block: + jvm_options_file.write("-Dcompss.execution.reuseOnBlock=true\n") + else: + jvm_options_file.write("-Dcompss.execution.reuseOnBlock=true\n") + + if nested_enabled: + jvm_options_file.write("-Dcompss.execution.nested.enabled=true\n") + else: + jvm_options_file.write("-Dcompss.execution.nested.enabled=false\n") + + jvm_options_file.write("-Dcompss.scheduler=" + scheduler + "\n") + jvm_options_file.write( + "-Dcompss.scheduler.config=" + scheduler_config + "\n" + ) + jvm_options_file.write( + "-Dcompss.profile.input=" + profile_input + "\n" + ) + jvm_options_file.write( + "-Dcompss.profile.output=" + profile_output + "\n" + ) + + jvm_options_file.write("-Dcompss.project.file=" + project_xml + "\n") + jvm_options_file.write( + "-Dcompss.resources.file=" + resources_xml + "\n" + ) + jvm_options_file.write( + "-Dcompss.project.schema=" + + compss_home + + DEFAULT_PROJECT_PATH + + "project_schema.xsd\n" + ) + jvm_options_file.write( + "-Dcompss.resources.schema=" + + compss_home + + DEFAULT_RESOURCES_PATH + + "resources_schema.xsd\n" + ) + + jvm_options_file.write("-Dcompss.conn=" + conn + "\n") + jvm_options_file.write( + "-Dcompss.external.adaptation=" + external_adaptation + "\n" + ) + + jvm_options_file.write( + "-Dcompss.checkpoint.policy=" + str(checkpoint_policy) + "\n" + ) + jvm_options_file.write( + "-Dcompss.checkpoint.params=" + str(checkpoint_params) + "\n" + ) + jvm_options_file.write( + "-Dcompss.checkpoint.folder=" + str(checkpoint_folder) + "\n" + ) + + jvm_options_file.write("-Dcompss.lang=python\n") + + jvm_options_file.write("-Dcompss.core.count=" + str(task_count) + "\n") + + jvm_options_file.write( + "-Djava.class.path=" + + cp + + ":" + + compss_home + + "/Runtime/compss-engine.jar:" + + classpath + + "\n" + ) + jvm_options_file.write("-Djava.library.path=" + ld_library_path + "\n") + + # SPECIFIC JVM OPTIONS FOR PYTHON + jvm_options_file.write( + "-Dcompss.worker.pythonpath=" + cp + ":" + pythonpath + "\n" + ) + jvm_options_file.write( + "-Dcompss.python.interpreter=" + python_interpreter + "\n" + ) + jvm_options_file.write( + "-Dcompss.python.version=" + python_version + "\n" + ) + jvm_options_file.write( + "-Dcompss.python.virtualenvironment=" + + python_virtual_environment + + "\n" + ) + virtualenv_prefix = "-Dcompss.python.propagate_virtualenvironment=" + if propagate_virtual_environment: + jvm_options_file.write(virtualenv_prefix + "true\n") + else: + jvm_options_file.write(virtualenv_prefix + "false\n") + if mpi_worker: + jvm_options_file.write("-Dcompss.python.mpi_worker=true\n") + else: + jvm_options_file.write("-Dcompss.python.mpi_worker=false\n") + if worker_cache: + jvm_options_file.write("-Dcompss.python.worker_cache=true\n") + else: + jvm_options_file.write("-Dcompss.python.worker_cache=false\n") + + if cache_profiler: + jvm_options_file.write( + "-Dcompss.python.cache_profiler=" + + str(worker_cache).lower() + + "\n" + ) + else: + jvm_options_file.write("-Dcompss.python.cache_profiler=false\n") + + # SPECIFIC FOR STREAMING + if streaming_backend == "": + jvm_options_file.write("-Dcompss.streaming=NONE\n") + else: + jvm_options_file.write( + "-Dcompss.streaming=" + str(streaming_backend) + "\n" + ) + if streaming_master_name == "": + jvm_options_file.write("-Dcompss.streaming.masterName=null\n") + else: + jvm_options_file.write( + "-Dcompss.streaming.masterName=" + + str(streaming_master_name) + + "\n" + ) + if streaming_master_port == "": + jvm_options_file.write("-Dcompss.streaming.masterPort=null\n") + else: + jvm_options_file.write( + "-Dcompss.streaming.masterPort=" + + str(streaming_master_port) + + "\n" + ) + + # STORAGE SPECIFIC + jvm_options_file.write( + "-Dcompss.task.execution=" + task_execution + "\n" + ) + if storage_conf == "": + jvm_options_file.write("-Dcompss.storage.conf=null\n") + else: + jvm_options_file.write( + "-Dcompss.storage.conf=" + storage_conf + "\n" + ) + + # TOOLS SPECIFIC + if extrae_cfg == "": + extrae_xml_name = "extrae_basic.xml" + extrae_xml_path = ( + compss_home + DEFAULT_TRACING_PATH + extrae_xml_name + ) + extrae_cfg = "null" + else: + extrae_xml_name = os.path.basename(extrae_cfg) + extrae_xml_path = extrae_cfg + if trace: + jvm_options_file.write("-Dcompss.tracing=true\n") + # Process extrae_xml_path + extrae_xml_final_path_dir = os.path.join(log_dir, "cfgfiles") + pathlib.Path(extrae_xml_final_path_dir).mkdir( + parents=False, exist_ok=True + ) + extrae_trace_path = os.path.join(log_dir, "trace") + extrae_xml_final_path = os.path.join( + extrae_xml_final_path_dir, extrae_xml_name + ) + got_extrae_final_directory = __process_extrae_file( + extrae_xml_path, + extrae_xml_final_path, + extrae_trace_path, + compss_home, + ) + if extrae_cfg == "null": + os.environ["EXTRAE_CONFIG_FILE"] = extrae_xml_final_path + else: + # Any other case: deactivated + jvm_options_file.write("-Dcompss.tracing=false" + "\n") + got_extrae_final_directory = "null" + if tracing_task_dependencies: + jvm_options_file.write("-Dcompss.tracing.task.dependencies=true\n") + else: + jvm_options_file.write( + "-Dcompss.tracing.task.dependencies=false\n" + ) + if extrae_final_directory == "": + jvm_options_file.write( + "-Dcompss.extrae.working_dir=" + + got_extrae_final_directory + + "\n" + ) + else: + jvm_options_file.write( + "-Dcompss.extrae.working_dir=" + extrae_final_directory + "\n" + ) + jvm_options_file.write("-Dcompss.extrae.file=" + extrae_cfg + "\n") + if extrae_cfg_python == "": + jvm_options_file.write("-Dcompss.extrae.file.python=null\n") + else: + jvm_options_file.write( + "-Dcompss.extrae.file.python=" + str(extrae_cfg_python) + "\n" + ) + if trace_label == "": + jvm_options_file.write("-Dcompss.trace.label=None\n") + else: + jvm_options_file.write( + "-Dcompss.trace.label=" + str(trace_label) + "\n" + ) + + # WALLCLOCK LIMIT + jvm_options_file.write("-Dcompss.wcl=" + str(wcl) + "\n") + + # EAR + if ear: + jvm_options_file.write("-Dcompss.ear=true\n") + else: + jvm_options_file.write("-Dcompss.ear=false\n") + + # Uncomment for debugging purposes + # jvm_options_file.write("-Xcheck:jni\n") + # jvm_options_file.write("-verbose:jni\n") + + # Close the temporary file + os.close(temp_fd) + os.environ["JVM_OPTIONS_FILE"] = temp_path + + # Uncomment if you want to check the configuration file path: + # print("JVM_OPTIONS_FILE: %s" % temp_path) + + +def __process_extrae_file( + extrae_xml_path: str, + extrae_xml_final_path: str, + extrae_trace_path: str, + compss_home: str, +) -> str: + """Sed and grep extrae file. + + :param extrae_xml_path: Source extrae xml file path. + :param extrae_xml_final_path: Destination extrae xml file path. + :param extrae_trace_path: Extrae trace working directory + :param compss_home: COMPSs home directory + :return: final_directory from extrae_xml_path + """ + got_extrae_final_directory = "null" + with open(extrae_xml_path, "r") as extrae_xml_fd: + extrae_xml_data = extrae_xml_fd.readlines() + # Look for final-directory + for i, line in enumerate(extrae_xml_data): + if "{{EXTRAE_HOME}}" in line: + # Sed with real path + extrae_home = str( + os.path.join(compss_home, "Dependencies", "extrae") + ) + extrae_xml_data[i] = line.replace("{{EXTRAE_HOME}}", extrae_home) + if "{{TRACE_OUTPUT_DIR}}" in line: + # Sed with real path + extrae_xml_data[i] = line.replace( + "{{TRACE_OUTPUT_DIR}}", extrae_trace_path + ) + if "final-directory" in line: + # Extract final-directory + key = '(.*)' + found = re.search(key, extrae_xml_data[i]).group(2) # type: ignore + got_extrae_final_directory = found + with open(extrae_xml_final_path, "w") as extrae_xml_fd: + extrae_xml_fd.writelines(extrae_xml_data) + return got_extrae_final_directory + + +def __create_log_dir() -> str: + """Create log directory. + + :return: Updated log directory. + """ + home_folder = str(pathlib.Path.home()) + compss_log_folder = os.path.join(home_folder, ".COMPSs") + pathlib.Path(compss_log_folder).mkdir(parents=False, exist_ok=True) + log_folder_prefix = "Interactive_" + sub_folders = list(os.walk(compss_log_folder))[0][1] + existing_folders = [] + for folder in sub_folders: + if folder.startswith(log_folder_prefix): + existing_folders.append(folder) + next_int = len(existing_folders) + 1 + next_subfolder = log_folder_prefix + f"{next_int:02d}" + log_dir = os.path.join(compss_log_folder, next_subfolder) + pathlib.Path(log_dir).mkdir(parents=False, exist_ok=True) + return log_dir + + +def __create_master_working_dir(log_dir: str) -> str: + """Create master working directory (log_dir/tmpFiles). + + :return: Updated master working directory. + """ + master_working_dir = os.path.join(log_dir, "tmpFiles") + pathlib.Path(master_working_dir).mkdir(parents=False, exist_ok=True) + return master_working_dir diff --git a/examples/dds/pycompss/util/exceptions.py b/examples/dds/pycompss/util/exceptions.py new file mode 100644 index 00000000..49abc539 --- /dev/null +++ b/examples/dds/pycompss/util/exceptions.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Exceptions. + +This file defines the internal PyCOMPSs exceptions. +""" + +from pycompss.util.typing_helper import typing + + +class SerializerException(Exception): + """Exception on serialization class.""" + + +class PyCOMPSsException(Exception): + """Generic PyCOMPSs exception class.""" + + def __init__(self, message: str) -> None: + """Create a new PyCOMPSsException instance. + + :param message: Exception message. + """ + msg = f"PyCOMPSs Exception: {message}" + super().__init__(msg) + + +class NotInPyCOMPSsException(Exception): + """Not within PyCOMPSs scope exception class.""" + + def __init__(self, message: str) -> None: + """Create a new NotInPyCOMPSsException instance. + + :param message: Exception message. + """ + msg = f"Outside PyCOMPSs scope: {message}" + super().__init__(msg) + + +class NotImplementedException(Exception): + """Not implemented exception class.""" + + def __init__(self, functionality: str) -> None: + """Create a new NotImplementedException instance. + + :param functionality: Exception message. + """ + msg = f"Functionality {functionality} not implemented yet." + super().__init__(msg) + + +class MissingImplementedException(Exception): + """Missing implemented exception class.""" + + def __init__(self, functionality: str) -> None: + """Create a new MissingImplementedException instance. + + :param functionality: Exception message. + """ + msg = f"Missing {functionality}. Needs to be overridden." + super().__init__(msg) + + +class DDSException(Exception): + """Generic DDS exception class.""" + + def __init__(self, message: str) -> None: + """Create a new DDSException instance. + + :param message: Exception message. + """ + msg = f"DDS Exception: {message}" + super().__init__(msg) + + +class TimeOutError(Exception): + """Time out error exception class.""" + + +class CancelError(Exception): + """Cancel error exception class.""" + + +class StandardOutputError(Exception): + """Stdout error exception class.""" + + +class StandardErrorError(Exception): + """Stderr error exception class.""" + + +def task_timed_out(signum: int, frame: typing.Any) -> None: + """Task time out signal handler. + + Do not remove the parameters. + + :param signum: Signal number. + :param frame: Frame. + :return: None + :raises: TimeOutError exception. + """ + raise TimeOutError + + +def task_cancel(signum: int, frame: typing.Any) -> None: + """Task cancel signal handler. + + Do not remove the parameters. + + :param signum: Signal number. + :param frame: Frame. + :return: None + :raises: CancelError exception. + """ + raise CancelError diff --git a/examples/dds/pycompss/util/interactive/__init__.py b/examples/dds/pycompss/util/interactive/__init__.py new file mode 100644 index 00000000..a341100a --- /dev/null +++ b/examples/dds/pycompss/util/interactive/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the interactive utils helpers. + +Where the helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/util/interactive/events.py b/examples/dds/pycompss/util/interactive/events.py new file mode 100644 index 00000000..a6700688 --- /dev/null +++ b/examples/dds/pycompss/util/interactive/events.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Events. + +Provides the event handlers for the cell execution and registers the callbacks. +""" + +import os + +from pycompss.util.interactive.outwatcher import STDW +from pycompss.util.typing_helper import typing + +try: + # IPython imports + from IPython.display import display + from IPython.display import Javascript +except ImportError: + display = None + Javascript = None + + +####################################### +# EVENT CALLBACKS # +####################################### + + +def __pre_execute() -> None: + """Fire prior to interactive execution. + + :return: None. + """ + print("pre_execute") + + +def __pre_run_cell() -> None: + """Like pre_run_cell, but is triggered prior to any execution. + + Sometimes code can be executed by libraries, etc. which skipping the + history/display mechanisms, in which cases pre_run_cell will not fire. + + :return: None. + """ + print("pre_run_cell") + + +def __post_execute() -> None: + """Run after interactive execution (e.g. a cell in a notebook). + + :return: None. + """ + print("post_execute") + + +def __post_run_cell() -> None: + """Run for all cells after execution. + + The same as pre_execute, post_execute is like post_run_cell, but + fires for all executions, not just interactive ones. + + Notifies if any exception or task has been cancelled to the user. + + :return: None. + """ + messages = STDW.get_messages() + last_message = __process_messages(messages) + if last_message: + print(last_message) + + +def __process_messages(messages: typing.List[str]) -> typing.Optional[str]: + """Process the messages and builds a popup with the information. + + :param messages: List of messages to show. + :return: None or a message that needs to be printed. + """ + found_errors = False + runtime_crashed = False + if messages: + for message in messages: + if message.startswith("[ERRMGR]"): + # Errors found, but maybe not critical, like for example + # tasks that failed but recovered. + found_errors = True + if message == "[ERRMGR] - Shutting down COMPSs...": + # A critical error occurred --> notify that COMPSs runtime + # stopped working to avoid issues when running any PyCOMPSs + # function. + runtime_crashed = True + if runtime_crashed: + # Display popup with the error messages + current_flags = str(os.environ["PYCOMPSS_CURRENT_FLAGS"]) + header = [] # type: typing.List[str] + footer = [] # type: typing.List[str] + popup_body = header + messages + footer + error_messages_html = "

" + "
".join(popup_body) + "

" + error_messages_html = error_messages_html.replace("'", "") + popup_title_html = "COMPSs RUNTIME STOPPED" + # fmt: off + popup_code = f"""require(["base/js/dialog"], + function(dialog) OPENBRACKET + function restartCOMPSs()OPENBRACKET + var kernel = IPython.notebook.kernel; + kernel.execute("import base64; import json; from pycompss.interactive import stop, start; stop(_hard_stop=True); _COMPSS_START_FLAGS=json.loads(base64.b64decode('" + '{current_flags}' + "'.encode())); start(**_COMPSS_START_FLAGS)"); + CLOSEBRACKET + function continueWithoutCOMPSs()OPENBRACKET + var kernel = IPython.notebook.kernel; + kernel.execute("from pycompss.interactive import stop; stop(_hard_stop=True)"); + CLOSEBRACKET + dialog.modal(OPENBRACKET + title: '{popup_title_html}', + body: $('{error_messages_html}'), + buttons: OPENBRACKET + 'Continue without COMPSs': OPENBRACKET + click: function() OPENBRACKET + continueWithoutCOMPSs(); + CLOSEBRACKET + CLOSEBRACKET, + 'Restart COMPSs': OPENBRACKET + class: 'btn-primary', + click: function() OPENBRACKET + restartCOMPSs(); + CLOSEBRACKET + CLOSEBRACKET + CLOSEBRACKET + CLOSEBRACKET); + CLOSEBRACKET + );""" # noqa # pylint: disable=line-too-long + # fmt: on + popup_js = popup_code.replace("OPENBRACKET", "{").replace( + "CLOSEBRACKET", "}" + ) + popup = Javascript(popup_js) + display(popup) + warn_msg = ( + "WARNING: Some objects may have not been " + "synchronized and need to be recomputed." + ) + return f"\x1b[40;43m{warn_msg}\x1b[0m" + elif found_errors: + # Display popup with the warning messages + header = [] + footer = [] + popup_body = header + messages + footer + error_messages_html = "

" + "
".join(popup_body) + "

" + error_messages_html = error_messages_html.replace("'", "") + popup_title_html = "WARNING: Some tasks may have failed" + # fmt: off + popup_code = f"""require(["base/js/dialog"], + function(dialog) OPENBRACKET + dialog.modal(OPENBRACKET + title: '{popup_title_html}', + body: $('{error_messages_html}'), + buttons: OPENBRACKET + 'Continue': OPENBRACKET CLOSEBRACKET, + CLOSEBRACKET + CLOSEBRACKET); + CLOSEBRACKET + );""" # noqa # pylint: disable=line-too-long + # fmt: on + popup_js = popup_code.replace("OPENBRACKET", "{").replace( + "CLOSEBRACKET", "}" + ) + popup = Javascript(popup_js) + display(popup) # noqa + info_msg = "INFO: The ERRMGR displayed some error or warnings." + return f"\x1b[40;46m{info_msg}\x1b[0m" + else: + # No issue + return None + return None + + +####################################### +# EVENT MANAGEMENT FUNCTIONS # +####################################### + + +def setup_event_manager(ipython: typing.Any) -> None: + """Instantiate an Ipython event manager and registers the event handlers. + + :param ipython: IPython instance where to register the event manager. + :return: None. + """ + # ipython.events.register("pre_execute", __pre_execute) + # ipython.events.register("pre_run_cell", __pre_run_cell) + # ipython.events.register("post_execute", __post_execute) + ipython.events.register("post_run_cell", __post_run_cell) + + +def release_event_manager(ipython: typing.Any) -> None: + """Releases the event manager in the given ipython instance. + + :param ipython: IPython instance where to release the event manager. + :return: None. + """ + try: + # ipython.events.unregister("pre_execute", __pre_execute) + # ipython.events.unregister("pre_run_cell", __pre_run_cell) + # ipython.events.unregister("post_execute", __post_execute) + ipython.events.unregister("post_run_cell", __post_run_cell) + except ValueError: + # The event was already unregistered + pass diff --git a/examples/dds/pycompss/util/interactive/flags.py b/examples/dds/pycompss/util/interactive/flags.py new file mode 100644 index 00000000..278c75c9 --- /dev/null +++ b/examples/dds/pycompss/util/interactive/flags.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Mode Flags checker. + +Provides auxiliary methods for the interactive mode flags checking +""" + +from pycompss.util.typing_helper import typing + +# None type +NONETYPE = type(None) + +# Required flags - Structure: +# - key = flag name +# - value = [ [supported types], [supported values]] +# - supported values is optional +REQUIRED_FLAGS = { + "log_level": [[str], ["trace", "debug", "info", "api", "off"]], + "debug": [[bool]], + "o_c": [[bool]], + "graph": [[bool]], + "trace": [[bool, str], [True, False, "scorep", "arm-map", "arm-ddt"]], + "monitor": [[int, NONETYPE]], + "project_xml": [[str, NONETYPE]], + "resources_xml": [[str, NONETYPE]], + "summary": [[bool]], + "task_execution": [[str]], + "storage_impl": [[str, NONETYPE]], + "storage_conf": [[str, NONETYPE]], + "streaming_backend": [[str, NONETYPE]], + "streaming_master_name": [[str, NONETYPE]], + "streaming_master_port": [[str, NONETYPE]], + "task_count": [[int]], + "app_name": [[str]], + "uuid": [[str, NONETYPE]], + "log_dir": [[str, NONETYPE]], + "master_working_dir": [[str, NONETYPE]], + "extrae_cfg": [[str, NONETYPE]], + "comm": [[str]], + "conn": [ + [str], + [ + "es.bsc.compss.connectors.DefaultSSHConnector", + "es.bsc.compss.connectors.DefaultNoSSHConnector", + ], + ], + "master_name": [[str]], + "master_port": [[str]], + "scheduler": [[str]], + "jvm_workers": [[str]], + "cpu_affinity": [[str]], + "gpu_affinity": [[str]], + "fpga_affinity": [[str]], + "fpga_reprogram": [[str]], + "profile_input": [[str]], + "profile_output": [[str]], + "scheduler_config": [[str]], + "external_adaptation": [[bool]], + "propagate_virtual_environment": [[bool]], + "mpi_worker": [[bool]], +} # type: typing.Dict[str, typing.List[typing.List[typing.Union[object, str, bool, type]]]] # noqa # pylint: disable=line-too-long + + +def check_flags(all_vars: dict) -> typing.Tuple[bool, list]: + """Check that the provided flags are supported. + + :param all_vars: Flags dictionary. + :return: If all flags are supported and the issues if exists. + """ + is_ok = True + issues = [] + flags = all_vars.keys() + + missing_flags = [key for key in REQUIRED_FLAGS if key not in flags] + if missing_flags: + # There are missing flags + is_ok = False + for missing_flag in missing_flags: + issues.append(f"Missing flag: {missing_flag}") + else: + # Check that each element is of the correct type and supported value + for flag, requirements in REQUIRED_FLAGS.items(): + issues += __check_flag(all_vars, flag, requirements) + if issues: + is_ok = False + + return is_ok, issues + + +def __check_flag( + all_vars: dict, + flag: str, + requirements: typing.List[ + typing.List[typing.Union[object, str, bool, type]] + ], +) -> list: + """Check the given flag against the requirements looking for issues. + + :param all_vars: All variables. + :param flag: Flag to check. + :param requirements: Flag requirements. + :returns: A list of issues (empty if none). + """ + flag_header = "Flag " + is_not = " is not " + issues = [] + if len(requirements) == 1: + # Only check type + req_type = requirements[0] + if not type(all_vars[flag]) in req_type: + issues.append(flag_header + flag + is_not + str(req_type)) + if len(requirements) == 2: + # Check type + req_type = requirements[0] + req_values = requirements[1] + if not type(all_vars[flag]) in req_type: + issues.append(flag_header + flag + is_not + str(req_type)) + else: + # Check that it is also one of the options + if not all_vars[flag] in req_values: + issues.append( + flag_header + + flag + + "=" + + all_vars[flag] + + " is not supported. Available values: " + + str(req_values) + ) + return issues + + +def print_flag_issues(issues: typing.List[str]) -> None: + """Display the given issues on stdout. + + :param issues: Flag issues. + :return: None. + """ + print("[ERROR] The following flag issues were detected:") + for issue in issues: + print(f"\t - {issue}") diff --git a/examples/dds/pycompss/util/interactive/graphs.py b/examples/dds/pycompss/util/interactive/graphs.py new file mode 100644 index 00000000..51943d6a --- /dev/null +++ b/examples/dds/pycompss/util/interactive/graphs.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Graphs. + +Provides auxiliary methods for the interactive mode with regard to graphs +""" + +import os +import time + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + + +def show_graph( + log_path: str, + name: str = "complete_graph", + fit: bool = False, + refresh_rate: int = 1, + timeout: int = 0, + widget=None, +) -> None: + """Show graph. + + :param log_path: Folder where the logs are. + :param name: Graph to show (default: "complete_graph"). + :param fit: Fit to width [ True | False ] (default: False). + :param refresh_rate: Update the current task graph every "refresh_rate" + seconds. Default 1 second if timeout != 0. + :param timeout: Time during the current task graph is going to be updated. + :param widget: Widget where to show the graph. + :return: None. + """ + try: + from graphviz import Source # noqa + except ImportError: + print("Oops! graphviz is not available.") + return None + from IPython.display import clear_output # noqa + + # Check refresh rate and timeout + if timeout < 0: + raise PyCOMPSsException("ERROR: timeout has to be >= 0") + if timeout > 0: + if refresh_rate > timeout: + raise PyCOMPSsException( + "ERROR: refresh_rate can not be higher than timeout" + ) + # Set file name + file_name = os.path.join(log_path, "monitor", name) + # Act + if timeout == 0: + source = __get_graph_snapshot(file_name, fit, Source) + __show_snapshot(source, widget) + else: + try: + while timeout >= 0: + clear_output(wait=True) + source = __get_graph_snapshot(file_name, fit, Source) + __show_snapshot(source, widget) + time.sleep(refresh_rate) + timeout = timeout - refresh_rate + except KeyboardInterrupt: + # User hit stop on the cell + clear_output(wait=True) + source = __get_graph_snapshot(file_name, fit, Source) + __show_snapshot(source, widget) + return None + + +def __show_snapshot(source: typing.Any, widget: typing.Any) -> None: + """Display snapshot in the appropriate output. + + :param source: Source to be displayed. + :param widget: Widget to be used. Displayed normally if not provided. + :return: None + """ + from IPython.display import display # noqa + + if widget: + widget.append_display_data(source) + else: + display(source) + + +def __get_graph_snapshot( + file_name: str, fit: bool, source: typing.Any +) -> typing.Any: + """Read the graph file and returns it as graphviz object. + + It is able to fit the size if indicated. + + :param file_name: Absolute path to the graph to get the snapshot. + :param fit: Fit to size or not. + :param source: Graphviz Source object + :return: The graph snapshot to be rendered. + """ + # Read graph file + with open(file_name + ".dot", "r") as monitor_file: + text = monitor_file.read() + if fit: + try: + # Convert to png and show full picture + extension = "jpeg" + file = f"{file_name}.{extension}" + if os.path.exists(file): + os.remove(file) + graph_source = source(text, filename=file_name, format=extension) + graph_source.render() + from IPython.display import Image # noqa + + image = Image(filename=file) + return image + except Exception: # as general_exception: + print("Oops! Failed rendering the graph.") + # raise general_exception + else: + return source(text) diff --git a/examples/dds/pycompss/util/interactive/helpers.py b/examples/dds/pycompss/util/interactive/helpers.py new file mode 100644 index 00000000..c8151360 --- /dev/null +++ b/examples/dds/pycompss/util/interactive/helpers.py @@ -0,0 +1,836 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Mode Helpers. + +Provides auxiliary methods for the interactive mode. +""" + +import inspect +import os + +from pycompss.util.typing_helper import typing + +# Debug mode: Changed to true from interactive.py if specified by the user +# when starting the runtime. Enables the explicit prints. +DEBUG = False + + +SEPARATORS = { # for user defined lines in the entire/global scope + "globals_separator": "### GLOBALS ###", + # for user defined classes + "classes_separator": "### CLASSES ###", + # for user defined functions (that are not decorated) + "functions_separator": "### FUNCTIONS ###", + # for user defined tasks + "tasks_separator": "### TASKS ###", +} + + +PREFIXES = ( + "@implement", + "@constraint", + "@decaf", + "@mpi", + "@ompss", + "@binary", + "@opencl", +) + +# ################################################################# # +# ################# MAIN FUNCTION ################################# # +# ################################################################# # + + +def update_tasks_code_file(function: typing.Callable, file_path: str) -> None: + """Update the file where the tasks code is stored. + + IMPORTANT! Main interactive helper function. + + Analyses the user code that has been executed and parses it looking for: + - imports + - tasks + - functions + Builds a file where the necessary contents for the worker are. + Also updates the old code with the new if functions or tasks are redefined. + + :param function: New task function. + :param file_path: File where the code is stored. + :return: None. + """ + first_time = False + if not os.path.exists(file_path): + _create_tasks_code_file(file_path) + first_time = True + + if DEBUG: + print("Task definition detected.") + + # Intercept the code + imports = _get_ipython_imports() # [import\n, import\n, ...] + global_code = _get_ipython_globals() # [var\n, var\n, ...] + classes_code = _get_classes() # {"name": str(line\nline\n...)} + functions_code = _get_functions() # {"name": str(line\nline\n...)} + task_code = _get_task_code(function) # {"name": str(line\nline\n...)} + old_code = _get_old_code(file_path) # old_code structure: + ( + old_code_imports, + old_code_globals, + old_code_classes, + old_code_functions, + old_code_tasks, + ) = old_code + # {"imports":[import\n, import\n, ...], + # "tasks":{"name":str(line\nline\n...), + # "name":str(line\nline\n...), ...}} + + # Look for new/modified pieces of code and compares the existing code with + # the new additions. + new_imports = _update_imports(imports, old_code_imports) + new_globals = _update_globals(global_code, old_code_globals) + new_classes = _update_classes(classes_code, old_code_classes) + # Check that there are no functions with the same name as a newly defined + # tasks + for k in task_code: + _ = functions_code.pop(k, None) + old_code_functions.pop(k, None) + # Continue with comparisons + new_functions = _update_functions(functions_code, old_code_functions) + new_tasks = _update_tasks(task_code, old_code_tasks) + + is_updated = False + if new_imports or new_globals or new_classes or new_functions or new_tasks: + is_updated = True + + if not first_time and is_updated: + from pycompss.api.api import compss_delete_file + + compss_delete_file(file_path) + + # Update the file where the code is stored. + _update_code_file( + new_imports, + new_globals, + new_classes, + new_functions, + new_tasks, + file_path, + ) + + +# ################################################################### +# ############### AUXILIARY METHODS ################################# +# ################################################################### + +# CODE INTERCEPTION FUNCTIONS + + +def _create_tasks_code_file(file_path: str) -> None: + """Create a file where to store the user code. + + :param file_path: File location and name. + :return: None. + """ + with open(file_path, "a") as user_code_file: + user_code_file.write("\n") + user_code_file.write(SEPARATORS["globals_separator"] + "\n") + user_code_file.write("\n") + user_code_file.write(SEPARATORS["classes_separator"] + "\n") + user_code_file.write("\n") + user_code_file.write(SEPARATORS["functions_separator"] + "\n") + user_code_file.write("\n") + user_code_file.write(SEPARATORS["tasks_separator"] + "\n") + user_code_file.write("\n") + + +def _get_raw_code() -> typing.List[str]: + """Retrieve the raw code from interactive session. + + :return: the list of the blocks defined by the user that are currently + loaded in globals. + """ + import IPython # noqa + + ipython = IPython.get_ipython() + raw_code = ipython.user_ns["In"] # type: typing.List[str] + return raw_code + + +def _get_ipython_imports() -> list: + r"""Find the user imports. + + :return: A list of imports: [import\n, import\n, ...]. + """ + raw_code = _get_raw_code() + imports = [] + for i in raw_code: + # Each i can have more than one line (jupyter-notebook block) + # We only get the lines that start with from or import and do not + # have blank spaces before. + lines = i.split("\n") + for line in lines: + if ( + line.startswith("from") or line.startswith("import") + ) and "pycompss.interactive" not in line: + imports.append(line + "\n") + return imports + + +def _get_ipython_globals() -> dict: + r"""Find the user global variables. + + WARNING: Assignations using any of the master api calls will be ignored + in order to avoid the worker to try to call the runtime. + + WARNING2: We will consider only global variables that must be seen by the + workers if the variable name is defined in capital letters. + Please, take caution with the modification on the global + variables since they can lead to raise conditions in the user + code execution. + + :return: A list of lines: [var\n, var\n, ...]. + """ + api_calls = [ + "compss_open", + "compss_delete_file", + "compss_wait_on_file", + "compss_delete_object", + "compss_wait_on", + ] + + raw_code = _get_raw_code() + glob_lines = {} + glob_name = "" + for i in raw_code: + # Each i can have more than one line (jupyter-notebook block) + # We only get the lines that start with from or import and do not + # have blank spaces before. + lines = i.split("\n") + found_one = False + for line in lines: + # if the line starts without spaces and is a variable assignation + if not ( + line.startswith(" ") or line.startswith("\t") + ) and _is_variable_assignation(line): + line_parts = line.split() + glob_name = line_parts[0] + if not glob_name.isupper(): + # It is an assignation where the variable name is not in + # capital letters + found_one = False + elif any(call in line_parts[2:] for call in api_calls): + # It is an assignation that does not contain a master api + # call + found_one = False + else: + glob_lines[glob_name] = line.strip() + found_one = True + continue + # if the next line/s start with space or tab belong also to the + # global variable + if found_one and (line.startswith(" ") or line.startswith("\t")): + # It is a multiple lines global variable definition + glob_lines[glob_name] += line.strip() + else: + found_one = False + return glob_lines + + +def _is_variable_assignation(line: str) -> bool: + """Check if the line is a variable assignation. + + This function is used to check if a line of code represents a variable + assignation: + * if contains a "=" (assignation) and does not start with import, nor @, + nor def, nor class. + * then it is ==> is a global variable assignation. + + :param line: Line to parse. + :return: If is a variable assignation. + """ + if "=" in line: + parts = line.split() + # It is actually an assignation == True + # It is an import/function/decorator/class definition == False + return bool( + not ( + line.startswith("from") + or line.startswith("import") + or line.startswith("@") + or line.startswith("def") + or line.startswith("class") + ) + and len(parts) >= 3 + and parts[1] == "=" + ) + # Not an assignation if does not contain "=" + return False + + +def _get_classes() -> dict: + r"""Find the user defined classes in the code. + + Output dictionary: {"name": str(line\nline\n...)} + + :return: A dictionary with the user classes code. + """ + raw_code = _get_raw_code() + classes = {} + for block in raw_code: + lines = block.split("\n") + # Look for classes in the block + class_found = False + class_name = "" + for line in lines: + if line.startswith("class"): + # Class header: find name and include it in the functions dict + # split and remove empty spaces + header = [name for name in line.split(" ") if name] + # the name may be followed by the parameters parenthesis + class_name = header[1].split("(")[0].strip() + # create an entry in the functions dict + classes[class_name] = [line + "\n"] + class_found = True + elif ( + line.startswith(" ") + or (line.startswith("\t")) + or (line.startswith("\n")) + or (line == "") + ) and class_found: + # class body found: append + classes[class_name].append(line + "\n") + else: + class_found = False + # Plain classes content (from {key: [line, line,...]} to {key: line\nline}) + plain_classes = {} + for key, value in list(classes.items()): + # Collapse all lines into a single one + plain_classes[key] = "".join(value).strip() + return plain_classes + + +def _get_functions() -> typing.Dict[str, str]: + r"""Find the user defined functions in the code. + + Output dictionary: {"name": str(line\nline\n...)} + + :return: A dictionary with the user functions code. + """ + raw_code = _get_raw_code() + functions = {} + for block in raw_code: + lines = block.split("\n") + # Look for functions in the block + is_task = False + is_function = False + function_found = False + func_name = "" + decorators = "" + for line in lines: + if line.startswith("@task"): + # The following function detected will be a task --> ignore + is_task = True + if ( + line.startswith("@") + and not any(map(line.startswith, PREFIXES)) + and not is_task + ): + # It is a function preceded by a decorator + is_function = True + is_task = False + if line.startswith("def") and not is_task: + # A function which is not a task has been defined --> capture + # with is_function boolean + # Restore the is_task boolean to control if another task is + # defined in the same block. + is_function = True + is_task = False + if is_function: + if line.startswith("@"): + decorators += line + "\n" + if line.startswith("def"): + # Function header: find name and include it in the + # functions dict. Split and remove empty spaces + header = [name for name in line.split(" ") if name] + # the name may be followed by the parameters parenthesis + func_name = header[1].split("(")[0].strip() + # create an entry in the functions dict + functions[func_name] = [decorators + line + "\n"] + decorators = "" + function_found = True + elif ( + line.startswith(" ") + or (line.startswith("\t")) + or (line.startswith("\n")) + or (line == "") + ) and function_found: + # Function body: append + functions[func_name].append(line + "\n") + else: + function_found = False + # Plain functions content: + # from {key: [line, line,...]} to {key: line\nline} + plain_functions = {} + for key, value in list(functions.items()): + # Collapse all lines into one + plain_functions[key] = "".join(value).strip() + return plain_functions + + +def _get_task_code(function: typing.Callable) -> typing.Dict[str, str]: + r"""Find the task code. + + :param function: Task function. + :return: A dictionary with the task code: {"name": str(line\nline\n...)}. + """ + try: + task_code = inspect.getsource(function) + except TypeError: + # This is a numba jit declared task + task_code = inspect.getsource(function.py_func) # type: ignore + if task_code.startswith((" ", "\t")): + return {} + name = "" + lines = task_code.split("\n") + for line in lines: + # Ignore the decorator stack + if line.strip().startswith("def"): + name = line.replace("(", " (").split(" ")[1].strip() + break # Just need the first + return {name: task_code} + + +def _clean(lines_list: typing.List[str]) -> typing.List[str]: + r"""Remove the blank lines from a list of strings. + + * _get_old_code auxiliary method - Clean imports list. + + :param lines_list: List of strings. + :return: The list without "\n" strings. + """ + result = [] # type: typing.List[str] + if len(lines_list) == 1 and lines_list[0].strip() == "": + # If the lines_list only contains a single line jump remove it + return result + # If it is longer, remove all single \n appearances + for line in lines_list: + if line.strip() != "": + result.append(line) + return result + + +def _get_old_code( + file_path: str, +) -> typing.Tuple[ + typing.List[str], + typing.Dict[str, str], + typing.Dict[str, str], + typing.Dict[str, str], + typing.Dict[str, str], +]: + """Retrieve the old code from a file. + + :param file_path: The file where the code is located. + :return: A dictionary with the imports and existing tasks. + """ + # Read the entire file + with open(file_path, "r") as code_file: + contents = code_file.readlines() + + # Separate imports from tasks + file_imports = [] + file_globals = [] + file_classes = [] + file_functions = [] + file_tasks = [] + found_glob_separator = False + found_class_separator = False + found_func_separator = False + found_task_separator = False + for line in contents: + if line == SEPARATORS["globals_separator"] + "\n": + found_glob_separator = True + elif line == SEPARATORS["classes_separator"] + "\n": + found_class_separator = True + elif line == SEPARATORS["functions_separator"] + "\n": + found_func_separator = True + elif line == SEPARATORS["tasks_separator"] + "\n": + found_task_separator = True + else: + if ( + not found_glob_separator + and not found_class_separator + and not found_func_separator + and not found_task_separator + ): + file_imports.append(line) + elif ( + found_glob_separator + and not found_class_separator + and not found_func_separator + and not found_task_separator + ): + file_globals.append(line) + elif ( + found_glob_separator + and found_class_separator + and not found_func_separator + and not found_task_separator + ): + file_classes.append(line) + elif ( + found_glob_separator + and found_class_separator + and found_func_separator + and not found_task_separator + ): + file_functions.append(line) + else: + file_tasks.append(line) + + file_imports = _clean(file_imports) + file_globals = _clean(file_globals) + # No need to clean the following + # file_classes = _clean(file_classes) + # file_functions = _clean(file_functions) + # file_tasks = _clean(file_tasks) + + # Process globals + globs = {} + if len(file_globals) != 0: + # Collapse all lines into a single one + collapsed = "".join(file_globals).strip() + scattered = collapsed.split("\n") + # Add classes to dictionary by class name: + for glob in scattered: + glob_code = glob.strip() + glob_name = glob.split()[0].strip() + globs[glob_name] = glob_code + + # Process classes + classes = {} + # Collapse all lines into a single one + collapsed = "".join(file_classes).strip() + # Then split by "class" and filter the empty results, then iterate + # concatenating "class" to all results. + cls = [ + ("class " + class_line) + for class_line in [name for name in collapsed.split("class ") if name] + ] + # Add classes to dictionary by class name: + for cls_ in cls: + class_code = cls_.strip() + class_name = cls_.replace("(", " (").split(" ")[1].strip() + classes[class_name] = class_code + + # Process functions + functions = {} + # Clean empty lines + clean_functions = [f_line for f_line in file_functions if f_line] + # Iterate over the lines splitting by the ones that start with def + funcs = [] + func = "" + for line in clean_functions: + if line.startswith("def"): + if func: + funcs.append(func) + func = line + else: + func += line + # Add functions to dictionary by function name: + for func in funcs: + func_code = func.strip() + func_name = func.replace("(", " (").split(" ")[1].strip() + functions[func_name] = func_code + + # Process tasks + tasks = {} + # Collapse all lines into a single one + collapsed = "".join(file_tasks).strip() + # Then split by "@" and filter the empty results, then iterate + # concatenating "@" to all results. + tsks = [ + ("@" + deco_line) + for deco_line in [deco for deco in collapsed.split("@") if deco] + ] + # Take into account that other decorators my be over @task, so it is + # necessary to collapse the function stack + tasks_list = [] + tsk = "" + for task in tsks: + if any(map(task.startswith, PREFIXES)): + tsk += task + if task.startswith("@task"): + tsk += task + tasks_list.append(tsk) + tsk = "" + elif not any(map(task.startswith, PREFIXES)): + # If a decorator over the function is provided, it will + # have to be included in the last task + tasks_list[-1] += task + # Add functions to dictionary by function name: + for task in tasks_list: + # Example: "@task(returns=int)\ndef mytask(v):\n return v+1" + task_code = task.strip() + task_header = task.split("\ndef")[1] + task_name = task_header.replace("(", " (").split(" ")[1].strip() + tasks[task_name] = task_code + + # old = { + # "imports": file_imports, + # "globals": globs, + # "classes": classes, + # "functions": functions, + # "tasks": tasks, + # } + # return old + return file_imports, globs, classes, functions, tasks + + +# ####################### +# CODE UPDATE FUNCTIONS # +# ####################### + + +def _update_imports( + new_imports: typing.List[str], old_imports: typing.List[str] +) -> typing.List[str]: + """Update imports. + + Compare the old imports against the new ones and returns the old imports + with the new imports that did not exist previously. + + :param new_imports: All new imports. + :param old_imports: All old imports. + :return: A list of imports as strings. + """ + not_in_imports = [] + for i in new_imports: + already = False + for j in old_imports: + if i == j: + already = True + if not already: + not_in_imports.append(i) + # Merge the minimum imports + imports = old_imports + not_in_imports + return imports + + +def _update_globals( + new_globals: typing.Dict[str, str], old_globals: typing.Dict[str, str] +) -> typing.Dict[str, str]: + """Update global variables. + + Compare the old globals against the new ones and returns the old globals + with the new globals that did not exist previously. + + :param new_globals: All new globals. + :param old_globals: All old globals. + :return: A list of globals as strings. + """ + if len(old_globals) == 0: + return new_globals + for global_name in list(new_globals.keys()): + if ( + DEBUG + and global_name in old_globals + and (not new_globals[global_name] == old_globals[global_name]) + ): + print( + "WARNING! Global variable " + + global_name + + " has been redefined (the previous will be deprecated)." + ) + old_globals[global_name] = new_globals[global_name] + return old_globals + + +def _update_classes( + new_classes: typing.Dict[str, str], old_classes: typing.Dict[str, str] +) -> typing.Dict[str, str]: + """Update classes. + + Compare the old classes against the new ones. This function is essential + due to the fact that a jupyter-notebook user may rewrite a function and + the latest version is the one that needs to be kept. + + :param new_classes: dictionary containing all classes (last version). + :param old_classes: dictionary containing the existing classes. + :return: dictionary with the merging result (keeping all classes and + updating the old ones). + """ + if len(old_classes) == 0: + return new_classes + for class_name in list(new_classes.keys()): + if ( + DEBUG + and class_name in old_classes + and (not new_classes[class_name] == old_classes[class_name]) + ): + __show_redefinition_warning("Class", class_name) + old_classes[class_name] = new_classes[class_name] + return old_classes + + +def _update_functions( + new_functions: typing.Dict[str, str], old_functions: typing.Dict[str, str] +) -> typing.Dict[str, str]: + """Update functions. + + Compare the old functions against the new ones. This function is essential + due to the fact that a jupyter-notebook user may rewrite a function and + the latest version is the one that needs to be kept. + + :param new_functions: dictionary containing all functions (last version). + :param old_functions: dictionary containing the existing functions. + :return: dictionary with the merging result (keeping all functions and + updating the old ones). + """ + if len(old_functions) == 0: + return new_functions + for function_name in list(new_functions.keys()): + if ( + DEBUG + and function_name in old_functions + and ( + not new_functions[function_name] + == old_functions[function_name] + ) + ): + __show_redefinition_warning("Function", function_name) + old_functions[function_name] = new_functions[function_name] + return old_functions + + +def _update_tasks( + new_tasks: typing.Dict[str, str], old_tasks: typing.Dict[str, str] +) -> typing.Dict[str, str]: + """Update task decorated functions. + + Compare the old tasks against the new ones. This function is essential due + to the fact that a jupyter-notebook user may rewrite a task and the latest + version is the one that needs to be kept. + + :param new_tasks: new tasks code. + :param old_tasks: existing tasks. + :return: dictionary with the merging result. + """ + if not new_tasks: + # when new_tasks is empty, means that the update was triggered by a + # class task. No need to update as a tasks since the class has already + # been updated + pass + else: + task_name = list(new_tasks.keys())[0] + if ( + DEBUG + and task_name in old_tasks + and (not new_tasks[task_name] == old_tasks[task_name]) + ): + __show_redefinition_warning("Task", task_name) + old_tasks[task_name] = new_tasks[task_name] + return old_tasks + + +def __show_redefinition_warning(kind: str, name: str) -> None: + """Show a warning notifying the redefinition of "kind" type. + + :returns: None. + """ + print( + f"WARNING! {kind} {name} has been redefined " + f"(the previous will be deprecated)." + ) + + +# ####################### +# FILE UPDATE FUNCTIONS # +# ####################### + + +def _update_code_file( + new_imports: typing.List[str], + new_globals: typing.Dict[str, str], + new_classes: typing.Dict[str, str], + new_functions: typing.Dict[str, str], + new_tasks: typing.Dict[str, str], + file_path: str, +) -> None: + """Write the results to the code file used by the workers. + + :param new_imports: new imports. + :param new_globals: new global variables. + :param new_classes: new classes. + :param new_functions: new functions. + :param new_tasks: new tasks. + :param file_path: File to update. + :return: None. + """ + with open(file_path, "w") as code_file: + # Write imports + for i in new_imports: + code_file.write(i) + code_file.write("\n") + # Write globals separator + code_file.write(SEPARATORS["globals_separator"] + "\n") + # Write globals + if len(new_globals) == 0: + code_file.write("\n") + else: + for _, global_value in list(new_globals.items()): + for line in global_value: + code_file.write(line) + code_file.write("\n") + code_file.write("\n") + # Write classes separator + code_file.write(SEPARATORS["classes_separator"] + "\n") + # Write classes + if len(new_classes) == 0: + code_file.write("\n") + else: + for _, class_value in list(new_classes.items()): + for line in class_value: + code_file.write(line) + code_file.write("\n") + code_file.write("\n") + # Write functions separator + code_file.write(SEPARATORS["functions_separator"] + "\n") + # Write functions + if len(new_functions) == 0: + code_file.write("\n") + else: + for _, function_value in list(new_functions.items()): + for line in function_value: + code_file.write(line) + code_file.write("\n") + code_file.write("\n") + # Write tasks separator + code_file.write(SEPARATORS["tasks_separator"] + "\n") + # Write tasks + if len(new_tasks) == 0: + code_file.write("\n") + else: + for _, task_value in list(new_tasks.items()): + for line in task_value: + code_file.write(line) + code_file.write("\n") + code_file.write("\n") + code_file.flush() diff --git a/examples/dds/pycompss/util/interactive/monitor.py b/examples/dds/pycompss/util/interactive/monitor.py new file mode 100644 index 00000000..c9f3ea7d --- /dev/null +++ b/examples/dds/pycompss/util/interactive/monitor.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - monitor. + +Provides auxiliary methods for the interactive mode. +""" + +import os + +from pycompss.util.typing_helper import typing + +from pycompss.util.interactive.state import display +from pycompss.util.interactive.state import supports_dynamic_state +from pycompss.util.interactive.state import __get_play_widget +from pycompss.util.interactive.state import HTML +from pycompss.util.interactive.state import tabulate + + +def show_monitoring_information(log_path): + """Show the monitoring information widget. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + if supports_dynamic_state(): + + def play_widget( + i: typing.Any, # pylint: disable=unused-argument + ) -> None: + __show_monitoring_info(log_path) + + play = __get_play_widget(play_widget, interval=250) + display(play) # noqa + else: + __show_monitoring_info(log_path) + + +def read_monitoring_file(log_path: str) -> typing.List[str]: + """Read the monitoring file. + + :param log_path: Absolute path of the log folder. + :return: List of messages in monitoring file. + """ + # monitoring_file = os.path.join(log_path, "monitoring.txt") + monitoring_file = os.environ["COMPSS_MONITORING_FILE"] + with open(monitoring_file, "r") as monitoring_fd: + contents = [line.rstrip() for line in monitoring_fd] + return contents + + +def __show_monitoring_info(log_path: str) -> None: + """Show tasks status. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + monitoring_info = read_monitoring_file(log_path) + if len(monitoring_info) == 0: + # Do not show anything if there is no information to display + return + # Display table with values + display(HTML(tabulate.tabulate(monitoring_info, tablefmt="html"))) diff --git a/examples/dds/pycompss/util/interactive/outwatcher.py b/examples/dds/pycompss/util/interactive/outwatcher.py new file mode 100644 index 00000000..0a054a2d --- /dev/null +++ b/examples/dds/pycompss/util/interactive/outwatcher.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Output watcher. + +Keeps track of the stdout/stderr and saves all ERRORs or ISSUEs into a +queue that can be checked with the events or on the runtime closing. +""" + +import os +import threading +import time +from queue import Queue + +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + + +class StdWatcher: + """Standard output and standard error watcher class. + + This class implements the stdout and stderr files watcher for the + interactive executions. + + To this end, starts a new thread in charge of observing the files + where the runtime is dumping the stdout and stderr. If detects + an error or more, pushes those lines into a queue, that can be + later consulted. + """ + + __slots__ = ["running", "messages"] + + def __init__(self) -> None: + """Create a new StdWatcher object. + + :returns: None. + """ + self.running = False + self.messages = Queue() # type: Queue[str] + + @staticmethod + def __watcher__( + fd_out: typing.TextIO, fd_err: typing.TextIO + ) -> typing.Iterator[str]: # pylint: disable=unused-argument + """Look for new lines in fd_out and fd_err. + + Static method that checks the stderr file descriptor looking + for new lines added at the end. + It is enabled to also look into the stdout file descriptor, but + currently not being used. + + :param fd_out: Standard output file descriptor. + :param fd_err: Standard error file descriptor. + :return: Yields each line found in the fd_err. + """ + while True: + line = fd_err.readline() + if line: + yield line.strip() + else: + time.sleep(0.1) + + def __std_follower__(self, out_file_name: str, err_file_name: str) -> None: + """Look for errors within out_file_name and err_file_name. + + Opens the out and error files and looks inside them thanks to the + __watcher__ generator. This function puts into the queue any line + of the error file which starts with "[ERRMGR]". + + :param out_file_name: Output file name. + :param err_file_name: Error file name. + :return: None. + """ + with open(out_file_name, "r") as fd_out: + with open(err_file_name, "r") as fd_err: + for line in self.__watcher__(fd_out, fd_err): + if self.running: + if line.startswith("[ERRMGR]"): + self.messages.put(str(line)) + else: + # Stop following std + return + + def start_watching(self) -> None: + """Start a new thread to monitor the stdout and stderr files provided. + + :return: None. + """ + if COMPSs.is_redirected(): + self.running = True + out_file_name, err_file_name = COMPSs.get_redirection_file_names() + thread = threading.Thread( + target=self.__std_follower__, + args=(out_file_name, err_file_name), + ) + thread.start() + else: + raise PyCOMPSsException("Can not find the stdout and stderr.") + + def stop_watching(self, clean: bool = True) -> None: + """Stop the monitoring thread and cleans the redirection files. + + :param clean: Remove the redirection files. + :return: None. + """ + self.running = False + if clean: + out_file_name, err_file_name = COMPSs.get_redirection_file_names() + if os.path.exists(out_file_name): + os.remove(out_file_name) + if os.path.exists(err_file_name): + os.remove(err_file_name) + + def get_messages(self) -> list: + """Get messages found in standard output and standard error. + + Retrieves the current messages stored in the queue as a list + of strings (one per line reported by the stdout and stderr files). + + :return: A list with the reported messages. + """ + current_messages = [] + while not self.messages.empty(): + current_messages.append(self.messages.get()) + return current_messages + + +# Global instance of the STD watcher +STDW = StdWatcher() diff --git a/examples/dds/pycompss/util/interactive/state.py b/examples/dds/pycompss/util/interactive/state.py new file mode 100644 index 00000000..4ece4123 --- /dev/null +++ b/examples/dds/pycompss/util/interactive/state.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - State. + +Provides auxiliary methods for the interactive mode to get the state. +""" + +import os +from collections import defaultdict +from xml.etree import ElementTree + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.typing_helper import typing + +try: + from IPython.display import HTML # noqa + from IPython.display import display # noqa + + MISSING_DEPENDENCY = "tabulate" # noqa + import tabulate # noqa + + MISSING_DEPENDENCY = "matplotlib" # noqa + import matplotlib.pyplot as plt # noqa + + MISSING_DEPENDENCY = "numpy" # noqa + import numpy as np # noqa + + np.random.seed(0) + MISSING_DEPENDENCY = "None" # noqa +except ImportError: + HTML = None # type: ignore + display = None # type: ignore + plt = None # type: ignore + +try: + import ipywidgets as widgets # noqa +except ImportError: + widgets = None # type: ignore + + +def supports_dynamic_state() -> bool: + """Check if the state can be displayed with widgets. + + :return: True if widgets available. False otherwise. + """ + return widgets is not None + + +def check_monitoring_file(log_path: str) -> bool: + """Check if there monitoring file exists or not. + + :param log_path: Absolute path of the log folder. + :return: True if the compss monitoring file exists. False otherwise. + """ + compss_state_xml = get_compss_state_xml(log_path) + return os.path.exists(compss_state_xml) + + +def get_compss_state_xml(log_path: str) -> str: + """Check if there is any missing pkg and return the status xml full path. + + :param log_path: Absolute path of the log folder. + :return: The compss state full path. + """ + if MISSING_DEPENDENCY != "None": + raise PyCOMPSsException(f"Missing {MISSING_DEPENDENCY} package.") + compss_state_xml = os.path.join(log_path, "monitor", "COMPSs_state.xml") + return compss_state_xml + + +def parse_state_xml(log_path: str, field: str) -> typing.Any: + """Convert the given xml to dictionary. + + :param log_path: Absolute path of the log folder. + :param field: Field name to retrieve. + :return: The content as dictionary (unless CoresInfo, which is a list). + """ + state_xml = get_compss_state_xml(log_path) + tree = ElementTree.parse(state_xml) + root = tree.getroot() + state_xml_dict = element_tree_to_dict(root) + if state_xml_dict["COMPSsState"][field] == "": + print("Please, submit any task in order to see the information") + return None + if field == "TasksInfo": + return state_xml_dict["COMPSsState"][field]["Application"] + if field == "CoresInfo": + # This is a list if there is more than one core info + # Otherwise it is a dictionary with the only core + cores_info = state_xml_dict["COMPSsState"][field]["Core"] + if isinstance(cores_info, dict): + return [cores_info] + return cores_info + if field == "Statistics": + return state_xml_dict["COMPSsState"][field]["Statistic"] + if field == "ResourceInfo": + return state_xml_dict["COMPSsState"][field]["Resource"] + raise PyCOMPSsException("Unsupported status field") + + +def element_tree_to_dict(element_tree: ElementTree.Element) -> dict: + """Convert an element tree into a dictionary recursively. + + :param element_tree: Element tree. + :return: Dictionary. + """ + build_dict = { + element_tree.tag: {} if element_tree.attrib else None + } # type: dict + children = list(element_tree) + if children: + def_dict = defaultdict(list) + for child_dict in map(element_tree_to_dict, children): + for key, value in child_dict.items(): + def_dict[key].append(value) + build_dict = { + element_tree.tag: { + key: value[0] if len(value) == 1 else value + for key, value in def_dict.items() + } + } + if element_tree.attrib: + build_dict[element_tree.tag].update( + ("@" + key, value) for key, value in element_tree.attrib.items() + ) + if element_tree.text: + text = element_tree.text.strip() + if children or element_tree.attrib: + if text: + build_dict[element_tree.tag]["#text"] = text + else: + build_dict[element_tree.tag] = text + return build_dict + + +def show_tasks_info(log_path: str) -> None: + """Show tasks info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + if supports_dynamic_state(): + + def play_widget( + i: typing.Any, + ) -> None: # pylint: disable=unused-argument + __show_tasks_info(log_path) + + play = __get_play_widget(play_widget) + display(play) # noqa + else: + __show_tasks_info(log_path) + + +def __show_tasks_info(log_path: str) -> None: + """Show tasks info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + cores_info = parse_state_xml(log_path, "CoresInfo") + if cores_info is None: + # Do not show anything if there is no information to display + return + labels = [ + "Signature", + "ExecutedCount", + "MaxExecutionTime", + "MeanExecutionTime", + "MinExecutionTime", + ] + cores = [] + for core in cores_info: + new_task = [] + for label in labels: + if label == "Signature": + new_task.append(core["Impl"][label].split(".")[-1]) + elif label == "ExecutedCount": + new_task.append(int(core["Impl"][label])) + else: + new_task.append(int(core["Impl"][label]) / 1000) + cores.append(new_task) + # Display graph + task_names = [row[0] for row in cores] + task_names[0] = "TaskName" + task_count = [row[1] for row in cores] + task_max = [row[2] for row in cores] + task_mean = [row[3] for row in cores] + task_min = [row[4] for row in cores] + fig = plt.figure() + ax1 = fig.add_subplot(1, 2, 1) + ax2 = fig.add_subplot(1, 2, 2) + errs = [[], []] # type: typing.List[typing.List[str]] + for i in range(len(task_names)): + min_ = task_mean[i] - task_min[i] + max_ = task_max[i] - task_mean[i] + errs[0].append(min_) + errs[1].append(max_) + ax1.errorbar(task_names, task_mean, yerr=errs, lw=3, fmt="ok") + ax1.set_ylabel("Time (seconds)") + ax1.set_xlabel("Task name") + task_mean = [mean * 1000 for mean in task_mean] + ax2.scatter( + task_names, + task_count, + s=task_mean, + c=np.random.rand(len(task_names)), + alpha=0.5, + ) + ax2.set_ylim(0, max(task_count)) + ax2.ticklabel_format(axis="y", style="plain") + ax2.set_ylabel("Amount of tasks") + ax2.set_xlabel("Task name") + plt.tight_layout() + plt.show() + # Display table with values + display( + HTML(tabulate.tabulate(cores, tablefmt="html", headers=labels)) + ) # noqa + + +def show_tasks_status(log_path: str) -> None: + """Show tasks status. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + if supports_dynamic_state(): + + def play_widget(i): # pylint: disable=unused-argument + # type: (typing.Any) -> None + __show_tasks_status(log_path) + + play = __get_play_widget(play_widget) + display(play) # noqa + else: + __show_tasks_info(log_path) + + +def __show_tasks_status(log_path: str) -> None: + """Show tasks status. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + tasks_info_dict = parse_state_xml(log_path, "TasksInfo") + if tasks_info_dict is None: + # Do not show anything if there is no information to display + return + # Display graph + labels = ["InProgress", "Completed"] + sizes = [tasks_info_dict[labels[0]], tasks_info_dict[labels[1]]] + explode = (0, 0) + _, ax1 = plt.subplots() # first return (ignored) is fig1 + colors = ["b", "g"] + ax1.pie( + sizes, + explode=explode, + labels=labels, + colors=colors, + shadow=True, + startangle=90, + ) + ax1.axis("equal") + plt.show() + # Display table with values + labels, values = __plain_lists(tasks_info_dict) + display( + HTML(tabulate.tabulate([values], tablefmt="html", headers=labels)) + ) # noqa + + +def show_statistics(log_path: str) -> None: + """Show statistics info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + if supports_dynamic_state(): + + def play_widget( + i: typing.Any, + ) -> None: # pylint: disable=unused-argument + __show_statistics(log_path) + + play = __get_play_widget(play_widget) + display(play) # noqa + else: + __show_statistics(log_path) + + +def __show_statistics(log_path: str) -> None: + """Show statistics info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + statistics_dict = parse_state_xml(log_path, "Statistics") + if statistics_dict is None: + # Do not show anything if there is no information to display + return + # Display table with values + labels = [statistics_dict["Key"]] + values = [statistics_dict["Value"]] + display( + HTML(tabulate.tabulate([values], tablefmt="html", headers=labels)) + ) # noqa + + +def show_resources_status(log_path: str) -> None: + """Show resources status info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + if supports_dynamic_state(): + + def play_widget( + i: typing.Any, + ) -> None: # pylint: disable=unused-argument + __show_resources_status(log_path) + + play = __get_play_widget(play_widget) + display(play) # noqa + else: + __show_resources_status(log_path) + + +def __show_resources_status(log_path: str) -> None: + """Show resources status info. + + :param log_path: Absolute path of the log folder. + :return: None. + """ + resource_info_dict = parse_state_xml(log_path, "ResourceInfo") + if resource_info_dict is None: + # Do not show anything if there is no information to display + return + # Display table with values + labels, values = __plain_lists(resource_info_dict) + display( + HTML(tabulate.tabulate([values], tablefmt="html", headers=labels)) + ) # noqa + + +def __plain_lists(dictionary: dict) -> typing.Tuple[list, list]: + """Convert a dictionary to two lists. + + IMPORTANT! Removes last element. + + :param dictionary: Dictionary to plain. + :return: Labels and values. + """ + labels = [] + values = [] + for key, value in dictionary.items(): + labels.append(key) + values.append(value) + labels.pop() + values.pop() + return labels, values + + +def __get_play_widget( + function: typing.Callable, interval: int = 5000 +) -> widgets.interactive: + """Generate play widget. + + :param function: Function to associate with Play. + :param interval: Refresh interval. + :return: Play widget. + """ + play = widgets.interactive( + function, + i=widgets.Play( + value=0, + min=0, + max=500, + step=1, + interval=interval, + description="Press play", + disabled=False, + ), + ) + return play diff --git a/examples/dds/pycompss/util/interactive/utils.py b/examples/dds/pycompss/util/interactive/utils.py new file mode 100644 index 00000000..f1123b55 --- /dev/null +++ b/examples/dds/pycompss/util/interactive/utils.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Interactive - Utils. + +Provides auxiliary methods for the interactive mode. +""" + +from pycompss.util.typing_helper import typing + + +def parameters_to_dict( + log_level: str, + debug: bool, + o_c: bool, + graph: bool, + trace: bool, + monitor: int, + project_xml: str, + resources_xml: str, + summary: bool, + task_execution: str, + storage_impl: str, + storage_conf: str, + streaming_backend: str, + streaming_master_name: str, + streaming_master_port: str, + task_count: int, + app_name: str, + uuid: str, + log_dir: str, + master_working_dir: str, + extrae_cfg: str, + extrae_final_directory: str, + comm: str, + conn: str, + master_name: str, + master_port: str, + scheduler: str, + jvm_workers: str, + cpu_affinity: str, + gpu_affinity: str, + fpga_affinity: str, + fpga_reprogram: str, + profile_input: str, + profile_output: str, + scheduler_config: str, + external_adaptation: bool, + propagate_virtual_environment: bool, + mpi_worker: bool, + worker_cache: typing.Union[bool, str], + shutdown_in_node_failure: bool, + io_executors: int, + env_script: str, + reuse_on_block: bool, + nested_enabled: bool, + tracing_task_dependencies: bool, + trace_label: str, + extrae_cfg_python: str, + wcl: int, + cache_profiler: bool, + ear: bool, + data_provenance: bool, + checkpoint_policy: str, + checkpoint_params: str, + checkpoint_folder: str, +) -> dict: + """Convert all given parameters into a dictionary.""" + all_vars = { + "log_level": log_level, + "debug": debug, + "o_c": o_c, + "graph": graph, + "trace": trace, + "monitor": monitor, + "project_xml": project_xml, + "resources_xml": resources_xml, + "summary": summary, + "task_execution": task_execution, + "storage_impl": storage_impl, + "storage_conf": storage_conf, + "streaming_backend": streaming_backend, + "streaming_master_name": streaming_master_name, + "streaming_master_port": streaming_master_port, + "task_count": task_count, + "app_name": app_name, + "uuid": uuid, + "log_dir": log_dir, + "master_working_dir": master_working_dir, + "extrae_cfg": extrae_cfg, + "extrae_final_directory": extrae_final_directory, + "comm": comm, + "conn": conn, + "master_name": master_name, + "master_port": master_port, + "scheduler": scheduler, + "jvm_workers": jvm_workers, + "cpu_affinity": cpu_affinity, + "gpu_affinity": gpu_affinity, + "fpga_affinity": fpga_affinity, + "fpga_reprogram": fpga_reprogram, + "profile_input": profile_input, + "profile_output": profile_output, + "scheduler_config": scheduler_config, + "external_adaptation": external_adaptation, + "propagate_virtual_environment": propagate_virtual_environment, + "mpi_worker": mpi_worker, + "worker_cache": worker_cache, + "shutdown_in_node_failure": shutdown_in_node_failure, + "io_executors": io_executors, + "env_script": env_script, + "reuse_on_block": reuse_on_block, + "nested_enabled": nested_enabled, + "tracing_task_dependencies": tracing_task_dependencies, + "trace_label": trace_label, + "extrae_cfg_python": extrae_cfg_python, + "wcl": wcl, + "cache_profiler": cache_profiler, + "ear": ear, + "data_provenance": data_provenance, + "checkpoint_policy": checkpoint_policy, + "checkpoint_params": checkpoint_params, + "checkpoint_folder": checkpoint_folder, + } + return all_vars diff --git a/examples/dds/pycompss/util/jvm/__init__.py b/examples/dds/pycompss/util/jvm/__init__.py new file mode 100644 index 00000000..bf61d77d --- /dev/null +++ b/examples/dds/pycompss/util/jvm/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the jvm utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/jvm/parser.py b/examples/dds/pycompss/util/jvm/parser.py new file mode 100644 index 00000000..d795841c --- /dev/null +++ b/examples/dds/pycompss/util/jvm/parser.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - JVM - Configuration Parser. + +This file contains all methods required to parse the jvm options file. +""" + +from pycompss.util.typing_helper import typing + + +def convert_to_dict( + jvm_opt_file: str, +) -> typing.Dict[str, typing.Union[bool, str]]: + """Convert the JVM parameters of jvm_opt_file into a dictionary. + + :param jvm_opt_file: JVM parameters file. + :return: Dictionary with the parameters specified on the file. + """ + opts = {} # type: typing.Dict[str, typing.Union[bool, str]] + with open(jvm_opt_file) as jvm_opt_file_fd: + for line in jvm_opt_file_fd: + line = line.strip() + if line: + if line.startswith("-XX:"): + # These parameters have no value + key = line.split(":")[1].replace("\n", "") + opts[key] = True + elif line.startswith("-D"): + key = line.split("=")[0] + value = line.split("=")[1].replace("\n", "") + value = value.strip() + opts[key] = value + else: + key = line.replace("\n", "") + opts[key] = True + return opts diff --git a/examples/dds/pycompss/util/location.py b/examples/dds/pycompss/util/location.py new file mode 100644 index 00000000..f942d522 --- /dev/null +++ b/examples/dds/pycompss/util/location.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Location. + +This file defines the internal PyCOMPSs location functions. +""" + +import inspect +import os + + +def _get_module_path() -> str: + """Find the current file path when __file__ is not available. + + The '__file__' variable is not available if the code is compiled + with mypy. + :return: The current module path. + """ + # Get the current call stack + stack = inspect.stack() + # Get the filename of the current file + file_name = stack[0].filename + # Get the directory path of the current file + dir_name = os.path.dirname(file_name) + return dir_name + + +def _get_current_path() -> str: + """Get the current path. + + Checks if __file__ fails (with mypy) and retrieves it as an alternative + way. + :return: The current module path. + """ + try: + return os.path.dirname(os.path.realpath(__file__)) + except KeyError: + # Using mypy __file__ does not exist. + return _get_module_path() + + +def get_binding_location() -> str: + """Get the binding main path. + + Removes the "util" folder from the last path: + /path/to/pycompss/util -> /path/to/pycompss + + :return: The PyCOMPSs binding main path. + """ + return os.path.dirname(_get_current_path()) diff --git a/examples/dds/pycompss/util/logger/__init__.py b/examples/dds/pycompss/util/logger/__init__.py new file mode 100644 index 00000000..f0164a1e --- /dev/null +++ b/examples/dds/pycompss/util/logger/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the logger utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/logger/cfg/__init__.py b/examples/dds/pycompss/util/logger/cfg/__init__.py new file mode 100644 index 00000000..f065aba2 --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the logger cfg files.""" diff --git a/examples/dds/pycompss/util/logger/cfg/logging_container_worker.json b/examples/dds/pycompss/util/logger/cfg/logging_container_worker.json new file mode 100644 index 00000000..3f11cefb --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/logging_container_worker.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)s %(module)s %(process)d %(thread)d - %(message)s" + }, + "medium": { + "format": "%(asctime)s %(levelname)s %(name)s %(module)s - %(message)s" + }, + "simple": { + "format": "%(name)s %(module)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "medium", + "stream": "ext://sys.stdout" + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "medium", + "stream": "ext://sys.stderr" + } + }, + "loggers": { + "dataclay": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "hecuba": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "redis": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "storage": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "user": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" } + }, + "root": { + "level": "DEBUG" + } +} diff --git a/examples/dds/pycompss/util/logger/cfg/logging_gat_worker.json b/examples/dds/pycompss/util/logger/cfg/logging_gat_worker.json new file mode 100644 index 00000000..3f11cefb --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/logging_gat_worker.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)s %(module)s %(process)d %(thread)d - %(message)s" + }, + "medium": { + "format": "%(asctime)s %(levelname)s %(name)s %(module)s - %(message)s" + }, + "simple": { + "format": "%(name)s %(module)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "medium", + "stream": "ext://sys.stdout" + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "medium", + "stream": "ext://sys.stderr" + } + }, + "loggers": { + "dataclay": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "hecuba": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "redis": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "storage": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "user": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" } + }, + "root": { + "level": "DEBUG" + } +} diff --git a/examples/dds/pycompss/util/logger/cfg/logging_master.json b/examples/dds/pycompss/util/logger/cfg/logging_master.json new file mode 100644 index 00000000..746e8b9e --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/logging_master.json @@ -0,0 +1,69 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)s %(module)s %(process)d %(thread)d - %(message)s" + }, + "medium": { + "format": "%(asctime)s %(levelname)s %(name)s %(module)s - %(message)s" + }, + "simple": { + "format": "%(name)s %(module)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "medium", + "stream": "ext://sys.stdout" + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "medium", + "stream": "ext://sys.stderr" + }, + "debug_master_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "medium", + "filename": "pycompss.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8", + "delay": "true" + }, + "info_master_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "medium", + "filename": "pycompss.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8", + "delay": "true" + }, + "error_master_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "ERROR", + "formatter": "medium", + "filename": "pycompss.err", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8", + "delay": "true" + } + }, + "loggers": { + "dataclay": { "level": "DEBUG", "handlers": ["debug_master_file_handler", "error_master_file_handler"], "propagate": "no" }, + "hecuba": { "level": "DEBUG", "handlers": ["debug_master_file_handler", "error_master_file_handler"], "propagate": "no" }, + "redis": { "level": "DEBUG", "handlers": ["debug_master_file_handler", "error_master_file_handler"], "propagate": "no" }, + "storage": { "level": "DEBUG", "handlers": ["debug_master_file_handler", "error_master_file_handler"], "propagate": "no" }, + "user": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" } + }, + "root": { + "level": "DEBUG" + } +} diff --git a/examples/dds/pycompss/util/logger/cfg/logging_mpi_worker.json b/examples/dds/pycompss/util/logger/cfg/logging_mpi_worker.json new file mode 100644 index 00000000..3f11cefb --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/logging_mpi_worker.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)s %(module)s %(process)d %(thread)d - %(message)s" + }, + "medium": { + "format": "%(asctime)s %(levelname)s %(name)s %(module)s - %(message)s" + }, + "simple": { + "format": "%(name)s %(module)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "medium", + "stream": "ext://sys.stdout" + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "medium", + "stream": "ext://sys.stderr" + } + }, + "loggers": { + "dataclay": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "hecuba": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "redis": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "storage": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" }, + "user": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" } + }, + "root": { + "level": "DEBUG" + } +} diff --git a/examples/dds/pycompss/util/logger/cfg/logging_worker.json b/examples/dds/pycompss/util/logger/cfg/logging_worker.json new file mode 100644 index 00000000..5b26a064 --- /dev/null +++ b/examples/dds/pycompss/util/logger/cfg/logging_worker.json @@ -0,0 +1,69 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)s %(module)s %(process)d %(thread)d - %(message)s" + }, + "medium": { + "format": "%(asctime)s %(levelname)s %(name)s %(module)s - %(message)s" + }, + "simple": { + "format": "%(name)s %(module)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "medium", + "stream": "ext://sys.stdout" + }, + "error_console": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "medium", + "stream": "ext://sys.stderr" + }, + "debug_worker_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "DEBUG", + "formatter": "medium", + "filename": "binding_worker.out", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8", + "delay": "true" + }, + "info_worker_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "medium", + "filename": "binding_worker.out", + "maxBytes": 10485760, + "backupCount": 99, + "encoding": "utf8", + "delay": "true" + }, + "error_worker_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "ERROR", + "formatter": "medium", + "filename": "binding_worker.err", + "maxBytes": 10485760, + "backupCount": 99, + "encoding": "utf8", + "delay": "true" + } + }, + "loggers": { + "dataclay": { "level": "DEBUG", "handlers": ["debug_worker_file_handler", "error_worker_file_handler"], "propagate": "no" }, + "hecuba": { "level": "DEBUG", "handlers": ["debug_worker_file_handler", "error_worker_file_handler"], "propagate": "no" }, + "redis": { "level": "DEBUG", "handlers": ["debug_worker_file_handler", "error_worker_file_handler"], "propagate": "no" }, + "storage": { "level": "DEBUG", "handlers": ["debug_worker_file_handler", "error_worker_file_handler"], "propagate": "no" }, + "user": { "level": "DEBUG", "handlers": ["console", "error_console"], "propagate": "no" } + }, + "root": { + "level": "DEBUG" + } +} diff --git a/examples/dds/pycompss/util/logger/helpers.py b/examples/dds/pycompss/util/logger/helpers.py new file mode 100644 index 00000000..ee3d9417 --- /dev/null +++ b/examples/dds/pycompss/util/logger/helpers.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Logger - Helpers. + +This file contains all logging methods. +""" + +import copy +import json +import logging +import os +import pathlib +from contextlib import contextmanager +from logging import config + +from pycompss import PYCOMPSS_HOME +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.logger.level import check_log_level +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.typing_helper import typing + + +LOG_CFG_PATH = os.path.join(PYCOMPSS_HOME, "util", "logger", "cfg") +CONFIG_FUNC = config.dictConfig +# Keep configs to avoid read the cfg many times +CONFIGS = {} # type: typing.Dict[str, dict] + + +def clean_log_configs() -> None: + """Remove all stored log configurations. + + :return: None + """ + CONFIGS.clear() + + +def __get_logging_cfg_file(remittent: str) -> str: + """Retrieve the logging configuration file. + + :param remittent: Logging remittent. + :return: Logging configuration file. + :raise PyCOMPSsException: Unsupported log remittent. + """ + log_cfg = os.path.join(LOG_CFG_PATH, f"logging_{remittent}.json") + if os.path.isfile(log_cfg): + return log_cfg + raise PyCOMPSsException( + f"Logging configuration file {log_cfg} does not exist!" + ) + + +def __read_log_config_file( + remittent: str, log_level: str +) -> typing.Dict[str, dict]: + """Read the required config file and update. + + If already read, retrieves from global dictionary. + + :param remittent: Logging remittent. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :return: Configuration file content. + """ + log_config_file = __get_logging_cfg_file(remittent) + + # Get base configuration + if log_config_file in CONFIGS: + conf = CONFIGS[log_config_file] + else: + with open(log_config_file, "rt") as lcf_fd: + conf = json.loads(lcf_fd.read()) + CONFIGS[log_config_file] = conf + + # Check if the log level is supported + check_log_level(log_level) + # Adapt log level to configuration format and set off log level with error + log_level = log_level.upper() + if log_level == "OFF": + log_level = "ERROR" + # Add all loggers (from current python files) + conf = __add_loggers(conf, remittent, log_level) + # Update configuration with log level + conf = __update_log_level(conf, log_level) + + return conf + + +def __find_source_files(path, extensions): + """Find all binding source files. + + :param path: Folder to look recursively. + :param extensions: List of extensions considered as source. + :return: Generator with all source files found. + """ + main_home = os.path.dirname(path) + if not main_home.endswith("/"): + main_home = f"{main_home}/" + for root, _, files in os.walk(path): + for file in files: + if "." in file and file.split(".")[1] in extensions: + relative = root.replace(main_home, "") + file_path = os.path.join(relative, file) + yield str(pathlib.Path(file_path).with_suffix("")) + + +def __add_loggers( + conf: typing.Dict[str, dict], remittent: str, log_level: str +) -> typing.Dict[str, dict]: + """Add all needed loggers to the configuration. + + Traverses the whole binding folder looking for python files and includes + them in the base configuration. + + :para conf: Configuration file content. + :param remittent: Logging remittent. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :return: Updated configuration file content. + """ + loggers = [] + for source_file in __find_source_files(PYCOMPSS_HOME, ["py"]): + logger = source_file.replace("/", ".") + loggers.append(logger) + + if remittent == LOG_REMITTENT.MASTER: + if log_level == "DEBUG": + handlers = [ + "debug_master_file_handler", + "error_master_file_handler", + ] + elif log_level == "INFO": + handlers = [ + "info_master_file_handler", + "error_master_file_handler", + ] + else: + # ERROR level + # "debug_master_file_handler" as well? + handlers = ["error_master_file_handler"] + elif remittent == LOG_REMITTENT.WORKER: + if log_level == "DEBUG": + handlers = [ + "debug_worker_file_handler", + "error_worker_file_handler", + ] + elif log_level == "INFO": + handlers = [ + "info_worker_file_handler", + "error_worker_file_handler", + ] + else: + # ERROR level + # "debug_worker_file_handler" as well? + handlers = ["error_worker_file_handler"] + elif remittent == LOG_REMITTENT.GAT_WORKER: + handlers = ["console", "error_console"] + elif remittent == LOG_REMITTENT.MPI_WORKER: + handlers = ["console", "error_console"] + elif remittent == LOG_REMITTENT.CONTAINER_WORKER: + handlers = ["console", "error_console"] + else: + raise PyCOMPSsException( + f"Unexpected remittent received updating loggers in config: " + f"{remittent}" + ) + for logger in loggers: + conf["loggers"][logger] = { + "level": log_level, + "handlers": handlers, + "propagate": "no", + } + return conf + + +def __update_log_level( + conf: typing.Dict[str, dict], log_level: str +) -> typing.Dict[str, dict]: + """Update the log level in the configuration. + + :para conf: Configuration file content. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :return: Updated configuration file content. + """ + conf["root"]["level"] = log_level + conf["handlers"]["console"]["level"] = log_level + return conf + + +def init_logging(remittent: str, log_level: str, log_path: str) -> None: + """Initialize logging in master. + + :param remittent: Logging remittent. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :param log_path: Json log files path. + :return: None. + """ + conf = __read_log_config_file(remittent, log_level) + handler = "error_master_file_handler" + if handler in conf["handlers"]: + errors_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = log_path + errors_file + handler = "info_master_file_handler" + if handler in conf["handlers"]: + info_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = log_path + info_file + handler = "debug_master_file_handler" + if handler in conf["handlers"]: + debug_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = log_path + debug_file + CONFIG_FUNC(conf) + + +def init_logging_worker( + remittent: str, + log_level: str, + tracing: bool, + job_out: typing.Optional[str] = None, + job_err: typing.Optional[str] = None, +) -> None: + """Initialize logger in worker. + + :param remittent: Logging remittent. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :param tracing: If tracing is enabled (the log dir changes). + :param job_out: out file path. + :param job_err: err file path. + :return: None. + """ + conf = __read_log_config_file(remittent, log_level) + if tracing: + # The workspace is within the folder "workspace/python" + # Remove the last folder + handler = "error_worker_file_handler" + if handler in conf["handlers"]: + errors_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = "../" + errors_file + handler = "info_worker_file_handler" + if handler in conf["handlers"]: + info_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = "../" + info_file + handler = "debug_worker_file_handler" + if handler in conf["handlers"]: + debug_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = "../" + debug_file + # If within task + if job_err: + handler = "error_worker_file_handler" + if handler in conf["handlers"]: + conf["handlers"][handler]["filename"] = job_err + if job_out: + handler = "info_worker_file_handler" + if handler in conf["handlers"]: + conf["handlers"][handler]["filename"] = job_out + handler = "debug_worker_file_handler" + if handler in conf["handlers"]: + conf["handlers"][handler]["filename"] = job_out + + CONFIG_FUNC(conf) + + +def init_logging_worker_piper( + remittent: str, log_level: str, log_dir: str +) -> None: + """Initialize logger in piper worker. + + :param remittent: Logging remittent. + :param log_level: Log level [ "trace"|"debug"|"info"|"api"|"off" ]. + :param log_dir: Log directory. + :return: None. + """ + conf = __read_log_config_file(remittent, log_level) + handler = "error_worker_file_handler" + if handler in conf["handlers"]: + errors_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = os.path.join( + log_dir, errors_file + ) + handler = "info_worker_file_handler" + if handler in conf["handlers"]: + info_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = os.path.join( + log_dir, info_file + ) + handler = "debug_worker_file_handler" + if handler in conf["handlers"]: + debug_file = conf["handlers"][handler].get("filename") + conf["handlers"][handler]["filename"] = os.path.join( + log_dir, debug_file + ) + CONFIG_FUNC(conf) + + +def add_new_logger(logger_name: str) -> None: + """Add a new logger for the user in the master. + + Creates a copy of the "user" or "piper_worker" logger and renames it with + the given logger_name. + + :param logger_name: New logger name. + :returns: None + """ + # Get "user" logger information (used as source) + log_config_file = list(CONFIGS.keys())[0] + users_logger = CONFIGS[log_config_file]["loggers"]["user"] + # Copy "user" logger and set its new name + new_logger = copy.deepcopy(users_logger) + CONFIGS[log_config_file]["loggers"][logger_name] = new_logger + # Update the logger with the new handler + CONFIG_FUNC(CONFIGS[log_config_file]) + + +@contextmanager +def swap_logger_name( + logger: logging.Logger, new_name: str +) -> typing.Iterator[None]: + """Swap the current logger with the new one. + + :param logger: Logger facility. + :param new_name: Logger name. + :return: None. + """ + previous_name = logger.name + logger.name = new_name + yield # here the code runs + logger.name = previous_name + + +@contextmanager +def keep_logger() -> typing.Iterator[None]: + """Do nothing with the logger. + + It is used when the swap_logger_name does not need to be applied. + + :return: None + """ + yield # here the code runs diff --git a/examples/dds/pycompss/util/logger/level.py b/examples/dds/pycompss/util/logger/level.py new file mode 100644 index 00000000..84969d07 --- /dev/null +++ b/examples/dds/pycompss/util/logger/level.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This file contains the logger level functions.""" + +from pycompss.util.exceptions import PyCOMPSsException + + +class _LogLevel: # pylint: disable=too-few-public-methods + """Supported logging levels.""" + + INFO = "info" + TRACE = "debug" + DEBUG = "debug" + API = "off" + OFF = "off" + + +LOG_LEVEL = _LogLevel() + + +def get_log_level(level: str) -> str: + """Translate the given level to the real log level tag. + + :param level: Logging level. + :return: Real log level tag. + :raise PyCOMPSsException: Unsupported logging level + """ + level = level.lower() + if level == "info": + return LOG_LEVEL.INFO + if level == "trace": + return LOG_LEVEL.TRACE + if level == "debug": + return LOG_LEVEL.DEBUG + if level == "api": + return LOG_LEVEL.API + if level == "off": + return LOG_LEVEL.OFF + raise PyCOMPSsException("Unsupported logging level (get).") + + +def check_log_level(level: str) -> bool: + """Check that the log level is supported. + + :param level: Log level. + return: If the log level is supported + """ + if level in [ + LOG_LEVEL.API, + LOG_LEVEL.DEBUG, + LOG_LEVEL.INFO, + LOG_LEVEL.OFF, + LOG_LEVEL.TRACE, + ]: + return True + raise PyCOMPSsException("Unsupported logging level (check).") diff --git a/examples/dds/pycompss/util/logger/remittent.py b/examples/dds/pycompss/util/logger/remittent.py new file mode 100644 index 00000000..386b5033 --- /dev/null +++ b/examples/dds/pycompss/util/logger/remittent.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This file contains the logger remittent functions.""" + + +class _LogRemittent: # pylint: disable=too-few-public-methods + """Supported PyCOMPSs modules.""" + + MASTER = "master" + WORKER = "worker" + GAT_WORKER = "gat_worker" + MPI_WORKER = "mpi_worker" + CONTAINER_WORKER = "container_worker" + + +LOG_REMITTENT = _LogRemittent() diff --git a/examples/dds/pycompss/util/mpi/__init__.py b/examples/dds/pycompss/util/mpi/__init__.py new file mode 100644 index 00000000..c3e8731b --- /dev/null +++ b/examples/dds/pycompss/util/mpi/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the mpi utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/mpi/helper.py b/examples/dds/pycompss/util/mpi/helper.py new file mode 100644 index 00000000..27a8204a --- /dev/null +++ b/examples/dds/pycompss/util/mpi/helper.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - MPI - Helper. + +This file contains all MPI helper methods. +""" + +from mpi4py import MPI +from pycompss.util.typing_helper import typing + +comm = MPI.COMM_WORLD +size = comm.size +rank = comm.rank + + +def rank_distributor( + collection_layout: typing.Tuple[int, int, int] +) -> typing.List[int]: + """Distribute mpi ranks to data given a collection layout. + + :param collection_layout: Layout of the collection. + :return distribution: distribution of rank x + """ + block_count, block_length, stride = collection_layout + if block_count == -1: + block_count = size + if stride == -1: + stride = 1 + if block_length == -1: + block_length = 1 + distribution = [] + if block_count == size: + # Number of block bigger than processes + # (one block per process) + offset = rank * stride + distribution = list(range(offset, offset + block_length)) + else: + # If number of blocks is bigger than processes + # (blocks round-robin distributed) + block = rank + while block < block_count: + offset = block * stride + offset_bl = offset + block_length + distribution = distribution + list(range(offset, offset_bl)) + block = block + size + return distribution diff --git a/examples/dds/pycompss/util/objects/__init__.py b/examples/dds/pycompss/util/objects/__init__.py new file mode 100644 index 00000000..241599a6 --- /dev/null +++ b/examples/dds/pycompss/util/objects/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the object utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/objects/properties.py b/examples/dds/pycompss/util/objects/properties.py new file mode 100644 index 00000000..84a4a138 --- /dev/null +++ b/examples/dds/pycompss/util/objects/properties.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Objects - Properties. + +Offers some functions that check properties about objects. +For example, check if an object belongs to a module and so on. +""" + +import builtins +import inspect +import os +import sys +from collections import OrderedDict + +from pycompss.util.typing_helper import typing + + +def get_module_name(path: str, file_name: str) -> str: + """Get the module name considering its path and filename. + + Example: runcompss -d src/kmeans.py + path = "test/kmeans.py" + file_name = "kmeans" (without py extension) + return mod_name = "test.kmeans" + + :param path: relative path until the file.py from where the runcompss has + been executed. + :param file_name: python file to be executed name + (without the py extension). + :return: the module name + """ + dirs = path.split(os.path.sep) + mod_name = file_name + i = len(dirs) - 1 + while i > 0: + new_l = len(path) - (len(dirs[i]) + 1) + path = path[0:new_l] + if "__init__.py" in os.listdir(path): + # directory is a package + i -= 1 + mod_name = dirs[i] + "." + mod_name + else: + break + return mod_name + + +def get_wrapped_source(function: typing.Callable) -> str: + """Get the text of the source code for the given function. + + :param function: Input function. + :return: Source. + """ + if hasattr(function, "__wrapped__"): + # Has __wrapped__: going deep + wrapped = function.__wrapped__ # type: ignore + return get_wrapped_source(wrapped) + # Returning getsource + try: + source = inspect.getsource(function) + except TypeError: + # This is a numba jit declared task + py_func = function.py_func # type: ignore + source = inspect.getsource(py_func) + return source + + +def is_module_available(module_name: str) -> bool: + """Check if a module is available in the current Python installation. + + :param module_name: Name of the module. + :return: True if the module is available. False otherwise. + """ + try: + try: + import importlib + + _importlib = importlib # type: typing.Any + module = _importlib.util.find_spec(module_name) # noqa + except AttributeError: + # This can only happen in conda + import imp # noqa # Deprecated in python 3 + + module = imp.find_module(module_name) # noqa + if module: + return True + return False + except ImportError: + return False + + +def is_basic_iterable(obj: typing.Any) -> bool: + """Check if an object is a basic iterable. + + By basic iterable we want to mean objects that are iterable and from a + basic type. + + :param obj: Object to be analysed. + :return: True if obj is a basic iterable (see list below). False otherwise. + """ + return isinstance(obj, (list, tuple, bytearray, set, frozenset)) + + +def is_dict(obj: typing.Any) -> bool: + """Check if an object is a dictionary. + + :param obj: Object to be analysed. + :return: True if obj is of dict type. False otherwise. + """ + return isinstance(obj, (dict, OrderedDict)) + + +def object_belongs_to_module(obj: typing.Any, module_name: str) -> bool: + """Check if a given object belongs to a given module (or some sub-module). + + :param obj: Object to be analysed. + :param module_name: Name of the module we want to check. + :return: True if obj belongs to the given module. False otherwise. + """ + return any(module_name == x for x in type(obj).__module__.split(".")) + + +def create_object_by_con_type(con_type: str) -> typing.Any: + """Create an "empty" object knowing its class name. + + :param con_type: object type info in : format. + :return: "empty" object of a type. + """ + path, class_name = con_type.split(":") + if hasattr(builtins, class_name): + _obj = getattr(builtins, class_name) + return _obj() + + directory, module_name = os.path.split(path) + module_name = os.path.splitext(module_name)[0] + + klass = globals().get(class_name, None) + if klass: + return klass() + + if module_name not in sys.modules: + sys.path.append(directory) + module = __import__(module_name) + sys.modules[module_name] = module + else: + module = sys.modules[module_name] + + klass = getattr(module, class_name) + ret = klass() + return ret + + +######################################################### +# DEPRECATED FUNCTIONS # +######################################################### +# These functions are not currently used within # +# PyCOMPSs, but they are kept just in case needed. # +######################################################### +# +# def get_defining_class(method): +# # type: (...) -> ... +# """ Get the class from the given a method. +# +# :param method: Method to check its defining class. +# :return: Class which method belongs. None if not found. +# """ +# if inspect.ismethod(method): +# for cls in inspect.getmro(method.__self__.__class__): +# if cls.__dict__.get(method.__name__) is method: +# return cls +# if inspect.isfunction(method): +# return getattr(inspect.getmodule(method), +# method.__qualname__.split(".", +# 1)[0].rsplit(".", 1)[0]) +# # Return not required since None would have been implicitly +# # returned anyway +# return None +# +# +# def get_top_decorator(code, decorator_keys): +# # type: (list, list) -> str +# """ Retrieves the decorator which is on top of the current task +# decorators stack. +# +# :param code: Tuple which contains the task code to analyse and the number +# of lines of the code. +# :param decorator_keys: Tuple which contains the available decorator keys +# :return: the decorator name in the form "pycompss.api.__name__" +# """ +# # Code has two fields: +# # code[0] = the entire function code. +# # code[1] = the number of lines of the function code. +# func_code = code[0] +# decorators = [code_line.strip() for code_line in +# func_code if code_line.strip().startswith("@")] +# # Could be improved if it stops when the first line without @ is found, +# # but we have to be care if a decorator is commented (# before @) +# # The strip is due to the spaces that appear before functions +# # definitions, such as class methods. +# for dk in decorator_keys: +# for d in decorators: +# if d.startswith("@" + dk): +# # each decorator __name__ +# return "pycompss.api." + dk.lower() +# # If no decorator is found, the current decorator is the one to register +# return __name__ +# +# +# def get_wrapped_source_lines(f): +# # type: (...) -> tuple +# """ Gets a list of source lines and starting line number for the given +# function. +# +# :param f: Input function. +# :return: Source lines. +# """ +# if hasattr(f, "__wrapped__"): +# # has __wrapped__, apply the same function to the wrapped content +# return _get_wrapped_source_lines(f.__wrapped__) +# else: +# # Returning get_source_lines +# try: +# source_lines = inspect.getsourcelines(f) +# except TypeError: +# # This is a numba jit declared task +# source_lines = inspect.getsourcelines(f.py_func) +# return source_lines +# +# +# def _get_wrapped_source_lines(f): +# # type: (...) -> tuple +# """ Recursive function which gets a list of source lines and starting +# line number for the given function. +# +# :param f: Input function. +# :return: Source lines. +# """ +# if hasattr(f, "__wrapped__"): +# # has __wrapped__, going deep +# return _get_wrapped_source_lines(f.__wrapped__) +# else: +# # Returning get_source_lines +# try: +# source_lines = inspect.getsourcelines(f) +# except TypeError: +# # This is a numba jit declared task +# source_lines = inspect.getsourcelines(f.py_func) +# return source_lines diff --git a/examples/dds/pycompss/util/objects/replace.py b/examples/dds/pycompss/util/objects/replace.py new file mode 100644 index 00000000..feec03fe --- /dev/null +++ b/examples/dds/pycompss/util/objects/replace.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +SOURCE CODE TAKEN FROM BEN KURTOVIC'S GITHUB REPO replace.py FILE. + +https://gist.github.com/earwig/28a64ffb94d51a608e3d + ++ Added typing ++ pydocstyle +""" + +import ctypes +import sys +from ctypes import pythonapi as api +from types import ( + BuiltinFunctionType, + GetSetDescriptorType, + MemberDescriptorType, + MethodType, +) +from pycompss.util.typing_helper import typing + +try: + import guppy + from guppy.heapy import Path +except Exception: + raise ImportError("Cannot use local decorator without guppy!") + +hp = guppy.hpy() + + +def _w(x: int): # NOSONAR + def f(): # NOSONAR + x # NOSONAR + + return f + + +if sys.version_info < (3, 0): + CellType = type(_w(0).func_closure[0]) +else: + CellType = type(_w(0).__closure__[0]) + +del _w + + +# ----------------------------------------------------------------------------- +def _write_struct_attr(addr: int, value: typing.Any, add_offset: int) -> None: + ptr_size = ctypes.sizeof(ctypes.py_object) + ptrs_in_struct = (3 if hasattr(sys, "getobjects") else 1) + add_offset + offset = ptrs_in_struct * ptr_size + ctypes.sizeof(ctypes.c_ssize_t) + ref = ctypes.byref(ctypes.py_object(value)) + ctypes.memmove(addr + offset, ref, ptr_size) + + +def _replace_attribute( + source: typing.Any, + rel: str, + new: typing.Any, +) -> None: + if isinstance(source, (MethodType, BuiltinFunctionType)): + if rel == "__self__": + # Note: PyMethodObject->im_self and PyCFunctionObject->m_self + # have the same offset + _write_struct_attr(id(source), new, 1) + return + if rel == "im_self": + return # Updated via __self__ + if isinstance(source, type): + if rel == "__base__": + return # Updated via __bases__ + if rel == "__mro__": + return # Updated via __bases__ when important, otherwise futile + if isinstance(source, (GetSetDescriptorType, MemberDescriptorType)): + if rel == "__objclass__": # NOSONAR + _write_struct_attr(id(source), new, 0) + return + try: + setattr(source, rel, new) + except TypeError: + print("Unknown R_ATTRIBUTE (read-only):", rel, type(source)) + except AttributeError: + print("Unknown R_ATTRIBUTE (read-only):", rel, type(source)) + + +def _replace_indexval( + source: typing.Any, rel: typing.Any, new: typing.Any +) -> None: + if isinstance(source, tuple): + temp = list(source) + temp[rel] = new + replace(source, tuple(temp)) + return + source[rel] = new + + +def _replace_indexkey( + source: typing.Any, rel: typing.Any, new: typing.Any +) -> None: + source[new] = source.pop(list(source.keys())[rel]) + + +def _replace_interattr(source: typing.Any, rel: str, new: typing.Any) -> None: + if isinstance(source, CellType): + api.PyCell_Set(ctypes.py_object(source), ctypes.py_object(new)) + return + if rel == "ob_type": + source.__class__ = new + return + print("Unknown R_INTERATTR:", rel, type(source)) + + +def _replace_local_var(source: typing.Any, rel: str, new: typing.Any) -> None: + source.f_locals[rel] = new + api.PyFrame_LocalsToFast(ctypes.py_object(source), ctypes.c_int(0)) + + +_RELATIONS = { + Path.R_ATTRIBUTE: _replace_attribute, + Path.R_INDEXVAL: _replace_indexval, + Path.R_INDEXKEY: _replace_indexkey, + Path.R_INTERATTR: _replace_interattr, + Path.R_LOCAL_VAR: _replace_local_var, +} + + +def _path_key_func(path: Path) -> int: + reltype = type(path.path[1]).__bases__[0] + return 1 if reltype is Path.R_ATTRIBUTE else 0 + + +def replace(old: typing.Any, new: typing.Any) -> None: + """Replace the old object with the new object. + + :param old: Old object. + :param new: New object. + :returns: None. + """ + for path in sorted(hp.iso(old).pathsin, key=_path_key_func): + relation = path.path[1] + try: + func = _RELATIONS[type(relation).__bases__[0]] + except KeyError: + print("Unknown relation:", relation, type(path.src.theone)) + continue + func(path.src.theone, relation.r, new) + + +# Commented out due to fails with mypy. +# # --------------------------------------------------------------------------- +# class A(object): # NOSONAR +# def func(self): # NOSONAR +# return self # NOSONAR +# +# +# class B(object): # NOSONAR +# pass # NOSONAR +# +# +# class X(object): # NOSONAR +# cattr = None # type: typing.Any +# iattr = None # type: typing.Any +# +# +# def sure(obj): +# def inner(): +# return obj +# +# return inner +# +# +# def gen(obj): +# while 1: +# yield obj +# +# +# class S(object): # NOSONAR +# # __slots__ = ("p", "q") # NOSONAR +# +# def __init__(self): +# self.p = None # type: typing.Any +# self.q = None # type: typing.Any +# +# +# class T(object): # NOSONAR +# # __slots__ = ("p", "q") # NOSONAR +# +# def __init__(self): +# self.p = None # type: typing.Any +# self.q = None # type: typing.Any +# +# +# class U(object): # NOSONAR +# pass # NOSONAR +# +# +# class V(object): # NOSONAR +# pass # NOSONAR +# +# +# class W(U): # NOSONAR +# pass # NOSONAR +# +# +# # --------------------------------------------------------------------------- +# a = A() +# b = B() +# +# X.cattr = a +# x = X() +# x.iattr = a +# d = {a: a} +# L = [a] +# t = (a,) +# f = a.func +# meth = a.__sizeof__ +# clo = sure(a) +# g = gen(a) +# s = S() +# s.p = a +# u = U() +# ud = U.__dict__["__dict__"] +# s.q = S +# sd = S.q +# +# +# # --------------------------------------------------------------------------- +# def examine_vars(id1, id2, id3): +# def ex(v, id_): # NOSONAR +# return str(v) + ("" if id(v) == id_ else " - ERROR!") +# print("dict (local var): ", ex(a, id1)) +# print("dict (class attr): ", ex(X.cattr, id1)) +# print("dict (inst attr): ", ex(x.iattr, id1)) +# print("dict (key): ", ex(list(d.keys())[0], id1)) +# print("dict (value): ", ex(list(d.values())[0], id1)) +# print("list: ", ex(L[0], id1)) +# print("tuple: ", ex(t[0], id1)) +# print("method (instance): ", ex(f(), id1)) +# print("method (builtin): ", ex(meth.__self__, id1)) +# print("closure: ", ex(clo(), id1)) +# print("frame (generator): ", ex(next(g), id1)) +# print("slots: ", ex(s.p, id1)) +# print("class (instance): ", ex(type(u), id2)) +# print("class (subclass): ", ex(W.__bases__[0], id2)) +# print("class (g/s descr): ", ex(ud.__get__(u, U) or type(u), id2)) +# print("class (mem descr): ", ex(sd.__get__(s, S), id3)) +# +# +# # For testing purposes: +# # if __name__ == "__main__": +# # examine_vars(id(a), id(U), id(S)) +# # print("-" * 35) +# # replace(a, b) +# # replace(U, V) +# # replace(S, T) +# # print("-" * 35) +# # examine_vars(id(b), id(V), id(T)) diff --git a/examples/dds/pycompss/util/objects/sizer.py b/examples/dds/pycompss/util/objects/sizer.py new file mode 100644 index 00000000..c123c7e9 --- /dev/null +++ b/examples/dds/pycompss/util/objects/sizer.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Objects - Sizing algorithms. + +This file contains the object sizing algorithm method. +""" + +from __future__ import print_function + +import reprlib +from collections import deque +from itertools import chain +from sys import getsizeof +from sys import stderr + +from pycompss.util.typing_helper import typing + +from collections.abc import Iterator + + +def _dict_handler(given_dict: dict) -> Iterator: + """Convert dictionary to dictionary handler. + + :param given_dict: Dictionary. + :return: Dictionary handler. + """ + return chain.from_iterable(given_dict.items()) + + +def _user_object_handler(user_object: typing.Any) -> Iterator: + """Convert user object to dictionary handler. + + :param user_object: User object. + :return: Dictionary handler. + """ + return chain.from_iterable(user_object.__dict__.items()) + + +def total_sizeof( + given_object: typing.Any, + handlers: typing.Optional[Iterator] = None, + verbose: bool = False, +) -> int: + """Calculate the size of an object. + + Returns the approximate memory footprint an object and all of its contents. + Automatically finds the contents of the following builtin containers and + their subclasses: tuple, list, deque, dict, set and frozenset. + To search other containers, add handlers to iterate over their contents: + handlers = {SomeContainerClass: iter, + OtherContainerClass: OtherContainerClass.get_elements} + + :param given_object: Object to get its size. + :param handlers: Handlers. + :param verbose: Verbose mode [ True | False ] (default: False). + :return: Total size of the object. + """ + all_handlers = { + tuple: iter, + list: iter, + deque: iter, + dict: _dict_handler, + set: iter, + frozenset: iter, + } # type: typing.Dict[typing.Any, typing.Any] + if type(given_object) not in all_handlers and hasattr( + given_object, "__dict__" + ): + # It is something else include its __dict__ + all_handlers[type(given_object)] = _user_object_handler + if handlers is not None: + all_handlers.update(handlers) # user handlers take precedence + seen = set() # track which object id's have already been seen + default_size = getsizeof(0) # estimate sizeof object without __sizeof__ + + def sizeof(obj: typing.Any) -> int: + """Calculate the size o the given object in bytes. + + :param obj: Object to measure + :return: The object size in bytes + """ + if id(obj) in seen: # do not double count the same object + return 0 + seen.add(id(obj)) + new_size = getsizeof(obj, default_size) + + if verbose: + print(new_size, type(obj), reprlib.repr(obj), file=stderr) + + for typ, handler in all_handlers.items(): + if isinstance(obj, typ): + new_size += sum(map(sizeof, handler(obj))) + break + return new_size + + return sizeof(given_object) diff --git a/examples/dds/pycompss/util/objects/util.py b/examples/dds/pycompss/util/objects/util.py new file mode 100644 index 00000000..d721dde0 --- /dev/null +++ b/examples/dds/pycompss/util/objects/util.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Objects - Utils. + +This file contains the utilities that could be needed for the objects. +""" + +from pycompss.util.typing_helper import typing + + +def group_iterable( + iterable: typing.Iterable, number: int +) -> typing.Iterator[typing.Any]: + """Return a list of lists containing n elements. + + s -> [(s0, s1, s2, ..., sn-1), + (sn, sn+1, sn+2, ..., s2n-1), + (s2n, s2n+1, s2n+2, ..., s3n-1), + ..., + (sNn, sNn+1, sNn+2, ..., sMn-1)]" + :param iterable: Iterable to group. + :param number: Number of elements per group. + :return: A list of lists where the inner contain n elements. + """ + return zip(*[iter(iterable)] * number) diff --git a/examples/dds/pycompss/util/process/__init__.py b/examples/dds/pycompss/util/process/__init__.py new file mode 100644 index 00000000..fd2945b9 --- /dev/null +++ b/examples/dds/pycompss/util/process/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the process utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/process/manager.py b/examples/dds/pycompss/util/process/manager.py new file mode 100644 index 00000000..45857411 --- /dev/null +++ b/examples/dds/pycompss/util/process/manager.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Process - Manager. + +This file centralizes the multiprocessing management. +It helps to homogenize the behaviour between linux and mac. +""" + +import multiprocessing +from multiprocessing import Manager +from multiprocessing.managers import SyncManager # Used only for typing +from multiprocessing import Process # Used only for typing +from multiprocessing import Queue # Used only for typing + +from pycompss.util.typing_helper import typing + +try: + from multiprocessing.shared_memory import SharedMemory # noqa + from multiprocessing.shared_memory import ShareableList # noqa + from multiprocessing.managers import SharedMemoryManager # noqa +except ImportError: + # Unsupported in python < 3.8 + SharedMemory = None # type: ignore + ShareableList = None # type: ignore + SharedMemoryManager = None # type: ignore + +# Next import can be used only for typing with Python >= 3.8 +# from multiprocessing.managers import DictProxy +DictProxy = typing.Any # type: ignore + + +# Global variables +LOCK = None + + +def initialize_multiprocessing() -> None: + """Set global mechanism to start multiprocessing processes. + + https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods + + CAUTION: Using fork even in MacOS. + WARNING: This method must be called only once and at the very beginning. + + :return: None + """ + global LOCK + try: + multiprocessing.set_start_method("fork") + except AttributeError: + # Unsupported set_start_method (python 2 mainly). + # Use default start method. + pass + except RuntimeError: + # Already initialized + pass + manager = multiprocessing.Manager() + LOCK = manager.RLock() + + +def new_process() -> Process: + """Instantiate a new empty process. + + :return: Empty process. + """ + return multiprocessing.Process() + + +def new_queue() -> Queue: + """Instantiate a new queue. + + :return: New queue + """ + return multiprocessing.Queue() + + +def new_event() -> typing.Any: + """Instantiate a new event. + + :return: New event + """ + return multiprocessing.Event() + + +def new_manager() -> SyncManager: + """Instantiate a new empty multiprocessing manager. + + :return: Empty multiprocessing manager. + """ + return Manager() + + +def create_process( + target: typing.Callable, args: tuple = (), prepend_lock: bool = False +) -> Process: + """Create a new process for the given target with the provided arguments. + + :param target: Function to execute in a multiprocessing process. + :param args: function arguments. + :param prepend_lock: Include a lock for mutex purposes. + :return: New process. + """ + if prepend_lock: + args = (LOCK,) + tuple(args) + process = multiprocessing.Process(target=target, args=args) + return process + + +def create_shared_memory_manager( + address: typing.Tuple[str, int], authkey: typing.Optional[bytes] +) -> SharedMemoryManager: + """Create a new shared memory manager process. + + At the given address with the provided authkey. + + :param address: Shared memory manager address (IP, PORT). + :param authkey: Shared memory manager authentication key. + :return: New process. + """ + smm = SharedMemoryManager(address=address, authkey=authkey) + return smm + + +def create_proxy_dict() -> DictProxy: + """Create a proxy dictionary. + + Aimed at sharing the information across workers within the same node. + + :return: Proxy dictionary. + """ + manager = new_manager() + cache_ids = manager.dict() # type: DictProxy + return cache_ids diff --git a/examples/dds/pycompss/util/process/preloader.py b/examples/dds/pycompss/util/process/preloader.py new file mode 100644 index 00000000..8e5bc80e --- /dev/null +++ b/examples/dds/pycompss/util/process/preloader.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Process - Preloader. + +This file centralizes the library preloading functions. +It helps to import in parallel all indicated libraries. +""" +import logging +import os +import pkgutil +from concurrent.futures import ThreadPoolExecutor +from pycompss.util.typing_helper import typing + + +PRELOAD_PYTHON_LIBRARIES_EVNAME = "PRELOAD_PYTHON_LIBRARIES" + + +def preimports() -> bool: + """Check if imports have to be preloaded. + + True if the PRELOAD_PYTHON_LIBRARIES_EVNAME environment variable is + defined. + :return: If imports have to be preloaded. + """ + return PRELOAD_PYTHON_LIBRARIES_EVNAME in os.environ + + +def __default_imports() -> typing.List[str]: + """Retrieve the default imports list. + + :return: The default libraries to be preimported. + """ + default_imports = [ + "pickle", + "dill", + "numpy", + "pycompss.api.commons.decorator", + "pycompss.api.commons.error_msgs", + "pycompss.api.commons.implementation_types", + "pycompss.api.commons.private_tasks", + "pycompss.api.commons.data_type", + "pycompss.api.commons.constants", + "pycompss.api.api", + "pycompss.api.task", + "pycompss.api.constraint", + "pycompss.api.implement", + "pycompss.api.mpi", + "pycompss.api.multinode", + "pycompss.api.on_failure", + "pycompss.api.parameter", + "pycompss.api.prolog", + "pycompss.api.epilog", + "pycompss.api.reduction", + "pycompss.runtime.management.classes", + "pycompss.runtime.management.COMPSs", + "pycompss.runtime.management.direction", + "pycompss.runtime.management.object_tracker", + "pycompss.runtime.management.synchronization", + "pycompss.runtime.management.link.direct", + "pycompss.runtime.management.link.messages", + "pycompss.runtime.management.link.separate", + "pycompss.runtime.mpi.keys", + "pycompss.runtime.mpi", + "pycompss.runtime.start.initialization", + "pycompss.runtime.start", + "pycompss.runtime.task.arguments", + "pycompss.runtime.task.commons", + "pycompss.runtime.task.features", + "pycompss.runtime.task.keys", + "pycompss.runtime.task.master", + "pycompss.runtime.task.parameter", + "pycompss.runtime.task.shared_args", + "pycompss.runtime.task.worker", + "pycompss.runtime.task.definitions.arguments", + "pycompss.runtime.task.definitions.constraints", + "pycompss.runtime.task.definitions.core_element", + "pycompss.runtime.task.definitions.function", + "pycompss.runtime.task.definitions", + "pycompss.runtime.task.wrappers.psco_stream", + "pycompss.runtime.task.wrappers", + "pycompss.runtime.task", + "pycompss.runtime.binding", + "pycompss.runtime.commons", + "pycompss.runtime", + "pycompss.util.environment.configuration", + "pycompss.util.jvm.parser", + "pycompss.util.logger.helpers", + "pycompss.util.logger.level", + "pycompss.util.logger.remittent", + "pycompss.util.objects.properties", + # "pycompss.util.objects.replace", + "pycompss.util.objects.sizer", + "pycompss.util.objects.util", + "pycompss.util.process.manager", + "pycompss.util.serialization.extended_support", + "pycompss.util.serialization.serializer", + "pycompss.util.std.redirects", + "pycompss.util.storages.persistent", + "pycompss.util.supercomputer.scs", + "pycompss.util.tracing.types_events_master", + "pycompss.util.tracing.types_events_worker", + "pycompss.util.tracing.helpers", + "pycompss.util.warning.modules", + "pycompss.util.arguments", + "pycompss.util.context", + "pycompss.util.exceptions", + "pycompss.util.location", + "pycompss.util.typing_helper", + "pycompss.worker.commons.executor", + "pycompss.worker.commons.worker", + "pycompss.worker.piper.cache.classes", + "pycompss.worker.piper.cache.manager", + "pycompss.worker.piper.cache.profiler", + "pycompss.worker.piper.cache.setup", + "pycompss.worker.piper.cache.tracker", + "pycompss.worker.piper.commons.constants", + "pycompss.worker.piper.commons.executor", + "pycompss.worker.piper.commons.utils", + "pycompss.worker.piper.commons.utils_logger", + "pycompss.worker.piper.piper_worker", + "pycompss", + ] + return default_imports + + +def __load_import(library: str) -> None: + """Import the given library as string. + + :param library: Library name to be imported. + :return: None + """ + try: + __import__(library) + except Exception as e: + if __debug__: + print(f"WARNING: Pre-load import {library} failed: {e}") + + +def preload_imports( + logger: logging.Logger, header: str, subheader: str +) -> None: + """Resolve imports provided by an environment variable. + + This resolving is performed in parallel using threading, so that the + imports are done in the main worker process and inherited by the + executor processes to avoid that each of them has to do them + sequentially. + + :param logger: Logger. + :param header: Header to be shown in the logger messages. + :param subheader: Subheader to be shown in the logger messages. + :return: None + """ + imports = os.environ[PRELOAD_PYTHON_LIBRARIES_EVNAME] + show_memory = False + if __debug__: + try: + import psutil + + process = psutil.Process(os.getpid()) + used_memory = process.memory_info().rss / (1024 * 1024) + logger.debug( + "%s%s - Memory used before imports: %s", + header, + subheader, + str(used_memory), + ) + show_memory = True + except ImportError: + logger.debug( + "%s%s - Could not calculate used memory." + "Install psutil if you want to check it.", + header, + subheader, + str(), + ) + # Get the library names + to_be_imported = __default_imports() + if imports == "ALL": + # If the variable contains ALL, will import all possible packages + # Read all installed packages + for module_info in pkgutil.iter_modules(): + name = module_info.name + if isinstance(name, str): + if ( + not name.startswith("lib") + and "mpi" not in name + and name not in ["setup", "__init__"] + ): + to_be_imported.append(name.strip()) + elif ";" in imports: + # If the variable specifies explicitly a semicolon separated list of + # packages to be pre imported. + for name in imports.split(";"): + if name: + to_be_imported.append(name.strip()) + else: + # Read the whole file + with open(imports) as f: + lines = f.readlines() + # Filter comments and any line that does not contain the import word + full_imports = [] + for line in lines: + if "import" in line and not line.startswith("#"): + full_imports.append(line) + for i in full_imports: + lib = i.split(" ")[1].strip() + to_be_imported.append(lib) + if __debug__: + logger.debug( + "%s%s - Libraries pre-imported: %s", + header, + subheader, + str(len(to_be_imported)), + ) + if show_memory: + used_memory_after = process.memory_info().rss / (1024 * 1024) + logger.debug( + "%s%s - Memory used after imports: %s", + header, + subheader, + str(used_memory_after), + ) + amount = used_memory_after - used_memory + logger.debug( + "%s%s - Memory increase: %s", header, subheader, str(amount) + ) + # Import the libraries using the max amount of cores + pool = ThreadPoolExecutor() + pool.map(__load_import, to_be_imported) + pool.shutdown(wait=True) diff --git a/examples/dds/pycompss/util/serialization/__init__.py b/examples/dds/pycompss/util/serialization/__init__.py new file mode 100644 index 00000000..762383d7 --- /dev/null +++ b/examples/dds/pycompss/util/serialization/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the serialization utils helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/util/serialization/extended_support.py b/examples/dds/pycompss/util/serialization/extended_support.py new file mode 100644 index 00000000..fd39ed8e --- /dev/null +++ b/examples/dds/pycompss/util/serialization/extended_support.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Serialization - Extended support. + +This file contains the serialization extensions. +""" +import types + +from pycompss.util.typing_helper import typing + + +class GeneratorIndicator: + """GeneratorIndicator class.""" + + +def pickle_generator( + f_gen: typing.Generator, + handler: typing.BinaryIO, + serializer: types.ModuleType, +) -> None: + """Pickle a generator and store the serialization result in a file. + + :param f_gen: Generator object. + :param handler: Destination file for pickling generator. + :param serializer: Serializer to use. + """ + # Convert generator to list and pickle (less efficient but more reliable) + # The tuple will be useful to determine when to call unplickle generator. + # Using a key is weak, but otherwise, How can we difference a list from a + # generator when receiving it? + # At least, the key is complicated. + gen_snapshot = (GeneratorIndicator(), list(f_gen)) + serializer.dump(gen_snapshot, handler) + + +def convert_to_generator(lst: list) -> typing.Generator: + """Convert a list into a generator. + + :param lst: List to be converted. + :return: the generator from the list. + """ + return (n for n in lst) diff --git a/examples/dds/pycompss/util/serialization/serializer.py b/examples/dds/pycompss/util/serialization/serializer.py new file mode 100644 index 00000000..9bdaaa51 --- /dev/null +++ b/examples/dds/pycompss/util/serialization/serializer.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Serialization - Serializer/deserializer. + +This file implements the main serialization/deserialization functions. +All serialization/deserialization calls should be made using one of the +following functions: + +- serialize_to_file(obj, file_name) -> dumps the object "obj" to the file + "file_name" +- serialize_to_string(obj) -> dumps the object "obj" to a string +- serialize_to_handler(obj, handler) -> writes the serialized object using + the specified handler it also moves + the handler's pointer to the end of + the dump + +- deserialize_from_file(file_name) -> loads the first object from the tile + "file_name" +- deserialize_from_string(serialized_content) -> loads the first object + from the given string +- deserialize_from_handler(handler) -> deserializes an object using the + given handler, it also leaves the + handler's pointer pointing to the + end of the serialized object +""" + +import gc +import json +import logging # typing purposes +import os +import pickle +import struct +import traceback +import types + +from io import BytesIO + +from pycompss.util.exceptions import SerializerException +from pycompss.util.objects.properties import object_belongs_to_module +from pycompss.util.serialization.extended_support import GeneratorIndicator +from pycompss.util.serialization.extended_support import convert_to_generator +from pycompss.util.serialization.extended_support import pickle_generator +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing + +try: + import dill # noqa + + DILL_AVAILABLE = True +except ImportError: + DILL_AVAILABLE = False + +try: + import numpy + + NUMPY_AVAILABLE = True +except ImportError: + NUMPY_AVAILABLE = False + + +try: + import cupy + + CUPY_AVAILABLE = True +except ImportError: + CUPY_AVAILABLE = False + + +try: + import pyarrow + + PYARROW_AVAILABLE = True +except ImportError: + PYARROW_AVAILABLE = False + +try: + from pyeddl._core import Net as eddlNet + from pyeddl.eddl import serialize_net_to_onnx_string + from pyeddl.eddl import import_net_from_onnx_file + + PYEDDL_AVAILABLE = True +except ImportError: + PYEDDL_AVAILABLE = False + +# GLOBALS + +# LIB2IDX contains as key the serializer and value its associated integer +LIB2IDX = {pickle: 0} # type: typing.Dict[types.ModuleType, int] +if DILL_AVAILABLE: + LIB2IDX[dill] = 1 +if NUMPY_AVAILABLE: + LIB2IDX[numpy] = 2 +if PYARROW_AVAILABLE: + LIB2IDX[pyarrow] = 3 +LIB2IDX[json] = 4 +if PYEDDL_AVAILABLE: + LIB2IDX[eddlNet] = 5 +if CUPY_AVAILABLE: + LIB2IDX[cupy] = 6 +# IDX2LIB contains as key the integer and the value its associated serializer +IDX2LIB = dict( + ((v, k) for (k, v) in LIB2IDX.items()) +) # type: typing.Dict[int, types.ModuleType] +# Max integer +PLATFORM_C_MAXINT = 2 ** ((struct.Struct("i").size * 8 - 1) - 13) +# To force a specific serializer +FORCED_SERIALIZER = -1 # make a serializer the only option for serialization +# Control Garbage Collector +DISABLE_GC = False + + +def get_available_libraries() -> ( + typing.List[typing.Tuple[int, str, typing.Optional[str]]] +): + """Return the available serialization libraries. + + Checks the available serializers and returns a dict of names and + its module for the active serializers. + + :return: Dictionary of available serializers. + """ + active_serializers = [] + for library, priority in LIB2IDX.items(): + try: + active_serializers.append( + (priority, library.__name__, library.__file__) + ) + except AttributeError: + active_serializers.append( + (priority, library.__name__, str(library)) + ) + return active_serializers + + +def get_serializer_priority( + obj: typing.Any, logger: logging.Logger +) -> typing.List[types.ModuleType]: + """Compute the priority of the serializers. + + Returns a list with the available serializers in the most common order + (i.e: the order that will work for almost the 90% of our objects). + + :param obj: Object to be analysed. + :param logger: Logger to output the serialization messages. + :return: The serializers sorted by priority in descending order. + """ + if __debug__: + logger.debug( + "Get serializer priority for object of type: %s" % str(type(obj)) + ) + if FORCED_SERIALIZER > -1: + return [IDX2LIB[FORCED_SERIALIZER]] + primitives = (int, str, bool, float) + # primitives should be (de)serialized with for the compatibility with the + # Runtime- only JSON objects can be deserialized in Java. + if type(obj) in primitives: + return [json, pickle] + serializers = [pickle] + if DILL_AVAILABLE: + serializers = [pickle, dill] + if object_belongs_to_module(obj, "numpy") and NUMPY_AVAILABLE: + return [numpy] + serializers + if object_belongs_to_module(obj, "cupy") and CUPY_AVAILABLE: + return [cupy] + serializers + if object_belongs_to_module(obj, "pyarrow") and PYARROW_AVAILABLE: + return [pyarrow] + serializers + if object_belongs_to_module(obj, "pyeddl") and PYEDDL_AVAILABLE: + return [eddlNet] + serializers + return serializers + + +def serialize_to_handler( + obj: typing.Any, handler: typing.BinaryIO, logger: logging.Logger +) -> None: + """Serialize an object to a handler. + + :param obj: Object to be serialized. + :param handler: A handler object. It must implement methods like write, + writeline and similar stuff. + :param logger: Logger to output the serialization messages. + :return: None. + :raises SerializerException: If something wrong happens during + serialization. + """ + emit_manual_event_explicit( + TRACING_MASTER.binding_serialization_size_type, 0 + ) + if hasattr(handler, "name"): + emit_manual_event_explicit( + TRACING_MASTER.binding_serialization_object_num_type, + (abs(hash(os.path.basename(handler.name))) % PLATFORM_C_MAXINT), + ) + if __debug__: + if isinstance(handler, BytesIO): + logger.debug("Serializing to BytesIO handler: %s" % str(handler)) + else: + logger.debug("Serializing to handler: %s" % str(handler.name)) + if DISABLE_GC: + # Disable the garbage collector while serializing -> more performance? + gc.disable() + # Get the serializer priority + serializer_priority = get_serializer_priority(obj, logger) + if __debug__: + logger.debug("Serializer priority: %s" % str(serializer_priority)) + i = 0 + success = False + original_position = handler.tell() + is_json = False + # Lets try the serializers in the given priority + serialization_issues = [] + while i < len(serializer_priority) and not success: + # Reset the handlers pointer to the first position + handler.seek(original_position) + serializer = serializer_priority[i] + handler.write(bytearray(f"{LIB2IDX[serializer]:04d}", "utf8")) + + if __debug__: + logger.debug( + "Trying to serialize using %s" % str(serializer.__name__) + ) + + # Special case: obj is a generator + if isinstance(obj, types.GeneratorType): + try: + if __debug__: + logger.debug("Serializing a generator") + pickle_generator(obj, handler, serializer) + success = True + except Exception: # noqa + if __debug__: + logger.debug("Failed serializing a generator") + traceback.print_exc() + # General case + else: + try: + if ( + CUPY_AVAILABLE + and serializer is cupy + and isinstance(obj, cupy.ndarray) + ): + if __debug__: + logger.debug("Serializing using cupy...") + serializer.save(handler, obj, allow_pickle=False) + if __debug__: + logger.debug("Serializing using cupy success") + elif ( + NUMPY_AVAILABLE + and serializer is numpy + and isinstance(obj, (numpy.ndarray, numpy.matrix)) + ): + if __debug__: + logger.debug("Serializing using numpy...") + serializer.save(handler, obj, allow_pickle=False) + if __debug__: + logger.debug("Serializing using numpy success") + elif ( + PYARROW_AVAILABLE + and serializer is pyarrow + and object_belongs_to_module(obj, "pyarrow") + ): + if __debug__: + logger.debug("Serializing using pyarrow...") + writer = pyarrow.ipc.new_file(handler, obj.schema) # noqa + writer.write(obj) + writer.close() + if __debug__: + logger.debug("Serializing using pyarrow success") + elif ( + PYEDDL_AVAILABLE + and serializer is eddlNet + and object_belongs_to_module(obj, "pyeddl") + ): + if __debug__: + logger.debug("Serializing using pyeddl...") + handler.write(serialize_net_to_onnx_string(obj, False)) + if __debug__: + logger.debug("Serializing using pyeddl success") + elif serializer is json: + if __debug__: + logger.debug("Serializing using json...") + # JSON doesn't like the binary mode: close handler + h_name = handler.name + handler.close() + # Open the handler in normal mode + reopened_handler = open( + h_name, "w" + ) # pylint: disable=consider-using-with + reopened_handler.write(f"{LIB2IDX[serializer]:04d}") + serializer.dump(obj, reopened_handler) + is_json = True + if __debug__: + logger.debug("Serializing using json success") + else: + if __debug__: + logger.debug( + "Serializing using %s..." + % str(serializer.__name__) + ) + serializer.dump( + obj, handler, protocol=serializer.HIGHEST_PROTOCOL + ) + if __debug__: + logger.debug( + "Serializing using %s success" + % str(serializer.__name__) + ) + success = True + if __debug__: + logger.debug("Serialization accomplished") + except Exception: # noqa + success = False + traceback_exc = traceback.format_exc() + serialization_issues.append((serializer, traceback_exc)) + if __debug__: + logger.debug( + "Could not perform serialization using %s" + % str(serializer.__name__) + ) + i += 1 + if is_json: + serialization_size = reopened_handler.tell() + else: + serialization_size = handler.tell() + emit_manual_event_explicit( + TRACING_MASTER.binding_serialization_size_type, serialization_size + ) + emit_manual_event_explicit( + TRACING_MASTER.binding_serialization_object_num_type, 0 + ) + if DISABLE_GC: + # Enable the garbage collector and force to clean the memory + gc.enable() + gc.collect() + + # if ret_value is None then all the serializers have failed + if not success: + if __debug__: + logger.debug("Serialization TOTALLY FAILED") + try: + traceback.print_exc() + except AttributeError: + # Bug fixed in 3.5 - issue10805 + pass + error_msg = f"Cannot serialize object {obj!r}. Reason:\n" + for line in serialization_issues: + error_msg += f"ERROR with: {line[0]}\n{line[1]}\n" + raise SerializerException(error_msg) + + +def serialize_to_file( + obj: typing.Any, file_name: str, logger: logging.Logger +) -> None: + """Serialize an object to a file. + + :param obj: Object to be serialized. + :param file_name: File name where the object is going to be serialized. + :param logger: Logger to output the serialization messages. + :return: Nothing, it just serializes the object. + """ + with EventInsideWorker(TRACING_WORKER.serialize_to_file_event): + with open(file_name, "wb") as handler: + serialize_to_handler(obj, handler, logger) + + +def serialize_to_file_mpienv( + obj: typing.Any, + file_name: str, + rank_zero_reduce: bool, + logger: logging.Logger, +) -> None: + """Serialize an object to a file for Python MPI Tasks. + + :param obj: Object to be serialized. + :param file_name: File name where the object is going to be serialized. + :param rank_zero_reduce: A boolean to indicate whether objects should be + reduced to MPI rank zero. + False for INOUT objects and True otherwise. + :param logger: Logger to output the serialization messages. + :return: Nothing, it just serializes the object. + """ + with EventInsideWorker(TRACING_WORKER.serialize_to_file_mpienv_event): + if __debug__: + logger.debug("Serializing to file mpienv") + from mpi4py import MPI + + if rank_zero_reduce: + nprocs = MPI.COMM_WORLD.Get_size() + if nprocs > 1: + obj = MPI.COMM_WORLD.reduce([obj], root=0) + if MPI.COMM_WORLD.rank == 0: + serialize_to_file(obj, file_name, logger) + else: + serialize_to_file(obj, file_name, logger) + + +def serialize_to_bytes(obj: typing.Any, logger: logging.Logger) -> bytes: + """Serialize an object to a byte array. + + CAUTION: Used in redis storage integration. + + :param obj: Object to be serialized. + :param logger: Logger to output the serialization messages. + :return: The serialized content. + """ + handler = BytesIO() + serialize_to_handler(obj, handler, logger) + ret = handler.getvalue() + handler.close() + return ret + + +def deserialize_from_handler( + handler: typing.BinaryIO, show_exception: bool, logger: logging.Logger +) -> typing.Any: + """Deserialize an object from a file. + + :param handler: File name from where the object is going to be + deserialized. + :param show_exception: Show exception if happen (only with debug). + :param logger: Logger to output the deserialization messages. + :return: The object and if the handler has to be closed. + :raises SerializerException: If deserialization can not be done. + """ + # Retrieve the used library (if possible) + emit_manual_event_explicit( + TRACING_MASTER.binding_deserialization_size_type, 0 + ) + if hasattr(handler, "name"): + emit_manual_event_explicit( + TRACING_MASTER.binding_deserialization_object_num_type, + (abs(hash(os.path.basename(handler.name))) % PLATFORM_C_MAXINT), + ) + if __debug__: + if isinstance(handler, BytesIO): + logger.debug( + "Deserializing from BytesIO handler: %s" % str(handler) + ) + else: + logger.debug("Deserializing from handler: %s" % str(handler.name)) + original_position = 0 + try: + original_position = handler.tell() + ser_type = handler.read(4) + if ser_type == b"" or ser_type is None: + return None, True + serializer = IDX2LIB[int(ser_type)] + if __debug__: + logger.debug("Using deserializer: %s" % str(serializer.__name__)) + except KeyError as key_error: + # The first 4 bytes return a value that is not within IDX2LIB + handler.seek(original_position) + error_message = "Handler does not refer to a valid PyCOMPSs object" + raise SerializerException(error_message) from key_error + + close_handler = True + try: + if DISABLE_GC: + # Disable the garbage collector while serializing -> performance? + gc.disable() + if CUPY_AVAILABLE and serializer is cupy: + if __debug__: + logger.debug("Cupy available") + logger.debug("Deserializing using cupy") + ret = serializer.load(handler, allow_pickle=False) + elif NUMPY_AVAILABLE and serializer is numpy: + if __debug__: + logger.debug("Numpy available") + logger.debug("Deserializing using numpy") + ret = serializer.load(handler, allow_pickle=False) + elif PYARROW_AVAILABLE and serializer is pyarrow: + if __debug__: + logger.debug("Pyarrow available") + logger.debug("Deserializing using pyarrow") + ret = pyarrow.ipc.open_file(handler) + if isinstance(ret, pyarrow.ipc.RecordBatchFileReader): + close_handler = False + elif PYEDDL_AVAILABLE and serializer is eddlNet: + if __debug__: + logger.debug("Pyeddl available") + logger.debug("Deserializing using pyeddl") + h_name = handler.name + # handler.seek(4) # Ignore first byte? + ret = import_net_from_onnx_file(h_name) + elif serializer is json: + if __debug__: + logger.debug("Deserializing using json") + # Deserialization of json files is not in binary: close handler + h_name = handler.name + handler.close() + # Reopen handler with normal mode + reopened_handler = open(h_name, "r") + reopened_handler.seek(4) # Ignore first byte + ret = serializer.load(reopened_handler) + else: + if __debug__: + logger.debug( + "Deserializing using %s" % str(serializer.__name__) + ) + ret = serializer.load(handler) + # Special case: deserialized obj wraps a generator + if ( + isinstance(ret, tuple) + and ret + and isinstance(ret[0], GeneratorIndicator) + ): + if __debug__: + logger.debug("Recovering a generator") + ret = convert_to_generator(ret[1]) + if DISABLE_GC: + # Enable the garbage collector and force to clean the memory + gc.enable() + gc.collect() + if serializer is json: + deserialization_size = reopened_handler.tell() + else: + deserialization_size = handler.tell() + emit_manual_event_explicit( + TRACING_MASTER.binding_deserialization_size_type, + deserialization_size, + ) + emit_manual_event_explicit( + TRACING_MASTER.binding_deserialization_object_num_type, 0 + ) + if __debug__: + logger.debug("Deserialization accomplished") + return ret, close_handler + except Exception as general_exception: + traceback_exc = traceback.format_exc() + if DISABLE_GC: + gc.enable() + if __debug__: + logger.debug("Deserialization TOTALLY FAILED") + if __debug__ and show_exception: + print(f"ERROR! Deserialization with {str(serializer)} failed.") + try: + traceback.print_exc() + except AttributeError: + # Bug fixed in 3.5 - issue10805 + pass + error_msg_head = ( + f"ERROR: Cannot deserialize object with serializer: {serializer}" + ) + error_msg = f"{error_msg_head}\n{traceback_exc}\n" + raise SerializerException(error_msg) from general_exception + + +def deserialize_from_file( + file_name: str, logger: logging.Logger +) -> typing.Any: + """Deserialize the contents in a given file. + + :param file_name: Name of the file with the contents to be deserialized. + :param logger: Logger to output the deserialization messages. + :return: A deserialized object. + """ + with EventInsideWorker(TRACING_WORKER.deserialize_from_file_event): + handler = open(file_name, "rb") + ret, close_handler = deserialize_from_handler(handler, True, logger) + if close_handler: + handler.close() + return ret + + +def deserialize_from_bytes( + serialized_content_bytes: bytes, + show_exception: bool, + logger: logging.Logger, +) -> typing.Any: + """Deserialize the contents in a given byte array. + + CAUTION: Used in redis storage integration. + + :param serialized_content_bytes: A byte array with serialized contents. + :param show_exception: Show exception if happen (only with debug - + uses to be True). + :param logger: Logger to output the deserialization messages. + :return: A deserialized object. + """ + with EventInsideWorker(TRACING_WORKER.deserialize_from_bytes_event): + handler = BytesIO(serialized_content_bytes) + ret, close_handler = deserialize_from_handler( + handler, show_exception, logger + ) + if close_handler: + handler.close() + return ret + + +def serialize_objects(to_serialize: list, logger: logging.Logger) -> None: + """Serialize a list of objects to file. + + If a single object fails to be serialized, then an Exception by + serialize_to_file will be thrown (and not caught). + The structure of the parameter is: + [(object1, file_name1), ... , (objectN, file_nameN)]. + + :param to_serialize: List of lists to be serialized. Each sublist is a + pair of the form ['object','file name']. + :param logger: Logger to output the deserialization messages. + :return: None. + """ + for obj, file in to_serialize: + serialize_to_file(obj, file, logger) diff --git a/examples/dds/pycompss/util/std/__init__.py b/examples/dds/pycompss/util/std/__init__.py new file mode 100644 index 00000000..a11c34d7 --- /dev/null +++ b/examples/dds/pycompss/util/std/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the stdout and stderr utils helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/util/std/redirects.py b/examples/dds/pycompss/util/std/redirects.py new file mode 100644 index 00000000..46432d0a --- /dev/null +++ b/examples/dds/pycompss/util/std/redirects.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Utils - Std - Redirects. + +This file contains the methods required to redirect the standard output +and standard error grabbing all kind of output and error (e.g. from C +or child processes). +""" + +import ctypes +import fcntl +import io +import os +import sys +from contextlib import contextmanager + +from pycompss.util.exceptions import StandardOutputError +from pycompss.util.exceptions import StandardErrorError +from pycompss.util.typing_helper import typing + +LIBC = ctypes.CDLL(None) # noqa +if sys.platform == "darwin": + C_STDOUT = ctypes.c_void_p.in_dll(LIBC, "__stdoutp") + C_STDERR = ctypes.c_void_p.in_dll(LIBC, "__stderrp") +else: + C_STDOUT = ctypes.c_void_p.in_dll(LIBC, "stdout") + C_STDERR = ctypes.c_void_p.in_dll(LIBC, "stderr") + + +class FDLocker: + """File descriptor locker. + + This class implements a context able to lock a file in order to ensure + that the access to it is done in exclusion. + """ + + def __init__(self, file_name): + """Initialize a FDLocker object (context). + + :param file_name: File path to lock. + """ + self.file_name = file_name + self.file_descriptor = open(file_name, "ab") + + def __enter__(self): + """Lock the file.""" + if self.file_descriptor.closed: + self.file_descriptor = open(self.file_name, "ab") + fcntl.flock(self.file_descriptor.fileno(), fcntl.LOCK_EX) + return self + + def __exit__(self, _type, value, tb): + """Flush and unlock the file.""" + self.file_descriptor.flush() + fcntl.flock(self.file_descriptor.fileno(), fcntl.LOCK_UN) + self.file_descriptor.close() + + +def _dup2(to_fd: int, from_fd: int) -> None: + """Wrap dup2 to do retries if fails. + + dup2 has a race condition in Linux with open, that can result in + error 16 EBUSY. + + :param to_fd: Destination file descriptor. + :param from_fd: Source file descriptor. + :return: None. + """ + retries = 5 + while retries > 0: + try: + os.dup2(to_fd, from_fd) + return + except OSError: + retries = retries - 1 + + +@contextmanager +def not_std_redirector() -> typing.Iterator[None]: + """Context which does nothing. + + Use this context instead of the std_redirector context to avoid + stdout and stderr redirection. + + :return: None. + """ + yield + + +@contextmanager +def std_redirector( + out_filename: str, err_filename: str +) -> typing.Iterator[None]: + """Stdout and stderr redirector to the given out and err file names. + + :param out_filename: Output file filename (where to redirect stdout) + :param err_filename: Error output file filename (where to redirect stderr) + :return: Generator + """ + stdout = sys.stdout + stderr = sys.stderr + try: + stdout_fd = stdout.fileno() + except ValueError: + sys.stdout = os.fdopen(1, "w") # , 0) + stdout_fd = sys.stdout.fileno() + try: + stderr_fd = stderr.fileno() + except ValueError: + sys.stderr = os.fdopen(2, "w") # , 0) + stderr_fd = sys.stderr.fileno() + + def _redirect_stdout(to_fd: int) -> None: + """Redirect stdout to the given file descriptor. + + :param to_fd: Destination file descriptor + :return: None + """ + # Flush the C-level buffer stdout + LIBC.fflush(C_STDOUT) + # Flush and close sys.stdout (also closes the file descriptor) + sys.stdout.flush() + sys.stdout.close() + # Make stdout_fd point to_fd + _dup2(to_fd, stdout_fd) + # Create a new sys.stdout that points to the redirected fd + sys.stdout = io.TextIOWrapper(os.fdopen(stdout_fd, "wb")) + + def _redirect_stderr(to_fd: int) -> None: + """Redirect stderr to the given file descriptor. + + :param to_fd: Destination file descriptor + :return: None + """ + # Flush the C-level buffer stderr + LIBC.fflush(C_STDERR) + # Flush and close sys.stderr (also closes the file descriptor) + sys.stderr.flush() + sys.stderr.close() + # Make stderr_fd point to_fd + _dup2(to_fd, stderr_fd) + # Create a new sys.stderr that points to the redirected fd + sys.stderr = io.TextIOWrapper(os.fdopen(stderr_fd, "wb")) + + # Save a copy of the original stdout and stderr + stdout_fd_backup = os.dup(stdout_fd) + stderr_fd_backup = os.dup(stderr_fd) + + with FDLocker(out_filename) as f_out: + _redirect_stdout(f_out.file_descriptor.fileno()) + with FDLocker(err_filename) as f_err: + _redirect_stderr(f_err.file_descriptor.fileno()) + + # Yield to caller + yield + + # Then redirect stdout and stderr back to the backup file descriptors + with FDLocker(out_filename) as f_out: + _redirect_stdout(stdout_fd_backup) + with FDLocker(err_filename) as f_err: + _redirect_stderr(stderr_fd_backup) + # Close file descriptors + os.close(stdout_fd_backup) + os.close(stderr_fd_backup) + + +@contextmanager +def ipython_std_redirector( + out_filename: str, err_filename: str +) -> typing.Iterator[None]: + """Redirects stdout and stderr to the given files within ipython envs. + + :param out_filename: Output file filename (where to redirect stdout) + :param err_filename: Error output file filename (where to redirect stderr) + :return: Generator + """ + if sys.__stdout__ is None: + raise StandardOutputError() + stdout = sys.__stdout__ + if sys.__stderr__ is None: + raise StandardErrorError() + stderr = sys.__stderr__ + try: + stdout_fd = stdout.fileno() + except ValueError: + sys.stdout = os.fdopen(1, "w") # , 0) + stdout_fd = sys.stdout.fileno() + try: + stderr_fd = stderr.fileno() + except ValueError: + sys.stderr = os.fdopen(2, "w") # , 0) + stderr_fd = sys.stderr.fileno() + + def _redirect_stdout(to_fd: int) -> None: + """Redirect stdout to the given file descriptor. + + :param to_fd: Destination file descriptor + :return: None + """ + # Flush the C-level buffer stdout + LIBC.fflush(C_STDOUT) + # Flush and close sys.__stdout__ (also closes the file descriptor) + if sys.__stdout__ is None: + raise StandardOutputError() + sys.__stdout__.flush() + sys.__stdout__.close() + sys.stdout.flush() + sys.stdout.close() + # Make stdout_fd point to_fd + _dup2(to_fd, stdout_fd) + # Create a new sys.__stdout__ that points to the redirected fd + new_out = io.TextIOWrapper(os.fdopen(stdout_fd, "wb")) + sys.__stdout__ = new_out # type: ignore + sys.stdout = sys.__stdout__ + + def _redirect_stderr(to_fd: int) -> None: + """Redirect stderr to the given file descriptor. + + :param to_fd: Destination file descriptor + :return: None + """ + # Flush the C-level buffer stderr + LIBC.fflush(C_STDERR) + # Flush and close sys.__stderr__ (also closes the file descriptor) + if sys.__stderr__ is None: + raise StandardErrorError() + sys.__stderr__.flush() + sys.__stderr__.close() + sys.stderr.flush() + sys.stderr.close() + # Make stderr_fd point to_fd + _dup2(to_fd, stderr_fd) + # Create a new sys.__stderr__ that points to the redirected fd + new_err = io.TextIOWrapper(os.fdopen(stderr_fd, "wb")) + sys.__stderr__ = new_err # type: ignore + sys.stderr = sys.__stderr__ + + # Save a copy of the original stdout and stderr + stdout_fd_backup = os.dup(stdout_fd) + stderr_fd_backup = os.dup(stderr_fd) + + with FDLocker(out_filename) as f_out: + _redirect_stdout(f_out.file_descriptor.fileno()) + with FDLocker(err_filename) as f_err: + _redirect_stderr(f_err.file_descriptor.fileno()) + + # Yield to caller + yield + + # Then redirect stdout and stderr back to the backup file descriptors + with FDLocker(out_filename) as f_out: + _redirect_stdout(stdout_fd_backup) + with FDLocker(err_filename) as f_err: + _redirect_stderr(stderr_fd_backup) + # Close file descriptors + os.close(stdout_fd_backup) + os.close(stderr_fd_backup) diff --git a/examples/dds/pycompss/util/storages/__init__.py b/examples/dds/pycompss/util/storages/__init__.py new file mode 100644 index 00000000..2709bd45 --- /dev/null +++ b/examples/dds/pycompss/util/storages/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the storage utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/storages/persistent.py b/examples/dds/pycompss/util/storages/persistent.py new file mode 100644 index 00000000..0ee2f437 --- /dev/null +++ b/examples/dds/pycompss/util/storages/persistent.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Utils - Storage - Persistent. + +This file contains the methods required to manage PSCOs. +Isolates the API signature calls. +""" + +import logging + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.helpers import EventMaster +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing + + +# Definition helpers +def __dummy_function() -> None: + """Do nothing function. + + To be used as definition of INIT, FINISH and GET_BY_ID globals. + + :return: None + :raises PyCOMPSsException: Always raises this function if invoked. + """ + raise PyCOMPSsException( + "Invoking unexpected dummy function from persistent storage API." + ) + + +# Globals +# Contain the actual storage api functions set on initialization +INIT = __dummy_function # type: typing.Callable +FINISH = __dummy_function # type: typing.Callable +GET_BY_ID = __dummy_function # type: typing.Callable +TaskContext = None # type: typing.Any +DUMMY_STORAGE = False # type: bool + + +class DummyTaskContext(object): + """Dummy task context to be used with storage frameworks.""" + + def __init__( + self, + logger: logging.Logger, + values: typing.Any, + config_file_path: typing.Optional[str] = None, + ) -> None: + """Create a new instance of DummyTaskContext. + + :param logger: Logger facility. + :param values: Values. + :param config_file_path: Configuration file path. + :returns: None + """ + self.logger = logger + err_msg = "Unexpected call to dummy storage task context." + self.logger.error(err_msg) + self.values = values + self.config_file_path = config_file_path + raise PyCOMPSsException(err_msg) + + def __enter__(self) -> None: + """Execute before starting the task. + + :returns: None. + :raises: PyCOMPSsException: If dummy task context is used. + """ + # Ready to start the task + err_msg = "Unexpected call to dummy storage task context __enter__" + self.logger.error(err_msg) + raise PyCOMPSsException(err_msg) + + def __exit__( + self, type: typing.Any, value: typing.Any, traceback: typing.Any + ) -> None: + """Execute when the task has finished. + + Signature from context. + + :param type: Type. + :param value: Value. + :param traceback: Traceback. + :returns: None. + :raises: PyCOMPSsException: If dummy task context is used. + """ + # Task finished + err_msg = "Unexpected call to dummy storage task context __exit__" + self.logger.error(err_msg) + raise PyCOMPSsException(err_msg) + + +def load_storage_library() -> None: + """Import the proper storage libraries. + + :return: None. + """ + global INIT + global FINISH + global GET_BY_ID + global TaskContext + global DUMMY_STORAGE + error_msg = "UNDEFINED" + + def dummy_init(config_file_path: typing.Optional[str] = None) -> None: + """Initialize the storage library. + + :returns: None. + :raises: PyCOMPSsException: If dummy task context is used. + """ + raise PyCOMPSsException( + f"Unexpected call to init from storage. Reason: {error_msg}" + ) + + def dummy_finish() -> None: + """Finish the storage library. + + :returns: None. + :raises: PyCOMPSsException: If dummy task context is used. + """ + raise PyCOMPSsException( + f"Unexpected call to finish from storage. Reason: {error_msg}" + ) + + def dummy_get_by_id(identifier: str) -> None: + """Get object by id from the storage library. + + :param identifier: Object identifier. + :returns: None. + :raises: PyCOMPSsException: If dummy task context is used. + """ + raise PyCOMPSsException( + f"Unexpected call to getByID. Reason: {error_msg}" + ) + + try: + # Try to import the external storage API module methods + from storage.api import init as real_init # noqa + from storage.api import finish as real_finish # noqa + from storage.api import getByID as real_get_by_id # noqa + from storage.api import TaskContext as RealTaskContext # noqa + + DUMMY_STORAGE = False + print("INFO: Storage API successfully imported.") + except ImportError as import_error: + # print("INFO: No storage API defined.") + # Defined methods throwing exceptions. + error_msg = str(import_error) + DUMMY_STORAGE = True + + # Prepare the imports + if DUMMY_STORAGE: + INIT = dummy_init + FINISH = dummy_finish + GET_BY_ID = dummy_get_by_id + TaskContext = DummyTaskContext + else: + INIT = real_init + FINISH = real_finish + GET_BY_ID = real_get_by_id + TaskContext = RealTaskContext + + +def is_psco(obj: typing.Any) -> bool: + """Check if obj is a persistent object (external storage). + + :param obj: Object to check. + :return: True if is persistent object. False otherwise. + """ + # Check from storage object requires a dummy storage object class + # from storage.storage_object import storage_object + # return issubclass(obj.__class__, storage_object) and + # get_id(obj) not in [None, "None"] + return has_id(obj) and get_id(obj) not in [None, "None"] + + +def has_id(obj: typing.Any) -> bool: + """Check if the object has a getID method. + + :param obj: Object to check. + :return: True if is persistent object. False otherwise. + """ + if "getID" in dir(obj): + return True + return False + + +def get_id(psco: typing.Any) -> typing.Union[str, None]: + """Retrieve the persistent object identifier. + + :param psco: Persistent object. + :return: Persistent object identifier. + """ + with EventInsideWorker(TRACING_WORKER.getid_event): + return psco.getID() + + +def get_by_id(identifier: str) -> typing.Any: + """Retrieve the object from the given identifier. + + :param identifier: Persistent object identifier. + :return: object associated to the persistent object identifier. + """ + with EventInsideWorker(TRACING_WORKER.get_by_id_event): + return GET_BY_ID(identifier) + + +def master_init_storage(storage_conf: str, logger: logging.Logger) -> bool: + """Call to init storage from the master. + + This function emits the event in the master. + + :param storage_conf: Storage configuration file. + :param logger: Logger where to log the messages. + :return: True if initialized. False on the contrary. + """ + with EventMaster(TRACING_MASTER.init_storage_event): + return __init_storage(storage_conf, logger) + + +def use_storage(storage_conf: str) -> bool: + """Evaluate if the storage_conf is defined. + + The storage will be used if storage_conf is not None nor "null". + + :param storage_conf: Storage configuration file. + :return: True if defined. False on the contrary. + """ + return storage_conf != "" and not storage_conf == "null" + + +def init_storage(storage_conf: str, logger: logging.Logger) -> bool: + """Call to init storage. + + This function emits the event in the worker. + + :param storage_conf: Storage configuration file. + :param logger: Logger where to log the messages. + :return: True if initialized. False on the contrary. + """ + with EventWorker(TRACING_WORKER.init_storage_event): + return __init_storage(storage_conf, logger) + + +def __init_storage(storage_conf: str, logger: logging.Logger) -> bool: + """Call to init storage. + + Initializes the persistent storage with the given storage_conf file. + + :param storage_conf: Storage configuration file. + :param logger: Logger where to log the messages. + :return: True if initialized. False on the contrary. + """ + if use_storage(storage_conf): + if __debug__: + logger.debug("Starting storage") + logger.debug("Storage configuration file: %s", storage_conf) + load_storage_library() + # Call to storage init + INIT(config_file_path=storage_conf) # noqa + return True + return False + + +def master_stop_storage(logger: logging.Logger) -> None: + """Stop the persistent storage. + + This function emits the event in the master. + + :param logger: Logger where to log the messages. + :return: None + """ + with EventMaster(TRACING_MASTER.stop_storage_event): + __stop_storage(logger) + + +def stop_storage(logger: logging.Logger) -> None: + """Stop the persistent storage. + + This function emits the event in the worker. + + :param logger: Logger where to log the messages. + :return: None + """ + with EventWorker(TRACING_WORKER.stop_storage_event): + __stop_storage(logger) + + +def __stop_storage(logger: logging.Logger) -> None: + """Stop the persistent storage. + + :param logger: Logger where to log the messages. + :return: None. + """ + if __debug__: + logger.debug("Stopping storage") + FINISH() diff --git a/examples/dds/pycompss/util/supercomputer/__init__.py b/examples/dds/pycompss/util/supercomputer/__init__.py new file mode 100644 index 00000000..cf7b89c9 --- /dev/null +++ b/examples/dds/pycompss/util/supercomputer/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the supercomputer utils helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/util/supercomputer/scs.py b/examples/dds/pycompss/util/supercomputer/scs.py new file mode 100644 index 00000000..949f2aaa --- /dev/null +++ b/examples/dds/pycompss/util/supercomputer/scs.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Utils - Supercomputer - Helpers. + +This file contains the methods used within Supercomputers. +They are used when running a notebook in interactive mode within a SC. +""" + +import os + +from pycompss.util.typing_helper import typing + + +def get_master_node() -> str: + """Get the master node. + + TIP: The environment variable COMPSS_MASTER_NODE is defined in the + launch_compss script. + + :return: Master node name. + """ + return os.environ["COMPSS_MASTER_NODE"] + + +def get_master_port() -> str: + """Get the master port. + + TIP: The environment variable COMPSS_MASTER_PORT is defined in the + launch_compss script. + + :return: Master port. + """ + return os.environ["COMPSS_MASTER_PORT"] + + +def get_worker_nodes() -> str: + """Get the worker nodes. + + TIP: The environment variable COMPSS_WORKER_NODES is defined in the + launch_compss script. + + :return: List of worker nodes. + """ + return os.environ["COMPSS_WORKER_NODES"] + + +def get_xmls() -> typing.Tuple[str, str]: + """Get the project and resources. + + They are taken from the environment variable exported from the + submit_jupyter_job.sh. + + :return: the project and resources paths. + """ + project = os.environ["COMPSS_PROJECT_XML"] + resources = os.environ["COMPSS_RESOURCES_XML"] + return project, resources + + +def get_uuid() -> str: + """Get UUID. + + TIP: The environment variable COMPSS_UUID is defined in the + launch_compss script. + + :return: UUID as string. + """ + return os.environ["COMPSS_UUID"] + + +def get_log_dir() -> str: + """Get log dir. + + TIP: The environment variable COMPSS_LOG_DIR is defined in the + launch_compss script. + + :return: Log directory. + """ + return os.environ["COMPSS_LOG_DIR"] + + +def get_master_working_dir() -> str: + """Get master working directory. + + TIP: The environment variable COMPSS_MASTER_WORKING_DIR is defined in the + launch_compss script. + + :return: Master working directory. + """ + return os.environ["COMPSS_MASTER_WORKING_DIR"] + + +def get_log_level() -> str: + """Get log level. + + TIP: The environment variable COMPSS_LOG_LEVEL is defined in the + launch_compss script. + + :return: Log level. + """ + return os.environ["COMPSS_LOG_LEVEL"] + + +def get_tracing() -> bool: + """Get tracing boolean. + + TIP: The environment variable COMPSS_TRACING is defined in the + launch_compss script. + + :return: Tracing boolean. + """ + return "true" == os.environ["COMPSS_TRACING"] + + +def get_storage_conf() -> str: + """Get storage configuration file. + + TIP: The environment variable COMPSS_STORAGE_CONF is defined in the + launch_compss script. + + :return: Storage configuration file path. + """ + return os.environ["COMPSS_STORAGE_CONF"] diff --git a/examples/dds/pycompss/util/tracing/__init__.py b/examples/dds/pycompss/util/tracing/__init__.py new file mode 100644 index 00000000..f32dc870 --- /dev/null +++ b/examples/dds/pycompss/util/tracing/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This package contains the tracing utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/tracing/helpers.py b/examples/dds/pycompss/util/tracing/helpers.py new file mode 100644 index 00000000..70f9d224 --- /dev/null +++ b/examples/dds/pycompss/util/tracing/helpers.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Tracing - Helpers. + +This file contains a set of context managers and decorators to ease the +tracing events emission. +""" + +import time +from contextlib import contextmanager + +from pycompss.util.context import CONTEXT +from pycompss.util.tracing.types_events_master import TRACING_MASTER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER_CACHE +from pycompss.util.typing_helper import typing + + +class Tracing: + """Keep the tracing status.""" + + __slots__ = ["pyextrae", "tracing"] + + def __init__( + self, + py_extrae: typing.Any = None, + tracing: bool = False, + ): + """Instantiate a new Tracing class.""" + self.pyextrae = py_extrae + self.tracing = tracing + + def get_pyextrae(self) -> typing.Any: + """Get PYEXTRAE value. + + :return: PyExtrae value. + """ + return self.pyextrae + + def set_pyextrae(self, pyextrae: typing.Any) -> None: + """Set PYEXTRAE value. + + :param: pyextrae: New pyextrae value. + :return: None + """ + self.pyextrae = pyextrae + + def is_tracing(self) -> bool: + """Get Tracing value. + + :return: If tracing is enabled. + """ + return self.tracing + + def enable_tracing(self) -> None: + """Enable tracing value. + + :return: None + """ + self.tracing = True + + def disable_tracing(self) -> None: + """Disable tracing value. + + :return: None + """ + self.tracing = False + + +# Keep tracing library and status +TRACING = Tracing() + + +@contextmanager +def dummy_context() -> typing.Iterator[None]: + """Context which deactivates the tracing flag and nothing else. + + :return: None. + """ + TRACING.disable_tracing() + yield + + +@contextmanager +def trace_multiprocessing_worker() -> typing.Iterator[None]: + """Set up the tracing for the multiprocessing worker. + + :return: None. + """ + import pyextrae.multiprocessing as pyextrae # pylint: disable=C0415, E0401 + + # disable=import-outside-toplevel, import-error + + TRACING.set_pyextrae(pyextrae) + TRACING.enable_tracing() + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 1) + pyextrae.eventandcounters( + TRACING_WORKER.inside_worker_type, TRACING_WORKER.worker_running_event + ) + yield # here the worker runs + pyextrae.eventandcounters(TRACING_WORKER.inside_worker_type, 0) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 0) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, int(time.time())) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 0) + + +@contextmanager +def trace_mpi_worker() -> typing.Iterator[None]: + """Set up the tracing for the mpi worker. + + :return: None. + """ + import pyextrae.mpi as pyextrae # pylint: disable=C0415, E0401 + + # disable=import-outside-toplevel, import-error + + TRACING.set_pyextrae(pyextrae) + TRACING.enable_tracing() + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 1) + pyextrae.eventandcounters( + TRACING_WORKER.inside_worker_type, TRACING_WORKER.worker_running_event + ) + yield # here the worker runs + pyextrae.eventandcounters(TRACING_WORKER.inside_worker_type, 0) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 0) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, int(time.time())) + pyextrae.eventandcounters(TRACING_WORKER.sync_type, 0) + + +@contextmanager +def trace_mpi_executor() -> typing.Iterator[None]: + """Set up the tracing for each mpi executor. + + :return: None. + """ + import pyextrae.mpi as pyextrae # pylint: disable=C0415, E0401 + + # disable=import-outside-toplevel, import-error + + TRACING.set_pyextrae(pyextrae) + TRACING.enable_tracing() + yield # here the mpi executor runs + + +class EventMaster: + """Decorator that emits an event at master wrapping the desired code. + + Does nothing if tracing is disabled. + + :param event_id: Event identifier to emit. + :return: None. + """ + + __slots__ = ["emitted"] + + def __init__(self, event_id: int) -> None: + """Emit the given event identifier in the master group. + + :param event_id: Event identifier. + :returns: None. + """ + self.emitted = False + if TRACING.is_tracing() and CONTEXT.in_master(): + TRACING.get_pyextrae().eventandcounters( + TRACING_MASTER.binding_master_type, event_id + ) + self.emitted = True + + def __enter__(self) -> None: + """Do nothing. + + :returns: None. + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Emit the 0 event in the master group when the context is finished. + + * Signature from context structure. + + :param type: Type. + :param value: Value. + :param traceback: Traceback. + :returns: None. + """ + if TRACING.is_tracing() and self.emitted: + TRACING.get_pyextrae().eventandcounters( + TRACING_MASTER.binding_master_type, 0 + ) + + +class EventWorker: + """Decorator that emits an event at worker wrapping the desired code. + + Does nothing if tracing is disabled. + + :param event_id: Event identifier to emit. + :return: None. + """ + + __slots__ = ["emitted"] + + def __init__(self, event_id: int) -> None: + """Emit the given event identifier in the worker group. + + :param event_id: Event identifier. + :returns: None. + """ + self.emitted = False + if TRACING.is_tracing() and CONTEXT.in_worker(): + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER.inside_worker_type, event_id + ) # noqa + self.emitted = True + + def __enter__(self) -> None: + """Do nothing. + + :returns: None. + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Emit the 0 event in the worker group when the context is finished. + + * Signature from context structure. + + :param type: Type. + :param value: Value. + :param traceback: Traceback. + :returns: None. + """ + if TRACING.is_tracing() and self.emitted: + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER.inside_worker_type, 0 + ) # noqa + + +class EventInsideWorker: + """Decorator that emits an event at worker (inside task). + + Wraps the desired function code. + Does nothing if tracing is disabled. + + :param event_id: Event identifier to emit. + :return: None. + """ + + __slots__ = ["emitted"] + + def __init__(self, event_id: int) -> None: + """Emit the given event identifier in the inside worker group. + + :param event_id: Event identifier. + :returns: None. + """ + self.emitted = False + if TRACING.is_tracing() and CONTEXT.in_worker(): + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER.inside_tasks_type, event_id + ) # noqa + self.emitted = True + + def __enter__(self) -> None: + """Do nothing. + + :returns: None. + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Emit the 0 event in the inside worker group when context finishes. + + * Signature from context structure. + + :param type: Type. + :param value: Value. + :param traceback: Traceback. + :returns: None. + """ + if TRACING.is_tracing() and self.emitted: + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER.inside_tasks_type, 0 + ) + + +class EventWorkerCache: + """Decorator that emits an event at worker cache wrapping the desired code. + + Does nothing if tracing is disabled. + + :param event_id: Event identifier to emit. + :return: None. + """ + + __slots__ = ["emitted"] + + def __init__(self, event_id: int) -> None: + """Emit the given event identifier in the inside worker cache group. + + :param event_id: Event identifier. + :returns: None. + """ + self.emitted = False + if TRACING.is_tracing() and CONTEXT.in_worker(): + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER_CACHE.worker_cache_type, event_id + ) + self.emitted = True + + def __enter__(self) -> None: + """Do nothing. + + :returns: None. + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Emit the 0 event in the worker cache group when context finishes. + + * Signature from context structure. + + :param type: Type. + :param value: Value. + :param traceback: Traceback. + :returns: None. + """ + if TRACING.is_tracing() and self.emitted: + TRACING.get_pyextrae().eventandcounters( + TRACING_WORKER_CACHE.worker_cache_type, 0 + ) + + +def emit_manual_event( + event_id: int, + master: bool = False, + inside: bool = False, + cpu_affinity: bool = False, + gpu_affinity: bool = False, + cpu_number: bool = False, +) -> None: + """Emit a single event with the desired code. + + Does nothing if tracing is disabled. + + :param event_id: Event identifier to emit. + :param master: If the event is emitted as master. + :param inside: If the event is produced inside the worker. + :param cpu_affinity: If the event is produced inside the worker for + cpu affinity. + :param gpu_affinity: If the event is produced inside the worker for + gpu affinity. + :param cpu_number: If the event is produced inside the worker for + cpu number. + :return: None. + """ + if TRACING.is_tracing(): + event_group, event_id = __get_proper_type_event( + event_id, master, inside, cpu_affinity, gpu_affinity, cpu_number + ) + TRACING.get_pyextrae().eventandcounters(event_group, event_id) # noqa + + +def emit_manual_event_explicit(event_group: int, event_id: int) -> None: + """Emit a single event for a group. + + Does nothing if tracing is disabled. + + :param event_group: Event group to emit. + :param event_id: Event identifier to emit. + :return: None. + """ + if TRACING.is_tracing(): + TRACING.get_pyextrae().eventandcounters(event_group, event_id) # noqa + + +def __get_proper_type_event( + event_id: int, + master: bool, + inside: bool, + cpu_affinity: bool, + gpu_affinity: bool, + cpu_number: bool, +) -> typing.Tuple[int, int]: + """Parse the flags to retrieve the appropriate event_group. + + It also parses the event_id in case of affinity since it is received + as string. + + :param event_id: Event identifier to emit. + :param master: If the event is emitted as master. + :param inside: If the event is produced inside the worker. + :param cpu_affinity: If the event is produced inside the worker for + cpu affinity. + :param gpu_affinity: If the event is produced inside the worker for + gpu affinity. + :param cpu_number: If the event is produced inside the worker for + cpu number. + :return: Retrieves the appropriate event_group and event_id. + """ + if master: + event_group = TRACING_MASTER.binding_master_type + else: + if inside: + if cpu_affinity: + event_group = TRACING_WORKER.inside_tasks_cpu_affinity_type + event_id = __parse_affinity_event_id(event_id) + elif gpu_affinity: + event_group = TRACING_WORKER.inside_tasks_gpu_affinity_type + event_id = __parse_affinity_event_id(event_id) + elif cpu_number: + event_group = TRACING_WORKER.inside_tasks_cpu_count_type + event_id = int(event_id) + else: + event_group = TRACING_WORKER.inside_tasks_type + else: + event_group = TRACING_WORKER.inside_worker_type + return event_group, event_id + + +def __parse_affinity_event_id(event_id: typing.Any) -> int: + """Parse the affinity event identifier. + + :param event_id: Event identifier. + :return: The parsed event identifier as integer. + """ + if isinstance(event_id, str): + try: + event_id = int(event_id) + except ValueError: + # The event_id is a string with multiple cores + # Get only the first core + event_id = int(event_id.split(",")[0].split("-")[0]) + event_id += 1 # since it starts with 0 + return event_id + + +def enable_trace_master() -> None: + """Enable tracing for the master process. + + :return: None. + """ + import pyextrae.sequential as pyextrae # pylint: disable=C0415, E0401 + + # disable=import-outside-toplevel, import-error + + TRACING.set_pyextrae(pyextrae) + TRACING.enable_tracing() diff --git a/examples/dds/pycompss/util/tracing/types_events_master.py b/examples/dds/pycompss/util/tracing/types_events_master.py new file mode 100644 index 00000000..4502d88a --- /dev/null +++ b/examples/dds/pycompss/util/tracing/types_events_master.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Tracing - Types and events in master. + +This file contains a set of constants that identify the types and events +emitted from when the binding acts as master. + +IMPORTANT!!! +Must be equal to Tracer.java and TraceEvent.java definitions +""" + + +class TypesEventsMaster: + """Constant events that can be emitted for the master group.""" + + __slots__ = ( + "binding_master_type", + "start_runtime_event", + "stop_runtime_event", + "application_running_event", + "init_storage_event", + "stop_storage_event", + "accessed_file_event", + "open_file_event", + "delete_file_event", + "get_file_event", + "get_directory_event", + "delete_object_event", + "barrier_event", + "barrier_group_event", + "open_task_group_event", + "close_task_group_event", + "cancel_task_group_event", + "get_log_path_event", + "get_tmp_path_event", + "get_number_resources_event", + "request_resources_event", + "free_resources_event", + "register_core_element_event", + "wait_on_event", + "process_task_event", + "wall_clock_limit_event", + "snapshot_event", + "task_instantiation", + "inspect_function_arguments", + "inspect_constraints", + "get_function_information", + "get_function_signature", + "check_interactive", + "extract_core_element", + "prepare_core_element", + "update_core_element", + "get_upper_decorators_kwargs", + "process_other_arguments", + "process_parameters", + "process_return", + "build_return_objects", + "serialize_object", + "build_compss_types_directions", + "process_task_binding", + "attributes_cleanup", + "binding_serialization_size_type", + "binding_deserialization_size_type", + "binding_serialization_object_num_type", + "binding_deserialization_object_num_type", + ) + + def __init__(self) -> None: + """Create a new instance of TypesEventsWorker.""" + # Binding master type + self.binding_master_type = 9000300 + + # Binding master events + self.start_runtime_event = 1 + self.stop_runtime_event = 2 + self.application_running_event = 3 # application running + # 4 is empty + self.init_storage_event = 5 # init_storage (same as worker) + self.stop_storage_event = 6 # stop_storage (same as worker) + + # API EVENTS + self.accessed_file_event = 7 + self.open_file_event = 8 + self.delete_file_event = 9 + self.get_file_event = 10 + self.get_directory_event = 11 + self.delete_object_event = 12 + self.barrier_event = 13 + self.barrier_group_event = 14 + self.open_task_group_event = 15 + self.close_task_group_event = 16 + self.get_log_path_event = 17 + self.get_tmp_path_event = 18 + self.get_number_resources_event = 19 + self.request_resources_event = 20 + self.free_resources_event = 21 + self.register_core_element_event = 22 + self.wait_on_event = 23 + self.process_task_event = 24 + self.wall_clock_limit_event = 25 + self.snapshot_event = 26 + self.cancel_task_group_event = 27 + + # CALL EVENTS + self.task_instantiation = 100 + self.inspect_function_arguments = 101 + self.inspect_constraints = 102 + self.get_function_information = 103 + self.get_function_signature = 104 + self.check_interactive = 105 + self.extract_core_element = 106 + self.prepare_core_element = 107 + self.update_core_element = 108 + self.get_upper_decorators_kwargs = 109 + self.process_other_arguments = 110 + self.process_parameters = 111 + self.process_return = 112 + self.build_return_objects = 113 + self.serialize_object = 114 + self.build_compss_types_directions = 115 + self.process_task_binding = 116 + self.attributes_cleanup = 117 + + # Serialization/Deserialization types: + self.binding_serialization_size_type = 9000600 + self.binding_deserialization_size_type = 9000601 + self.binding_serialization_object_num_type = 9000700 + self.binding_deserialization_object_num_type = 9000701 + + +TRACING_MASTER = TypesEventsMaster() diff --git a/examples/dds/pycompss/util/tracing/types_events_worker.py b/examples/dds/pycompss/util/tracing/types_events_worker.py new file mode 100644 index 00000000..79dc5c5c --- /dev/null +++ b/examples/dds/pycompss/util/tracing/types_events_worker.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Util - Tracing - Types and events in worker. + +This file contains a set of constants that identify the types and events +emitted from when the binding acts as worker. + +IMPORTANT!!! +Must be equal to Tracer.java and TraceEvent.java definitions +""" + + +class TypesEventsWorker: + """Constant events that can be emitted for the worker group.""" + + __slots__ = ( + "sync_type", + "binding_tasks_func_type", + "process_identifier", + "executor_identifier", + "inside_tasks_type", + "process_worker_executor_event", + "process_worker_event", + "process_worker_cache_event", + "bind_cpus_event", + "bind_gpus_event", + "setup_environment_event", + "get_task_params_event", + "import_user_module_event", + "execute_user_code_event", + "deserialize_from_bytes_event", + "deserialize_from_file_event", + "serialize_to_file_event", + "serialize_to_file_mpienv_event", + "build_successful_message_event", + "build_compss_exception_message_event", + "build_exception_message_event", + "clean_environment_event", + "get_by_id_event", + "getid_event", + "make_persistent_event", + "delete_persistent_event", + "retrieve_object_from_cache_event", + "insert_object_into_cache_event", + "remove_object_from_cache_event", + "wait_on_event", + "worker_task_instantiation", + "cache_hit_event", + "cache_miss_event", + "check_access_gpu_event", + "cache_hit_gpu_event", + "cache_miss_gpu_event", + "retrieve_object_from_gpu_cache_event", + "insert_object_into_gpu_cache_event", + "cleanup_task_event", + "executor_load_ear_event", + "executor_finalize_ear_event", + "inside_tasks_cpu_affinity_type", + "inside_tasks_cpu_count_type", + "inside_tasks_gpu_affinity_type", + "inside_worker_type", + "worker_running_event", + "process_task_event", + "process_ping_event", + "process_quit_event", + "init_storage_event", + "stop_storage_event", + "init_storage_at_worker_event", + "finish_storage_at_worker_event", + "init_worker_postfork_event", + "finish_worker_postfork_event", + "load_ear_event", + "finalize_ear_event", + "preload_import_event", + "serialization_cache_size_type", + "deserialization_cache_size_type", + ) + + def __init__(self) -> None: + """Create a new instance of TypesEventsWorker.""" + # Binding worker type + self.sync_type = 8000666 + + # Tasks at master (id corresponds to task) type: + self.binding_tasks_func_type = 9000000 + + # Tasks at worker types: + self.process_identifier = 8001003 # define the process purpose + self.executor_identifier = 8001006 # define the executor identifier + self.inside_tasks_type = 9000100 + + # Process purpose events + self.process_worker_executor_event = 8 + self.process_worker_event = 9 + self.process_worker_cache_event = 10 + + # Binding worker events + self.bind_cpus_event = 1 + self.bind_gpus_event = 2 + self.setup_environment_event = 3 + self.get_task_params_event = 4 + self.import_user_module_event = 5 + self.execute_user_code_event = 6 + self.deserialize_from_bytes_event = 7 + self.deserialize_from_file_event = 8 + self.serialize_to_file_event = 9 + self.serialize_to_file_mpienv_event = 10 + self.build_successful_message_event = 11 + self.build_compss_exception_message_event = 12 + self.build_exception_message_event = 13 + self.clean_environment_event = 14 + self.get_by_id_event = 15 + self.getid_event = 16 + self.make_persistent_event = 17 # TODO: INCLUDE INTERFACE + self.delete_persistent_event = 18 # TODO: INCLUDE INTERFACE + self.retrieve_object_from_cache_event = 19 + self.insert_object_into_cache_event = 20 + self.remove_object_from_cache_event = 21 + self.wait_on_event = 22 + self.worker_task_instantiation = 23 # task worker __init__ + self.cache_hit_event = 24 + self.cache_miss_event = 25 + self.check_access_gpu_event = 26 + self.cache_hit_gpu_event = 27 + self.cache_miss_gpu_event = 28 + self.retrieve_object_from_gpu_cache_event = 29 + self.insert_object_into_gpu_cache_event = 30 + self.cleanup_task_event = 31 + self.executor_load_ear_event = 32 + self.executor_finalize_ear_event = 33 + + # Task affinity events: + self.inside_tasks_cpu_affinity_type = 9000150 + self.inside_tasks_cpu_count_type = 9000151 + self.inside_tasks_gpu_affinity_type = 9000160 + + # Worker events: + self.inside_worker_type = 9000200 + self.worker_running_event = 1 # task invocation + self.process_task_event = 2 # process_task + self.process_ping_event = 3 # process_ping + self.process_quit_event = 4 # process_quit + self.init_storage_event = 5 # init_storage + self.stop_storage_event = 6 # stop_storage + self.init_storage_at_worker_event = 7 # initStorageAtWorker + self.finish_storage_at_worker_event = 8 # finishStorageAtWorker + self.init_worker_postfork_event = 9 # initWorkerPostFork + self.finish_worker_postfork_event = 10 # finishWorkerPostFork + self.load_ear_event = 11 + self.finalize_ear_event = 12 + self.preload_import_event = 13 + # Other worker events: + self.serialization_cache_size_type = 9000602 + self.deserialization_cache_size_type = 9000603 + + +TRACING_WORKER = TypesEventsWorker() + + +class TypesEventsWorkerCache: + """Constant events that can be emitted for the worker cache group.""" + + __slots__ = ( + "worker_cache_type", + "cache_msg_receive_event", + "cache_msg_quit_event", + "cache_msg_end_profiling_event", + "cache_msg_get_event", + "cache_msg_put_event", + "cache_msg_remove_event", + "cache_msg_lock_event", + "cache_msg_unlock_event", + "cache_msg_is_locked_event", + "cache_msg_is_in_cache_event", + "cache_msg_put_gpu_event", + "cache_msg_get_gpu_event", + ) + + def __init__(self) -> None: + """Create a new instance of TypesEventsWorker.""" + # Worker events: + self.worker_cache_type = 9000201 + + # Cache process events + self.cache_msg_receive_event = 1 + self.cache_msg_quit_event = 2 + self.cache_msg_end_profiling_event = 3 + self.cache_msg_get_event = 4 + self.cache_msg_put_event = 5 + self.cache_msg_remove_event = 6 + self.cache_msg_lock_event = 7 + self.cache_msg_unlock_event = 8 + self.cache_msg_is_locked_event = 9 + self.cache_msg_is_in_cache_event = 10 + self.cache_msg_get_gpu_event = 11 + self.cache_msg_put_gpu_event = 12 + + +TRACING_WORKER_CACHE = TypesEventsWorkerCache() diff --git a/examples/dds/pycompss/util/typing_helper.py b/examples/dds/pycompss/util/typing_helper.py new file mode 100644 index 00000000..19c4392c --- /dev/null +++ b/examples/dds/pycompss/util/typing_helper.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Utils - typing_helper. + +This file contains the typing helpers. +""" + +import typing + + +class DummyMypycAttr: + """Dummy on mypy_attr class (decorator style).""" + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Create a new dummy mypy attribute. + + :returns: None. + """ + self.args = args + self.kwargs = kwargs + + def __call__(self, function: typing.Any) -> typing.Any: + """Execute the given function. + + :param function: Decorated function. + :returns: Decorated function execution result. + """ + + def wrapped_mypyc_attr( + *args: typing.Any, **kwargs: typing.Any + ) -> typing.Any: + return function(*args, **kwargs) + + return wrapped_mypyc_attr + + +IMPORT_OK = True +try: + from mypy_extensions import mypyc_attr as real_mypyc_attr + + # https://mypyc.readthedocs.io/en/latest/native_classes.html#inheritance +except ImportError: + # Dummy mypyc_attr just in case mypy_extensions is not installed + IMPORT_OK = False + +if IMPORT_OK: + mypyc_attr = real_mypyc_attr # pylint: disable=invalid-name +else: + mypyc_attr = DummyMypycAttr # pylint: disable=invalid-name + + +###################################### +# Boilerplate to mimic user fuctions # +###################################### + + +def dummy_function() -> None: + """Do nothing function. + + :returns: None + """ diff --git a/examples/dds/pycompss/util/warnings/__init__.py b/examples/dds/pycompss/util/warnings/__init__.py new file mode 100644 index 00000000..3197db6b --- /dev/null +++ b/examples/dds/pycompss/util/warnings/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the warning utils functions, constants and classes.""" diff --git a/examples/dds/pycompss/util/warnings/modules.py b/examples/dds/pycompss/util/warnings/modules.py new file mode 100644 index 00000000..912f34b6 --- /dev/null +++ b/examples/dds/pycompss/util/warnings/modules.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Utils - Warnings - Optional Modules. + +This file contains a list, alongside with some functions, of the +optional modules that are required in order to be able to use some +PyCOMPSs features. +""" + +from pycompss.util.objects.properties import is_module_available + +OPTIONAL_MODULES = { + "dill": "Dill is a pickle extension which is capable to serialize a " + "wider variety of objects." +} + + +def get_optional_module_warning( + module_name: str, module_description: str +) -> str: + """Get optional modules warning message. + + :param module_name: Module name. + :param module_description: Module description. + :return: String with the optional warning message. + """ + ret = [ + f"\n[ WARNING ]:\t{module_name} module is not installed.", + "\t\t%s" % module_description.replace("\n", "\n\t\t"), + f"\t\tPyCOMPSs can work without {module_name}, " + f"but it is recommended to have it installed.", + f"\t\tYou can install it via pip typing pip install {module_name}, " + f"or (probably) with your package manager.\n", + ] + return "\n".join(ret) + + +def show_optional_module_warnings() -> None: + """Display the optional modules warnings if not available. + + :return: None. + """ + for name, description in OPTIONAL_MODULES.items(): + if not is_module_available(name): + print(get_optional_module_warning(name, description)) diff --git a/examples/dds/pycompss/worker/__init__.py b/examples/dds/pycompss/worker/__init__.py new file mode 100644 index 00000000..ea4f3ee6 --- /dev/null +++ b/examples/dds/pycompss/worker/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the workers functions, constants and classes.""" diff --git a/examples/dds/pycompss/worker/commons/__init__.py b/examples/dds/pycompss/worker/commons/__init__.py new file mode 100644 index 00000000..ee5cec36 --- /dev/null +++ b/examples/dds/pycompss/worker/commons/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the common worker functions, constants and classes.""" diff --git a/examples/dds/pycompss/worker/commons/executor.py b/examples/dds/pycompss/worker/commons/executor.py new file mode 100644 index 00000000..9116201a --- /dev/null +++ b/examples/dds/pycompss/worker/commons/executor.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Commons - Executor. + +This file contains the common worker executor methods. +""" + +from pycompss.api.parameter import TYPE +from pycompss.util.exceptions import PyCOMPSsException + + +def build_return_params_message(types: list, values: list) -> str: + """Build the return message with the parameters output. + + :param types: List of the parameter's types. + :param values: List of the parameter's values. + :return: Message as string. + """ + err_msg = "return type-value length mismatch for return message." + if len(types) != len(values): + raise PyCOMPSsException(f"Inconsistent state: {err_msg}") + + pairs = list(zip(types, values)) + num_params = len(pairs) + params = [str(num_params)] + for pair in pairs: + if pair[0] == TYPE.COLLECTION: + value = __build_collection_representation(pair[1]) + else: + value = str(pair[1]) + params.append(str(pair[0])) + params.append(value) + message = " ".join(params) + return message + + +def __build_collection_representation(value: list) -> str: + """Create the representation of a collection from the list of lists format. + + CAUTION: Recursive function. + The runtime expects: + [ + [ + [(t1, v1), (t2, v2), ...], + [(t1, v1), (t2, v2), ...], + ... + ], + ... + ] + + :param value: Collection message before processing. + :return: The collection representation message. + """ + result = "[" + first = True + for i in value: + if isinstance(i[0], list): + # Has inner list + result = result + __build_collection_representation(i) + else: + coll_type = i[0] + coll_value = i[1].replace("'", "") + if first: + result = f"{result}({coll_type},{coll_value})" + first = False + else: + result = f"{result},({coll_type},{coll_value})" + return f"{result}]" diff --git a/examples/dds/pycompss/worker/commons/worker.py b/examples/dds/pycompss/worker/commons/worker.py new file mode 100644 index 00000000..f4c89d94 --- /dev/null +++ b/examples/dds/pycompss/worker/commons/worker.py @@ -0,0 +1,906 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +PyCOMPSs Worker - Commons - Worker. + +This file contains the common worker methods. +Currently, it is used by all worker types (container, external, gat and piper). +""" + +import base64 +import logging +import os +import pickle +import signal +import sys +import traceback +import importlib + +from pycompss.api import parameter +from pycompss.api.commons.data_type import DataType +from pycompss.api.exceptions import COMPSsException +from pycompss.runtime.commons import CONSTANTS +from pycompss.runtime.task.parameter import COMPSsFile +from pycompss.runtime.task.parameter import JAVA_MAX_INT +from pycompss.runtime.task.parameter import JAVA_MIN_INT +from pycompss.runtime.task.parameter import Parameter +from pycompss.util.exceptions import SerializerException, PyCOMPSsException +from pycompss.util.exceptions import TimeOutError +from pycompss.util.exceptions import task_cancel +from pycompss.util.exceptions import task_timed_out +from pycompss.util.process.manager import DictProxy # typing only +from pycompss.util.serialization.serializer import deserialize_from_file +from pycompss.util.serialization.serializer import serialize_to_file +from pycompss.util.storages.persistent import load_storage_library +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing + +# First load the storage library +load_storage_library() +# Then import the appropriate functions +from pycompss.util.storages.persistent import ( # noqa # pylint: disable=C0413 + # disable=wrong-import-position + TaskContext, + is_psco, + get_by_id, +) + +default_logger = logging.getLogger(__name__) + + +def build_task_parameter( + p_type: int, + p_stream: int, + p_prefix: str, + p_name: str, + p_value: str, + p_c_type: str, + args: typing.Optional[list] = None, + pos: int = 0, + logger: logging.Logger = default_logger, +) -> typing.Tuple[Parameter, int]: + """Build task parameter object from the given parameters. + + :param p_type: Parameter type. + :param p_stream: Parameter stream. + :param p_prefix: Parameter prefix. + :param p_name: Parameter name. + :param p_value: Parameter value. + :param p_c_type: Parameter Python Type. + :param args: Arguments (Default: None). + :param pos: Position (Default: 0). + :param logger: Logger where to push the logging messages. + :return: Parameter object and the number fo substrings. + """ + num_substrings = 0 + if p_type in [ + parameter.TYPE.FILE, + parameter.TYPE.DIRECTORY, + parameter.TYPE.COLLECTION, + parameter.TYPE.DICT_COLLECTION, + ]: + # Maybe the file is a object, we do not care about this here + # We will decide whether to deserialize or to forward the value + # when processing parameters in the task decorator + _param = Parameter( + name=p_name, + content_type=p_type, + stream=p_stream, + prefix=p_prefix, + file_name=COMPSsFile(p_value), + extra_content_type=str(p_c_type), + ) + return _param, 0 + + if p_type == parameter.TYPE.EXTERNAL_PSCO: + # Next position contains R/W but we do not need it. Currently skipped. + return ( + Parameter( + content=p_value, + content_type=p_type, + stream=p_stream, + prefix=p_prefix, + name=p_name, + extra_content_type=str(p_c_type), + ), + 1, + ) + + if p_type == parameter.TYPE.EXTERNAL_STREAM: + # Next position contains R/W but we do not need it. Currently skipped. + return ( + Parameter( + content_type=p_type, + stream=p_stream, + prefix=p_prefix, + name=p_name, + file_name=COMPSsFile(p_value), + extra_content_type=str(p_c_type), + ), + 1, + ) + + if p_type in (parameter.TYPE.STRING, parameter.TYPE.STRING_64): + aux = "" # type: typing.Union[str, bytes] + new_aux = "" # type: typing.Union[str, bytes] + if args is not None: + num_substrings = int(p_value) # noqa + aux_str = [] + for j in range(6, num_substrings + 6): + aux_str.append(args[pos + j]) + aux = " ".join(aux_str) + else: + aux = str(p_value) + if p_type == parameter.TYPE.STRING_64: + # Decode the received string + # Note that we prepend a sharp to all strings in order to avoid + # getting empty encodings in the case of empty strings, so we need + # to remove it when decoding + aux = base64.b64decode(aux.encode()) + new_aux = aux[1:] + else: + new_aux = aux + + if new_aux: + if p_type == parameter.TYPE.STRING_64: + ####### + # Check if the string is really an object + # Required in order to recover objects passed as parameters. + # - Option object_conversion + real_value = new_aux + # Can be a hidden object + if isinstance(new_aux, bytes): + new_aux = new_aux.decode(CONSTANTS.str_escape) + if new_aux.startswith("HiddenObj#"): + # It is really a hidden object + new_aux = new_aux[10:] # Remove HiddenObj# + try: + # try to recover the real object + # Convert bytes-string to actual bytes + p_bin = bytes( + new_aux[2:-1].encode("raw_unicode_escape") + ) + deserialized_aux = pickle.loads(p_bin) + p_type = parameter.TYPE.OBJECT + except ( + SerializerException, + ValueError, + EOFError, + AttributeError, + ) as error: + # Was not an object + raise PyCOMPSsException( + "Can not deserialize hidden object." + ) from error + else: + deserialized_aux = real_value # type: ignore + ####### + else: + deserialized_aux = real_value # type: ignore + else: + deserialized_aux = new_aux + else: + deserialized_aux = new_aux + + if isinstance(deserialized_aux, bytes): + deserialized_aux = deserialized_aux.decode("utf-8") + + if __debug__: + logger.debug("\t * Value: %s", str(aux)) + + return ( + Parameter( + content_type=p_type, + stream=p_stream, + prefix=p_prefix, + name=p_name, + content=deserialized_aux, + extra_content_type=str(p_c_type), + ), + num_substrings, + ) + + # Basic numeric types. These are passed as command line arguments + # and only a cast is needed + val = None # type: typing.Union[None, int, float, bool] + if p_type == parameter.TYPE.INT: + val = int(p_value) # noqa + elif p_type == parameter.TYPE.LONG: + val = float(p_value) + if val > JAVA_MAX_INT or val < JAVA_MIN_INT: + # A Python in parameter was converted to a Java long to prevent + # overflow. We are sure we will not overflow Python int, + # otherwise this would have been passed as a serialized object. + val = int(val) + elif p_type == parameter.TYPE.DOUBLE: + val = float(p_value) # noqa + p_type = parameter.TYPE.FLOAT + if __debug__: + logger.debug("Changing type from DOUBLE to FLOAT") + elif p_type == parameter.TYPE.BOOLEAN: + val = p_value == "true" + return ( + Parameter( + content=val, + content_type=p_type, + stream=p_stream, + prefix=p_prefix, + name=p_name, + extra_content_type=str(p_c_type), + ), + 0, + ) + + +def get_task_params( + num_params: int, logger: logging.Logger, args: list +) -> list: + """Get and prepare the input parameters from string to lists. + + :param num_params: Number of parameters. + :param logger: Logger. + :param args: Arguments (complete list of parameters with type, stream, + prefix and value). + :return: A list of TaskParameter objects. + """ + with EventInsideWorker(TRACING_WORKER.get_task_params_event): + pos = 0 + ret = [] + for i in range(0, num_params): # noqa + p_type = int(args[pos]) + p_stream = int(args[pos + 1]) + p_prefix = args[pos + 2] + p_name = args[pos + 3] + p_c_type = args[pos + 4] + p_value = args[pos + 5] + + if __debug__: + logger.debug("Parameter : %s", str(i)) + logger.debug("\t * Type : %s", str(p_type)) + logger.debug("\t * Std IO Stream : %s", str(p_stream)) + logger.debug("\t * Prefix : %s", str(p_prefix)) + logger.debug("\t * Name : %s", str(p_name)) + logger.debug("\t * Content Type: %r", p_c_type) + if p_type == parameter.TYPE.STRING: + logger.debug("\t * Number of substrings: %r", p_value) + else: + logger.debug("\t * Value: %r", p_value) + + task_param, offset = build_task_parameter( + p_type, + p_stream, + p_prefix, + p_name, + p_value, + p_c_type, + args, + pos, + logger, + ) + + if __debug__: + logger.debug( + "\t * Updated type : %s", str(task_param.content_type) + ) + + ret.append(task_param) + pos += offset + 6 + + return ret + + +def task_execution( + logger: logging.Logger, + process_name: str, + module: typing.Any, + method_name: str, + time_out: int, + types: list, + values: list, + compss_kwargs: dict, + persistent_storage: bool, + storage_conf: str, + is_interactive: bool, +) -> typing.Tuple[int, list, list, typing.Optional[str], bool, str]: + """Execute task. + + :param logger: Logger. + :param process_name: Process name. + :param module: Module which contains the function. + :param method_name: Function to invoke. + :param time_out: Time out. + :param types: List of the parameter's types. + :param values: List of the parameter's values. + :param compss_kwargs: PyCOMPSs keywords. + :param persistent_storage: If persistent storage is enabled. + :param storage_conf: Persistent storage configuration file. + :return: exit_code, new_types, new_values, target_direction, timed_out + and return_message. + """ + if __debug__: + logger.debug("Starting task execution") + logger.debug("module : %s ", str(module)) + logger.debug("method_name : %s ", str(method_name)) + logger.debug("time_out : %s ", str(time_out)) + logger.debug("Types : %s ", str(types)) + logger.debug("Values : %s ", str(values)) + logger.debug("P. storage : %s ", str(persistent_storage)) + logger.debug("Storage cfg : %s ", str(storage_conf)) + logger.debug("Is interactive: %s ", str(is_interactive)) + + if persistent_storage: + # TODO: Fix the values information. + # The values may not have the complete information here, since the + # runtime has not provided for example the direction. + # Alternatively, the after the execution we have the information + # since the @task decorator has been able to extract it. + # Then it is updated into the TaskContext.values before __exit__. + task_context = TaskContext( + logger, values, config_file_path=storage_conf + ) + + try: + # WARNING: the following call will not work if a user decorator + # overrides the return of the task decorator. + # new_types, new_values = getattr(module, method_name) + # (*values, compss_types=types, **compss_kwargs) + # If the @task is decorated with a user decorator, may include more + # return values, and consequently, the new_types and new_values will + # be within a tuple at position 0. + # Force users that use decorators on top of @task to return the task + # results first. This is tested with the timeit decorator in test 19. + signal.signal(signal.SIGALRM, task_timed_out) + signal.signal(signal.SIGUSR2, task_cancel) + signal.alarm(time_out) + if persistent_storage: + task_context.__enter__() # noqa + + # REAL CALL TO FUNCTION + task_output = getattr(module, method_name)( + *values, compss_types=types, logger=logger, **compss_kwargs + ) + except TimeOutError: + logger.exception( + "TIMEOUT ERROR IN %s - Time Out Exception", process_name + ) + logger.exception("Task has taken too much time to process") + new_values = _get_return_values_for_exception(types, values) + return task_returns(3, types, new_values, None, True, "", logger) + except COMPSsException as compss_exception: + logger.exception("COMPSS EXCEPTION IN %s", process_name) + return_message = "No message" + if compss_exception.message is not None: + return_message = compss_exception.message + new_values = _get_return_values_for_exception(types, values) + return task_returns( + 2, types, new_values, None, False, return_message, logger + ) + except AttributeError: + # Appears with functions that have not been well defined. + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + logger.exception( + "WORKER EXCEPTION IN %s - Attribute Error Exception", process_name + ) + logger.exception("".join(line for line in lines)) + logger.exception( + "Check that all parameters have been defined with " + "an absolute import path (even if in the same file)" + ) + # If exception is raised during the task execution, new_types and + # new_values are empty and target_direction is None + return task_returns(1, [], [], None, False, "", logger) + except BaseException: # pylint: disable=broad-except + # Catch any other user/decorators exception. + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + logger.exception("WORKER EXCEPTION IN %s", process_name) + logger.exception("".join(line for line in lines)) + # If exception is raised during the task execution, new_types and + # new_values are empty and target_direction is None + return task_returns(1, [], [], None, False, "", logger) + finally: + signal.alarm(0) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + + if isinstance(task_output[0], tuple): + # Weak but effective way to check it without doing inspect that + # another decorator has added another return thing. + new_types = task_output[0][0] + new_values = task_output[0][1] + target_direction = task_output[0][2] + updated_args = task_output[0][3] + else: + # The task_output is composed by the new_types and new_values returned + # by the task decorator. + new_types = task_output[0] + new_values = task_output[1] + target_direction = task_output[2] + updated_args = task_output[3] + + if persistent_storage: + task_context.values = updated_args + task_context.__exit__(*sys.exc_info()) # noqa + del task_context + + if is_interactive: + # Add to new_types and new_values the source file that was removed + new_types.insert(0, DataType.FILE) + new_values.insert(0, "null") + + # Clean objects + del updated_args + + return task_returns( + 0, new_types, new_values, target_direction, False, "", logger + ) + + +def _get_return_values_for_exception(types: list, values: list) -> list: + """Build the values list to retrieve on an exception. + + It takes the input types and returns a list of 'null' for each type + unless it is a PSCO, where it puts the psco identifier. + + :param types: List of input types. + :param values: List of input values. + :return: List of values to return. + """ + new_values = [] + for i, typ in enumerate(types): + if typ == parameter.TYPE.EXTERNAL_PSCO: + new_values.append(values[i]) + else: + new_values.append("null") + return new_values + + +def task_returns( + exit_code: int, + new_types: list, + new_values: list, + target_direction: typing.Optional[str], + timed_out: bool, + return_message: str, + logger: logging.Logger, +) -> typing.Tuple[int, list, list, typing.Optional[str], bool, str]: + """Log return. + + :param exit_code: Exit value (0 ok, 1 error). + :param new_types: New types to be returned. + :param new_values: New values to be returned. + :param target_direction: Target direction. + :param timed_out: If the task has reached time out. + :param return_message: Return exception message. + :param logger: Logger where to place the messages. + :return: exit code, new types, new values, target direction, time out, + and return message. + """ + if __debug__: + # The types may change + # (e.g. if the user does a makePersistent within the task) + logger.debug("Exit code: %s ", str(exit_code)) + logger.debug("Return Types: %s ", str(new_types)) + logger.debug("Return Values: %s ", str(new_values)) + logger.debug("Return target_direction: %s ", str(target_direction)) + logger.debug("Return timed_out: %s ", str(timed_out)) + logger.debug("Return exception_message: %s ", str(return_message)) + logger.debug("Finished task execution") + return ( + exit_code, + new_types, + new_values, + target_direction, + timed_out, + return_message, + ) + + +def import_user_module(path: str, logger: logging.Logger) -> typing.Any: + """Import the user module. + + :param path: Path to the user module. + :param logger: Logger. + :return: The loaded module. + """ + with EventInsideWorker(TRACING_WORKER.import_user_module_event): + module = importlib.import_module(path) + if path.startswith(CONSTANTS.interactive_file_name): + # Force reload in interactive mode. The user may have + # overwritten a function or task. + importlib.reload(module) + if __debug__: + logger.debug("Module successfully loaded") + return module + + +def execute_task( + process_name: str, + storage_conf: str, + params: list, + tracing: bool, + logger: logging.Logger, + log_files: tuple, + python_mpi: bool = False, + collections_layouts: typing.Dict[str, typing.Tuple[int, int, int]] = {}, + in_cache_queue: typing.Any = None, + out_cache_queue: typing.Any = None, + cache_ids: typing.Optional[DictProxy] = None, + cache_profiler: bool = False, +) -> typing.Tuple[int, list, list, typing.Optional[bool], str]: + """Execute task main method. + + :param process_name: Process name. + :param storage_conf: Storage configuration file path. + :param params: List of parameters. + :param tracing: Tracing flag. + :param logger: Logger to use. + :param log_files: Tuple with (out filename, err filename). + None to avoid stdout and sdterr fd redirection. + :param python_mpi: If it is a MPI task. + :param collections_layouts: collections layouts for python MPI tasks. + :param in_cache_queue: Cache tracker input communication queue. + :param out_cache_queue: Cache tracker output communication queue. + :param cache_ids: Cache proxy dictionary (read-only). + :param cache_profiler: Cache profiler. + :return: updated_args, exit_code, new_types, new_values, timed_out + and except_msg. + """ + if __debug__: + logger.debug("BEGIN TASK execution in %s", process_name) + + persistent_storage = False + if storage_conf != "null": + persistent_storage = True + + # Retrieve the parameters from the params argument + path = params[0] + method_name = params[1] + ppn = params[3] + num_slaves = int(params[4]) + time_out = int(params[2]) + slaves = [] + for i in range(4, 4 + num_slaves): + slaves.append(params[i]) + arg_position = 5 + num_slaves + + args = params[arg_position:] + cus = args[0] # noqa + args = args[1:] + has_target = args[0] + # Next parameter: return_type = args[1] + return_length = int(args[2]) + num_params = int(args[3]) + + args = args[4:] + + # Check if received the interactive source file as parameter + # and pop it from args. + compss_interactive_source_file = "" + is_interactive = False + if num_params > 0 and CONSTANTS.compss_interactive_source_file in args[3]: + compss_interactive_source_file = args[5].split(":")[0] + args = args[6:] + num_params -= 1 + is_interactive = True + + # COMPSs keywords for tasks (ie: tracing, process name...) + # compss_key is included to be checked in the @task decorator, so that + # the task knows if it has been called from the worker or from the + # user code (reason: ignore @task decorator if called from another task + # or decide if submit to runtime if nesting is enabled). + compss_kwargs = { + "compss_key": True, + "compss_tracing": tracing, + "compss_process_name": process_name, + "compss_storage_conf": storage_conf, + "compss_return_length": return_length, + "compss_logger": logger, + "compss_log_files": log_files, + "compss_python_MPI": python_mpi, + "compss_collections_layouts": collections_layouts, + "compss_cache": ( + in_cache_queue, + out_cache_queue, + cache_ids, + cache_profiler, + ), + } + + if __debug__: + logger.debug("COMPSs parameters:") + logger.debug("\t- Storage conf: %s", str(storage_conf)) + if log_files: + logger.debug("\t- Log out file: %s", str(log_files[0])) + logger.debug("\t- Log err file: %s", str(log_files[1])) + else: + logger.debug("\t- Log out and err not redirected") + logger.debug("\t- Params: %s", str(params)) + logger.debug("\t- Path: %s", str(path)) + logger.debug("\t- Method name: %s", str(method_name)) + logger.debug("\t- Num slaves: %s", str(num_slaves)) + logger.debug("\t- Slaves: %s", str(slaves)) + logger.debug("\t- Cus: %s", str(cus)) + logger.debug("\t- Has target: %s", str(has_target)) + logger.debug("\t- Num Params: %s", str(num_params)) + logger.debug("\t- Return Length: %s", str(return_length)) + logger.debug("\t- Args: %r", args) + logger.debug("\t- COMPSs kwargs:") + logger.debug("\t- Is interactive: %s", str(is_interactive)) + logger.debug( + "\t- Interactive source: %s", str(compss_interactive_source_file) + ) + for kw_key, kw_value in compss_kwargs.items(): + logger.debug("\t\t- %s: %s", str(kw_key), str(kw_value)) + + # Get all parameter values + if __debug__: + logger.debug("Processing parameters:") + # logger.debug(args) + values = get_task_params(num_params, logger, args) + types = [x.content_type for x in values] + + if __debug__: + logger.debug("RUN TASK with arguments:") + logger.debug("\t- Path: %s", path) + logger.debug("\t- Method/function name: %s", method_name) + logger.debug("\t- Has target: %s", str(has_target)) + logger.debug("\t- # parameters: %s", str(num_params)) + # Next parameters are the values: + # logger.debug("\t- Values:") + # for v in values: + # logger.debug("\t\t %r", v) + # logger.debug("\t- COMPSs types:") + # for t in types: + # logger.debug("\t\t %s", str(t)) + + import_error = False + if __debug__: + logger.debug("LOAD TASK:") + # Add dynamically source path to sys if received interactive source file + if is_interactive: + interactive_source_path = os.path.dirname( + compss_interactive_source_file + ) + if __debug__: + logger.debug( + "\t- Using interactive source file: %s", + compss_interactive_source_file, + ) + sys.path.append(interactive_source_path) + if __debug__: + logger.debug("\t- Added to path: %s", interactive_source_path) + try: + # Try to import the module (for functions) + if __debug__: + logger.debug("\t- Trying to import the user module: %s", path) + module = import_user_module(path, logger) + except ImportError: + if __debug__: + msg = "\t- Could not import the module. Reason: Method in class." + logger.debug(msg) + import_error = True + + if __debug__: + logger.debug("EXECUTE TASK:") + if not import_error: + # Module method declared as task + result = task_execution( + logger, + process_name, + module, + method_name, + time_out, + types, + values, + compss_kwargs, + persistent_storage, + storage_conf, + is_interactive, + ) + exit_code = result[0] + new_types = result[1] + new_values = result[2] + # Next result: target_direction = result[3] + timed_out = result[4] + except_msg = result[5] + else: + # Method declared as task in class + # Not the path of a module, it ends with a class name + class_name = path.split(".")[-1] + + if "." in path: + module_name = ".".join(path.split(".")[0:-1]) + else: + module_name = path + try: + module = __import__(module_name, fromlist=[class_name]) + klass = getattr(module, class_name) + except Exception: # pylint: disable=broad-except + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception( + exc_type, exc_value, exc_traceback + ) + exception_message = ( + f"EXCEPTION IMPORTING MODULE IN {process_name}\n" + ) + exception_message += "".join(line for line in lines) + logger.exception(exception_message) + return 1, [], [], None, exception_message + + if __debug__: + logger.debug( + "Method in class %s of module %s", class_name, module_name + ) + logger.debug("Has target: %s", str(has_target)) + + file_name = "None" + if has_target == "true": + # Instance method + # The self object needs to be an object in order to call the + # function. So, it can not be done in the @task decorator. + # Since the args structure is parameters + self + returns we pop + # the corresponding considering the return_length notified by the + # runtime (-1 due to index starts from 0). + self_index = num_params - return_length - 1 + self_elem = values.pop(self_index) + self_type = types.pop(self_index) + if self_type == parameter.TYPE.EXTERNAL_PSCO: + if __debug__: + logger.debug( + "Last element (self) is a PSCO with id: %s", + str(self_elem.content), + ) + obj = get_by_id(self_elem.content) + else: + obj = None + if self_elem.content == "": + file_name = self_elem.file_name.original_path + if __debug__: + logger.debug("\t- Deserialize self from file.") + try: + obj = deserialize_from_file(file_name, logger) + except Exception: # pylint: disable=broad-except + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception( + exc_type, exc_value, exc_traceback + ) + exception_message = ( + f"EXCEPTION DESERIALIZING SELF IN {process_name}\n" + ) + exception_message += "".join(line for line in lines) + logger.exception(exception_message) + return 1, [], [], None, exception_message + if __debug__: + logger.debug( + "Deserialized self object is: %s", + self_elem.content, + ) + logger.debug( + "Processing callee, a hidden object of " + "%s in file %s", + file_name, + type(self_elem.content), + ) + values.insert(0, obj) + + if not self_type == parameter.TYPE.EXTERNAL_PSCO: + types.insert(0, parameter.TYPE.OBJECT) + else: + types.insert(0, parameter.TYPE.EXTERNAL_PSCO) + + result = task_execution( + logger, + process_name, + klass, + method_name, + time_out, + types, + values, + compss_kwargs, + persistent_storage, + storage_conf, + is_interactive, + ) + exit_code = result[0] + new_types = result[1] + new_values = result[2] + target_direction = result[3] + timed_out = result[4] + except_msg = result[5] + + # Depending on the target_direction option, it is necessary to + # serialize again self or not. Since this option is only visible + # within the task decorator, the task_execution returns the value + # of target_direction in order to know here if self has to be + # serialized. This solution avoids to use inspect. + if target_direction is not None and target_direction in ( + parameter.INOUT.key, + parameter.COMMUTATIVE.key, + ): + if is_psco(obj): + # There is no explicit update if self is a PSCO. + # Consequently, the changes on the PSCO must have been + # pushed into the storage automatically on each PSCO + # modification. + if __debug__: + logger.debug( + "The changes on the PSCO must have been" + + " automatically updated by the storage." + ) + else: + if __debug__: + logger.debug( + "Serializing self (%r) to file: %s", obj, file_name + ) + try: + serialize_to_file(obj, file_name, logger) + except Exception: # noqa # pylint: disable=broad-except + # Catch any serialization exception + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception( + exc_type, exc_value, exc_traceback + ) + logger.exception( + "EXCEPTION SERIALIZING SELF IN %s", process_name + ) + logger.exception("".join(line for line in lines)) + exit_code = 1 + if __debug__: + logger.debug("Serialized successfully") + else: + # Class method - class is not included in values (e.g. values=[7]) + types.append(None) # class must be first type + + result = task_execution( + logger, + process_name, + klass, + method_name, + time_out, + types, + values, + compss_kwargs, + persistent_storage, + storage_conf, + is_interactive, + ) + exit_code = result[0] + new_types = result[1] + new_values = result[2] + # Next return: target_direction = result[3] + timed_out = result[4] + except_msg = result[5] + + # Clean + if __debug__: + logger.debug("CLEAN TASK:") + if is_interactive: + # Remove the interactive path from sys.path (last element) + last_elem = sys.path.pop() + if __debug__: + logger.debug("\t- [Interactive] Removed from path: %s", last_elem) + + if __debug__: + if exit_code != 0: + logger.debug("EXECUTE TASK FAILED: Exit code: %s", str(exit_code)) + else: + logger.debug("END TASK execution. Status: Ok") + + return int(exit_code), new_types, new_values, timed_out, except_msg diff --git a/examples/dds/pycompss/worker/container/__init__.py b/examples/dds/pycompss/worker/container/__init__.py new file mode 100644 index 00000000..2a2f9a20 --- /dev/null +++ b/examples/dds/pycompss/worker/container/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the container worker helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/worker/container/container_worker.py b/examples/dds/pycompss/worker/container/container_worker.py new file mode 100644 index 00000000..6c9b356b --- /dev/null +++ b/examples/dds/pycompss/worker/container/container_worker.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +PyCOMPSs Worker - Container - Worker. + +This file contains the code of a fake worker to execute Python tasks +inside containers. +""" + +import logging +import sys + +from pycompss.util.context import CONTEXT +from pycompss.worker.container.pythonpath_fixer import fix_pythonpath +from pycompss.util.logger.helpers import init_logging_worker +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.logger.level import LOG_LEVEL +from pycompss.util.typing_helper import typing # noqa: F401 +from pycompss.worker.commons.executor import build_return_params_message +from pycompss.worker.commons.worker import execute_task + + +def main() -> int: + """Process python task inside a container main method. + + Received parameters from ContainerInvoker.java. + + :return: Exit value. + """ + # Fix PYTHONPATH setup + fix_pythonpath() + + # Parse arguments + func_file_path = str(sys.argv[1]) + func_name = str(sys.argv[2]) + num_slaves = 0 + timeout = 0 + cus = 1 + ppn = 1 + log_level = sys.argv[3] + tracing = sys.argv[4] == "true" + has_target = str(sys.argv[5]).lower() == "true" + return_type = str(sys.argv[6]) + return_length = int(sys.argv[7]) + num_params = int(sys.argv[8]) + func_params = sys.argv[9:] + + # Log initialisation + if log_level in ("true", "debug"): + # Debug + init_logging_worker( + LOG_REMITTENT.CONTAINER_WORKER, LOG_LEVEL.DEBUG, tracing + ) + elif log_level == "info": + # Info or no debug + init_logging_worker( + LOG_REMITTENT.CONTAINER_WORKER, LOG_LEVEL.INFO, tracing + ) + else: + # Default + init_logging_worker( + LOG_REMITTENT.CONTAINER_WORKER, LOG_LEVEL.OFF, tracing + ) + if __debug__: + logger = logging.getLogger( + "pycompss.worker.container.container_worker" + ) + logger.debug("Initialising Python worker inside the container...") + + task_params = [ + func_file_path, + func_name, + timeout, + ppn, + num_slaves, + cus, + has_target, + return_type, + return_length, + num_params, + ] # type: typing.List[typing.Any] + execute_task_params = task_params + func_params + + if __debug__: + logger.debug("- File: %s", str(func_file_path)) + logger.debug("- Function: %s", str(func_name)) + logger.debug("- HasTarget: %s", str(has_target)) + logger.debug("- ReturnType: %s", str(return_type)) + logger.debug("- Num Returns: %s", str(return_length)) + logger.debug("- Num Parameters: %s", str(num_params)) + logger.debug("- Parameters: %s", str(func_params)) + logger.debug("DONE Parsing Python function and arguments") + + # Process task + if __debug__: + logger.debug("Processing task...") + + process_name = "ContainerInvoker" + storage_conf = "null" + tracing = False + log_files = () + python_mpi = False + collections_layouts = ( + {} + ) # type: typing.Dict[str, typing.Tuple[int, int, int]] + CONTEXT.set_worker() + result = execute_task( + process_name, + storage_conf, + execute_task_params, + tracing, + logger, + log_files, # noqa + python_mpi, + collections_layouts, + ) + # The ignored result is time out + exit_value, new_types, new_values, _, except_msg = result + + if __debug__: + logger.debug("DONE Processing task") + + # Process results + if __debug__: + logger.debug("Processing results...") + logger.debug("Task exit value = %s", str(exit_value)) + + if exit_value == 0: + # Task has finished without exceptions + if __debug__: + logger.debug("Building return parameters...") + logger.debug("New Types: %s", str(new_types)) + logger.debug("New Values: %s", str(new_values)) + build_return_params_message(new_types, new_values) + if __debug__: + logger.debug("DONE Building return parameters") + elif exit_value == 2: + # Task has finished with a COMPSs Exception + if __debug__: + except_msg = except_msg.replace(" ", "_") + logger.debug("Registered COMPSs Exception: %s", str(except_msg)) + else: + # An exception has been raised in task + if __debug__: + except_msg = except_msg.replace(" ", "_") + logger.debug( + "Registered Exception in task execution %s", str(except_msg) + ) + + # Return + if exit_value != 0: + logger.debug( + "ERROR: " + "Task execution finished with non-zero exit value (%s != 0)", + str(exit_value), + ) + else: + logger.debug("Task execution finished SUCCESSFULLY!") + return exit_value + + +# +# Entry point +# +if __name__ == "__main__": + main() diff --git a/examples/dds/pycompss/worker/container/pythonpath_fixer.py b/examples/dds/pycompss/worker/container/pythonpath_fixer.py new file mode 100644 index 00000000..dd4e09bc --- /dev/null +++ b/examples/dds/pycompss/worker/container/pythonpath_fixer.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +PyCOMPSs Worker - Container - PYTHONPATH Fixer. + +Fixes the PYTHONPATH for Container environments. +""" + +import sys + + +def fix_pythonpath() -> None: + """Reset the PYTHONPATH for clean container environments. + + :return: None. + """ + sys_version = sys.version_info[0:2] + version = f"{sys_version[0]}.{sys_version[1]}" + # Default Python installation in Docker containers + default_container_pythonpath = [ + f"/usr/lib/python{version}", + f"/usr/lib/python{version}/plat-x86_64-linux-gnu", + f"/usr/lib/python{version}/lib-tk", + f"/usr/lib/python{version}/lib-old", + f"/usr/lib/python{version}/lib-dynload", + f"/usr/local/lib/python{version}/dist-packages", + f"/usr/lib/python{version}/dist-packages", + ] + + # Build new PYTHONPATH + new_pythonpath = [] + + # Add entries not inherited by user's system default + # (application pythonpath only) + for path in sys.path: + if path.startswith("/apps/COMPSs/") or not ( + path.startswith("/apps/") or path.startswith("/gpfs/apps/") + ): + new_pythonpath.append(path) + + # Add default entries (at the end) + new_pythonpath.extend(default_container_pythonpath) + + # Reset PYTHONPATH + sys.path = new_pythonpath diff --git a/examples/dds/pycompss/worker/external/__init__.py b/examples/dds/pycompss/worker/external/__init__.py new file mode 100644 index 00000000..59c79262 --- /dev/null +++ b/examples/dds/pycompss/worker/external/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the external worker helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/worker/external/mpi_executor.py b/examples/dds/pycompss/worker/external/mpi_executor.py new file mode 100644 index 00000000..40ead4b7 --- /dev/null +++ b/examples/dds/pycompss/worker/external/mpi_executor.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - External - MPI Executor. + +This file contains the code of an executor running Python MPI execution +command that is passed from the runtime worker. +""" + +import copy +import logging +import os +import signal +import sys + +from pycompss.util.context import CONTEXT +from mpi4py import MPI +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.logger.helpers import init_logging_worker +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.logger.level import LOG_LEVEL +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.worker.commons.executor import build_return_params_message +from pycompss.worker.commons.worker import execute_task +from pycompss.worker.piper.commons.constants import TAGS + + +# TODO: Comments about exit value and return following values was in another +# branch need to be reviewed if it works in trunk +# SUCCESS_SIG = 0 +# FAILURE_SIG = 1 +# UNEXPECTED_SIG = 2 + + +def shutdown_handler( + signal: int, frame: typing.Any +) -> None: # pylint: disable=redefined-outer-name + """Handle shutdown - MPI exception signal handler. + + CAUTION! Do not remove the parameters. + + :param signal: shutdown signal + :param frame: Frame + :return: None + """ + raise PyCOMPSsException("Received SIGTERM") + + +###################### +# Processes body +###################### + + +def executor(process_name: str, command: str) -> None: + """Executor main method. + + Iterates over the input pipe in order to receive tasks (with their + parameters) and process them. + + :param process_name: Process name (MPI Process-X, where X is the MPI rank). + :param command: Command to execute. + :return: None + """ + # Replace Python Worker's SIGTERM handler. + signal.signal(signal.SIGTERM, shutdown_handler) + + log_level = command.split()[7] + tracing = command.split()[8] == "true" + + # Load log level configuration file + if log_level in ("true", "debug"): + # Debug + init_logging_worker(LOG_REMITTENT.MPI_WORKER, LOG_LEVEL.DEBUG, tracing) + elif log_level == "info": + # Info + init_logging_worker(LOG_REMITTENT.MPI_WORKER, LOG_LEVEL.INFO, tracing) + else: + # Default (off) + init_logging_worker(LOG_REMITTENT.MPI_WORKER, LOG_LEVEL.OFF, tracing) + + logger = logging.getLogger("pycompss.worker.external.mpi_executor") + logger_handlers = copy.copy(logger.handlers) + logger_level = logger.getEffectiveLevel() + try: + lh0_formatter = logger_handlers[0].formatter # type: typing.Any + logger_formatter = logging.Formatter( + lh0_formatter._fmt + ) # type: typing.Any + except IndexError: + logger_formatter = None + + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Starting process", str(process_name) + ) + + sig, _ = process_task( + command, + process_name, + logger, + logger_handlers, + logger_level, + logger_formatter, + ) + # Signal expected management: + # if sig == FAILURE_SIG: + # raise Exception("Task execution failed!", msg) + # elif sig == UNEXPECTED_SIG: + # raise Exception("Unexpected message!", msg) + + sys.stdout.flush() + sys.stderr.flush() + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Exiting process ", str(process_name) + ) + if sig != 0: + sys.exit(sig) + + +def process_task( + current_line: str, + process_name: str, + logger: logging.Logger, + logger_handlers: typing.Any, + logger_level: int, + logger_formatter: typing.Any, +) -> typing.Tuple[int, str]: + """Process command received from the current_line. + + :param current_line: Current command (line) to process. + :param process_name: Process name for logger messages. + :param logger: Logger. + :param logger_handlers: Logger handlers. + :param logger_level: Logger level. + :param logger_formatter: Logger formatter. + :return: exit_value and message. + """ + with EventWorker(TRACING_WORKER.process_task_event): + # Process properties + stdout = sys.stdout + stderr = sys.stderr + job_id = None + current_working_dir = os.getcwd() + + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Received message: %s", + str(process_name), + str(current_line), + ) + + splitted_current_line = current_line.split() + if splitted_current_line[0] == TAGS.execute_task: + num_collection_params = int(splitted_current_line[-1]) + collections_layouts = ( + {} + ) # type: typing.Dict[str, typing.Tuple[int, int, int]] + if num_collection_params > 0: + raw_layouts = splitted_current_line[ + ((num_collection_params * -4) - 1) : -1 # noqa: E203 + ] + for i in range(num_collection_params): + param = raw_layouts[i * 4] + layout = ( + int(raw_layouts[(i * 4) + 1]), + int(raw_layouts[(i * 4) + 2]), + int(raw_layouts[(i * 4) + 3]), + ) + collections_layouts[param] = layout + + # Remove the last elements: cpu and gpu bindings and + # collection params + current_line_filtered = splitted_current_line[0:-3] + + # task jobId command + job_id = current_line_filtered[1] + working_dir = current_line_filtered[2] + job_out = current_line_filtered[3] + job_err = current_line_filtered[4] + # current_line_filtered[5] = = tracing + # current_line_filtered[6] = = task id + # current_line_filtered[7] = = debug + # current_line_filtered[8] = = storage conf. + # current_line_filtered[9] = = operation type + # (e.g. METHOD) + # current_line_filtered[10] = = module + # current_line_filtered[11]= = method + # current_line_filtered[12]= = time out + # current_line_filteres[13]= = ppn + # current_line_filtered[14]= = Number of slaves + # (worker nodes)==#nodes + # <> + # current_line_filtered[14 + #nodes] = = computing units + # current_line_filtered[15 + #nodes] = = has target + # current_line_filtered[16 + #nodes] = = has return + # (always "null") + # current_line_filtered[17 + #nodes] = = Number of + # parameters + # <> + # !---> type, stream, prefix , value + + # Setting working directory + os.chdir(working_dir) + + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Received task with id: %s", + str(process_name), + str(job_id), + ) + logger.debug( + "[PYTHON EXECUTOR] [%s] Setting working directory: %s", + str(process_name), + str(working_dir), + ) + logger.debug( + "[PYTHON EXECUTOR] [%s] - TASK CMD: %s", + str(process_name), + str(current_line_filtered), + ) + + # Swap logger from stream handler to file handler + # All task output will be redirected to job.out/err + for log_handler in logger_handlers: + logger.removeHandler(log_handler) + + out_file_handler = logging.FileHandler(job_out) + out_file_handler.setLevel(logger_level) + out_file_handler.setFormatter(logger_formatter) + err_file_handler = logging.FileHandler(job_err) + err_file_handler.setLevel("ERROR") + err_file_handler.setFormatter(logger_formatter) + logger.addHandler(out_file_handler) + logger.addHandler(err_file_handler) + + if __debug__: + logger.debug("Received task in process: %s", str(process_name)) + logger.debug(" - TASK CMD: %s", str(current_line_filtered)) + + try: + # Setup out/err wrappers + out = open(job_out, "a") # pylint: disable=consider-using-with + err = open(job_err, "a") # pylint: disable=consider-using-with + sys.stdout = out + sys.stderr = err + + # Setup process environment + compss_ppn = int(current_line_filtered[13]) + compss_nodes = int(current_line_filtered[14]) + compss_nodes_names = ",".join( + current_line_filtered[15 : 15 + compss_nodes] # noqa: E203 + ) + computing_units = int(current_line_filtered[15 + compss_nodes]) + os.environ["COMPSS_NUM_NODES"] = str(compss_nodes) + os.environ["COMPSS_NUM_PROCS"] = str(compss_ppn * compss_nodes) + os.environ["COMPSS_HOSTNAMES"] = compss_nodes_names + os.environ["COMPSS_NUM_THREADS"] = str(computing_units) + os.environ["OMP_NUM_THREADS"] = str(computing_units) + if __debug__: + logger.debug("Process environment:") + logger.debug( + "\t - Number of nodes: %s", (str(compss_nodes)) + ) + logger.debug("\t - Hostnames: %s", str(compss_nodes_names)) + logger.debug( + "\t - Number of threads: %s", (str(computing_units)) + ) + + # Execute task + storage_conf = "null" + tracing = False + python_mpi = True + result = execute_task( + process_name, + storage_conf, + current_line_filtered[10:], + tracing, + logger, + (job_out, job_err), + python_mpi, + collections_layouts, + None, + None, + ) + exit_value, new_types, new_values, _, except_msg = result + + # Restore out/err wrappers + sys.stdout = stdout + sys.stderr = stderr + sys.stdout.flush() + sys.stderr.flush() + out.close() + err.close() + + # To reduce if necessary: + # global_exit_value = MPI.COMM_WORLD.reduce(exit_value, + # op=MPI.SUM, + # root=0) + # message = "" + + # if MPI.COMM_WORLD.rank == 0 and global_exit_value == 0: + if exit_value == 0: + # Task has finished without exceptions + # endTask jobId exitValue message + params = build_return_params_message(new_types, new_values) + message = " ".join( + ( + TAGS.end_task, + str(job_id), + str(exit_value), + str(params) + "\n", + ) + ) + elif exit_value == 2: + # Task has finished with a COMPSs Exception + # compssExceptionTask jobId exitValue message + except_msg = except_msg.replace(" ", "_") + message = " ".join( + ( + TAGS.compss_exception, + str(job_id), + str(except_msg) + "\n", + ) + ) + if __debug__: + logger.debug( + "%s - COMPSS EXCEPTION TASK MESSAGE: %s", + str(process_name), + str(except_msg), + ) + else: + # elif MPI.COMM_WORLD.rank == 0 and global_exit_value != 0: + # An exception has been raised in task + message = " ".join( + (TAGS.end_task, str(job_id), str(exit_value) + "\n") + ) + + if __debug__: + logger.debug( + "%s - END TASK MESSAGE: %s", + str(process_name), + str(message), + ) + # The return message is: + # + # TaskResult ==> jobId exitValue D List + # + # Where List has D * 2 length: + # D = #parameters == #task_parameters + + # (has_target ? 1 : 0) + + # #returns + # And contains a pair of elements per parameter: + # - Parameter new type. + # - Parameter new value: + # - "null" if it is NOT a PSCO + # - PSCOId (String) if is a PSCO + # Example: + # 4 null 9 null 12 + # + # The order of the elements is: parameters + self + returns + # + # This is sent through the pipe with the END_TASK message. + # If the task had an object or file as parameter and the worker + # returns the id, the runtime can change the type (and + # locations) to a EXTERNAL_OBJ_T. + + except ( + Exception + ) as general_exception: # pylint: disable=broad-except + logger.exception( + "%s - Exception %s", + str(process_name), + str(general_exception), + ) + exit_value = 7 + message = " ".join( + (TAGS.end_task, str(job_id), str(exit_value) + "\n") + ) + + # Clean environment variables + if __debug__: + logger.debug("Cleaning environment.") + + del os.environ["COMPSS_HOSTNAMES"] + + # Restore loggers + if __debug__: + logger.debug("Restoring loggers.") + logger.removeHandler(out_file_handler) + logger.removeHandler(err_file_handler) + for handler in logger_handlers: + logger.addHandler(handler) + + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Finished task with id: %s", + str(process_name), + str(job_id), + ) + # return SUCCESS_SIG, + # f"{str(process_name)} -- Task Ended Successfully!" + + else: + if __debug__: + logger.debug( + "[PYTHON EXECUTOR] [%s] Unexpected message: %s", + str(process_name), + str(current_line), + ) + exit_value = 7 + message = " ".join( + (TAGS.end_task, str(job_id), str(exit_value) + "\n") + ) + + # Go back to initial current working directory + os.chdir(current_working_dir) + + return exit_value, message + + +def main() -> None: + """MPI executor main method. + + :returns: None. + """ + # Set the binding in worker mode + CONTEXT.set_worker() + + executor(f"MPI Process-{MPI.COMM_WORLD.rank}", sys.argv[1]) + + +if __name__ == "__main__": + main() diff --git a/examples/dds/pycompss/worker/gat/__init__.py b/examples/dds/pycompss/worker/gat/__init__.py new file mode 100644 index 00000000..5cb12f9e --- /dev/null +++ b/examples/dds/pycompss/worker/gat/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the gat worker functions, constants and classes.""" diff --git a/examples/dds/pycompss/worker/gat/worker.py b/examples/dds/pycompss/worker/gat/worker.py new file mode 100644 index 00000000..1e14f773 --- /dev/null +++ b/examples/dds/pycompss/worker/gat/worker.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - GAT - Worker. + +This file contains the worker code for GAT. +Args: debug full_path (method_class) +method_name has_target num_params par_type_1 par_1 ... par_type_n par_n +""" + +import logging +import sys + +from pycompss.streams.components.distro_stream_client import ( + DistroStreamClientHandler, +) +from pycompss.util.context import CONTEXT +from pycompss.util.logger.helpers import init_logging_worker +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.logger.level import LOG_LEVEL +from pycompss.util.tracing.helpers import dummy_context +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.helpers import trace_multiprocessing_worker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.worker.commons.worker import execute_task + + +# Uncomment the next line if you do not want to reuse pyc files. +# sys.dont_write_bytecode = True + + +def compss_worker( + tracing: bool, + task_id: str, + storage_conf: str, + params: list, +) -> int: + """Worker main method (invoked from __main__). + + :param tracing: Tracing boolean. + :param task_id: Task identifier. + :param storage_conf: Storage configuration file. + :param params: Parameters following the common order of the workers. + :return: Exit code. + """ + if __debug__: + logger = logging.getLogger("pycompss.worker.gat.worker") + logger.debug("Starting Worker") + + # Set the binding in worker mode + CONTEXT.set_worker() + + result = execute_task( + "".join(("Task ", task_id)), + storage_conf, + params, + tracing, + logger, + (), + False, + {}, + None, + None, + ) + # Result contains: + # exit_code, new_types, new_values, timed_out, except_msg = result + exit_code, _, _, _, _ = result + + if __debug__: + logger.debug("Finishing Worker") + + return exit_code + + +def main() -> None: + """GAT worker main code. + + Executes the task provided by parameters. + + :return: None. + """ + # Emit sync event if tracing is enabled + tracing = sys.argv[1] == "true" + task_id = int(sys.argv[2]) + log_level = sys.argv[3] + storage_conf = sys.argv[4] + stream_backend = sys.argv[5] + stream_master_name = sys.argv[6] + stream_master_port = sys.argv[7] + # Next: method_type = sys.argv[8] + params = sys.argv[9:] + # Next parameters: + # class_name = sys.argv[10] + # method_name = sys.argv[11] + # num_slaves = sys.argv[12] + # i = 13 + num_slaves + # slaves = sys.argv[12..i] + # numCus = sys.argv[i+1] + # has_target = sys.argv[i+2] == 'true' + # num_params = int(sys.argv[i+3]) + # params = sys.argv[i+4..] + + if log_level in ("true", "debug"): + print(f"Tracing = {str(tracing)}") + print(f"Task id = {str(task_id)}") + print(f"Log level = {str(log_level)}") + print(f"Storage conf = {str(storage_conf)}") + + persistent_storage = False + if storage_conf != "null": + persistent_storage = True + + streaming = False + if stream_backend not in [None, "null", "NONE"]: + streaming = True + + with trace_multiprocessing_worker() if tracing else dummy_context(): + if streaming: + # Start streaming + DistroStreamClientHandler.init_and_start( + master_ip=stream_master_name, master_port=stream_master_port + ) + + # Load log level configuration file + if log_level in ("true", "debug"): + # Debug + init_logging_worker( + LOG_REMITTENT.GAT_WORKER, LOG_LEVEL.DEBUG, tracing + ) + elif log_level == "info": + # Info + init_logging_worker( + LOG_REMITTENT.GAT_WORKER, LOG_LEVEL.INFO, tracing + ) + else: + # Default (off) + init_logging_worker( + LOG_REMITTENT.GAT_WORKER, LOG_LEVEL.OFF, tracing + ) + + if persistent_storage: + # Initialize storage + with EventWorker(TRACING_WORKER.init_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + initWorker as initStorageAtWorker, + ) + + initStorageAtWorker(config_file_path=storage_conf) + + # Init worker + exit_code = compss_worker(tracing, str(task_id), storage_conf, params) + + if streaming: + # Finish streaming + DistroStreamClientHandler.set_stop() + + if persistent_storage: + # Finish storage + with EventWorker(TRACING_WORKER.finish_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + finishWorker as finishStorageAtWorker, + ) + + finishStorageAtWorker() + + if exit_code == 1: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/dds/pycompss/worker/piper/__init__.py b/examples/dds/pycompss/worker/piper/__init__.py new file mode 100644 index 00000000..fc81e527 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the piper worker functions, constants and classes.""" diff --git a/examples/dds/pycompss/worker/piper/cache/__init__.py b/examples/dds/pycompss/worker/piper/cache/__init__.py new file mode 100644 index 00000000..44e19e55 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the piper worker cache helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/worker/piper/cache/classes.py b/examples/dds/pycompss/worker/piper/cache/classes.py new file mode 100644 index 00000000..940d2099 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/classes.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Cache classes. + +This file contains the cache required classes. +""" + +from pycompss.util.process.manager import Queue + +# Used only for typing +from pycompss.util.process.manager import DictProxy # noqa: F401 +from pycompss.util.typing_helper import typing + + +class TaskWorkerCache: + """Class that represents the cache attributes in the worker.""" + + __slots__ = [ + "ids", + "in_queue", + "out_queue", + "references", + "profiler", + ] + + def __init__(self) -> None: + """Worker cache placeholder constructor.""" + # These variables are initialized on call since they are only for + # the worker + self.ids = None # type: typing.Optional[DictProxy] + self.in_queue = Queue() # type: Queue + self.out_queue = Queue() # type: Queue + # Placeholder to keep the object references and avoid garbage collector + self.references = [] # type: typing.List[typing.Any] + # If profiling cache + self.profiler = False + + +class CacheQueueMessage: + """Class that represents a message to the cache.""" + + __slots__ = [ + "action", + "messages", # list of strings + "size", # size for put messages + "d_type", # d_type for put messages + "shape", # shape for put messages + ] + + def __init__( + self, + action: str = "undefined", + messages: typing.Sequence[str] = (), + size: int = 0, + d_type: type = type(None), + shape: typing.Sequence[int] = (), + ) -> None: + """Cache message placeholder constructor.""" + self.action = action + # Store any kind of string messages + self.messages = messages + # Store specific parameters + self.size = size + self.d_type = d_type + self.shape = shape + + def __repr__(self) -> str: + """Message representation.""" + message = f"Action: {self.action}" + message += f" Messages: {self.messages}" + message += f" Size: {self.size}" + message += f" d_type: {self.d_type}" + message += f" Shape: {self.shape}" + return message diff --git a/examples/dds/pycompss/worker/piper/cache/manager.py b/examples/dds/pycompss/worker/piper/cache/manager.py new file mode 100644 index 00000000..f8947320 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/manager.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Cache Tracker Manager process. + +This file contains the cache object tracker manager process. +""" + + +from multiprocessing import Queue +from typing import Union +import base64 + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.helpers import EventWorkerCache +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.tracing.types_events_worker import TRACING_WORKER_CACHE +from pycompss.util.typing_helper import typing +from pycompss.worker.piper.cache.classes import CacheQueueMessage +from pycompss.worker.piper.cache.profiler import add_profiler_get_put +from pycompss.worker.piper.cache.profiler import add_profiler_get_struct +from pycompss.worker.piper.cache.profiler import profiler_print_message +from pycompss.worker.piper.cache.tracker import CacheTrackerConf +from pycompss.worker.piper.cache.tracker import get_file_name +from pycompss.worker.piper.cache.tracker import get_file_name_clean + + +CACHE_MANAGER_HEADER = "[PYTHON CACHE MANAGER]" + +CP = None # type: typing.Any + + +def __get_fraction_cache_size__(device: str, fraction: float) -> int: + global CP + + if device == "cpu": + with open("/proc/meminfo") as meminfo_fd: + full_meminfo = meminfo_fd.readline() + + cache_size = int(full_meminfo.split()[1]) * 1024 * fraction + else: + if CP: + gpu_max_mem = int(CP.cuda.Device().mem_info[1]) + cache_size = gpu_max_mem * fraction + else: + cache_size = 0.0 + + return int(cache_size) + + +def __calculate_cache_size__(device: str, size: Union[int, float]) -> int: + if size < 1: + return __get_fraction_cache_size__(device, size) + return int(size) + + +def __is_shared_type_cuda__(sharedtype): + return "Cuda" in sharedtype + + +def cache_manager( + in_queue: Queue, + out_queue: Queue, + process_name: str, + conf: CacheTrackerConf, +) -> None: + """Process main body. + + :param in_queue: Queue where to retrieve queue messages. + :param out_queue: Queue where to put output messages. + :param process_name: Process name. + :param conf: configuration of the cache tracker. + :return: None. + """ + # First thing to do is to emit the process identifier event + emit_manual_event_explicit( + TRACING_WORKER.process_identifier, + TRACING_WORKER.process_worker_cache_event, + ) + + global CP + try: + import cupy + + CP = cupy + + gpu_used_size_dict = {} + for i in range(cupy.cuda.runtime.getDeviceCount()): + gpu_used_size_dict[i] = 0 + with cupy.cuda.Device(i): + cupy.cuda.set_allocator(None) + cupy.cuda.set_pinned_memory_allocator(None) + except ImportError: + pass + + # Process properties + alive = True + logger = conf.logger + max_size = __calculate_cache_size__("cpu", conf.size) + gpu_max_size = __calculate_cache_size__("gpu", conf.gpu_cache_size) + cache_ids = conf.cache_ids + cache_hits = conf.cache_hits + profiler_dict = conf.profiler_dict + profiler_get_struct = conf.profiler_get_struct + analysis_dir = conf.analysis_dir + cache_profiler = conf.cache_profiler + + if __debug__: + logger.debug( + "%s [%s] Starting Cache Manager: CPU %.1fGB, GPU: %.1fGB", + CACHE_MANAGER_HEADER, + str(process_name), + max_size / 1e9, + gpu_max_size / 1e9, + ) + + # MAIN CACHE TRACKER LOOP + msg = CacheQueueMessage() + used_size = 0 + locked = set() # set containing the locked entries + while alive: + with EventWorkerCache(TRACING_WORKER_CACHE.cache_msg_receive_event): + msg = in_queue.get() + action = msg.action + if action == "QUIT": + with EventWorkerCache(TRACING_WORKER_CACHE.cache_msg_quit_event): + if __debug__: + logger.debug( + "%s [%s] Stopping Cache Tracker: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(msg), + ) + logger.debug( + "%s [%s] Cache hits status:", + CACHE_MANAGER_HEADER, + str(process_name), + ) + used_size = 0 + entries = 0 + for hits, elements in cache_hits.items(): + if elements: # not empty entry + logger.debug( + f"{CACHE_MANAGER_HEADER} [{process_name}] " + f"{hits} hits:" + ) + for obj_name, size in elements.items(): + logger.debug( + f"{CACHE_MANAGER_HEADER} [{process_name}] " + f"\t- {obj_name} {size}" + ) + used_size += size + entries += 1 + logger.debug( + "%s [%s] Entries: %s Max size: %s Used size: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(entries), + str(max_size), + str(used_size), + ) + alive = False + elif action == "END_PROFILING": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_end_profiling_event + ): + if cache_profiler: + profiler_print_message( + profiler_dict, profiler_get_struct, analysis_dir + ) + else: + try: + if action == "GET": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_get_event + ): + f_name, parameter, function = msg.messages + if f_name not in cache_ids: + # The object does not exist in the Cache + # It does not go inside here due to we check if it + # is in cache before trying to get (from + # runtime/task/worker.py). + if __debug__: + logger.debug( + "%s [%s] Cache miss", + CACHE_MANAGER_HEADER, + str(process_name), + ) + else: + if cache_profiler: + # PROFILER GET + add_profiler_get_put( + profiler_dict, + function, + parameter, + f_name, + "GET", + ) + # PROFILER GET STRUCTURE + add_profiler_get_struct( + profiler_get_struct, + function, + parameter, + f_name, + ) + # Increment the number of hits + if __debug__: + logger.debug( + "%s [%s] Cache hit", + CACHE_MANAGER_HEADER, + str(process_name), + ) + # Increment hits + current = cache_ids[f_name] + obj_size = current[3] + current_hits = current[4] + new_hits = current_hits + 1 + current[4] = new_hits + cache_ids[f_name] = ( + current # forces updating whole entry + ) + # Keep cache_hits structure + try: + cache_hits[current_hits].pop(f_name) + except KeyError: + pass + if new_hits in cache_hits: + cache_hits[new_hits][f_name] = obj_size + else: + cache_hits[new_hits] = {f_name: obj_size} + elif action == "PUT": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_put_event + ): + ( + f_name, + cache_id, + shared_type, + parameter, + function, + ) = msg.messages + obj_size = msg.size + dtype = msg.d_type + shape = msg.shape + + if f_name in cache_ids: + # The object already exists + if __debug__: + logger.debug( + "%s [%s] The object already exists " + "NOT adding: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(msg), + ) + else: + # Add new entry request + if __debug__: + logger.debug( + "%s [%s] Cache add entry: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(msg), + ) + if cache_profiler: + # PROFILER PUT + add_profiler_get_put( + profiler_dict, + function, + parameter, + get_file_name_clean(f_name), + "PUT", + ) + # Check if it is going to fit + # and remove if necessary + obj_size = int(obj_size) + if used_size + obj_size > max_size: + # Cache is full, need to evict + used_size = __free_cache_space__( + conf, used_size, obj_size + ) + # Accumulate size + used_size += obj_size + # Initial hits + hits = 0 + # Add without problems + cache_ids[f_name] = [ + cache_id, + shape, + dtype, + obj_size, + hits, + shared_type, + ] + # Register in hits dictionary + if hits not in cache_hits: + cache_hits[hits] = {} + cache_hits[hits][f_name] = obj_size + elif action == "PUT_GPU": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_put_gpu_event + ): + ( + f_name, + cache_id, + shared_type, + parameter, + function, + device_pci_bus_id, + ) = msg.messages + + obj_size = msg.size + dtype = msg.d_type + shape = msg.shape + + obj_size = int(obj_size) + + for i in range(CP.cuda.runtime.getDeviceCount()): + if ( + CP.cuda.Device(i).pci_bus_id + == device_pci_bus_id + ): + device_id = i + + logger.debug( + "%s [%s] DEVICE ID: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(device_pci_bus_id), + ) + + if f_name in cache_ids: + # The object already exists + if __debug__: + logger.debug( + "%s [%s] The object already exists " + "NOT adding: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(msg), + ) + elif obj_size > gpu_max_size: + logger.debug( + "%s [%s] The object (%.1fMB) is bigger " + "than max GPU cache size(%.1fMB)", + CACHE_MANAGER_HEADER, + str(process_name), + obj_size / 1e6, + gpu_max_size / 1e6, + ) + else: + # Add new entry request + if __debug__: + logger.debug( + "%s [%s] Cache add entry: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(msg), + ) + if cache_profiler: + # PROFILER PUT + add_profiler_get_put( + profiler_dict, + function, + parameter, + get_file_name_clean(f_name), + "PUT_GPU", + ) + + gpu_used_size = gpu_used_size_dict[device_id] + with CP.cuda.Device(device_id): + if gpu_used_size + obj_size > gpu_max_size: + gpu_used_size_dict[device_id] = ( + __free_gpu_cache_space__( + conf, + gpu_used_size, + obj_size, + device_id, + ) + ) + + cache_mem = CP.cuda.memory.BaseMemory() + cache_mem.ptr = CP.cuda.runtime.malloc( + obj_size + ) + + try: + handler = base64.b64decode(cache_id) + array_open = ( + CP.cuda.runtime.ipcOpenMemHandle( + handler + ) + ) + + CP.cuda.runtime.memcpy( + cache_mem.ptr, array_open, obj_size, 0 + ) + + CP.cuda.runtime.ipcCloseMemHandle( + array_open + ) + + new_handler = ( + CP.cuda.runtime.ipcGetMemHandle( + cache_mem.ptr + ) + ) + new_cache_id = base64.b64encode( + new_handler + ).decode("ascii") + + conf.gpu_arr_ptr[new_cache_id] = [ + cache_mem.ptr, + device_id, + ] + gpu_used_size_dict[device_id] += obj_size + + hits = 0 + cache_ids[f_name] = [ + new_cache_id, + shape, + dtype, + obj_size, + hits, + shared_type, + ] + + if hits not in cache_hits: + cache_hits[hits] = {} + cache_hits[hits][f_name] = obj_size + except Exception: + import traceback + + logger.debug( + "%s [%s] PUT GPU ERROR: %s %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(device_id), + str(traceback.format_exc()), + ) + CP.cuda.runtime.free(cache_mem.ptr) + out_queue.put("OK") + elif action == "REMOVE": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_remove_event + ): + f_name_msg = msg.messages[0] + f_name = get_file_name(f_name_msg) + logger.debug( + "%s [%s] Removing: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(f_name), + ) + ( + shared_type, + obj_size, + device_id, + ) = __remove_from_cache__( + f_name, + cache_ids, + cache_hits, + gpu_arr_ptr=conf.gpu_arr_ptr, + ) + + if __is_shared_type_cuda__(shared_type): + gpu_used_size_dict[device_id] -= obj_size + else: + used_size -= obj_size + elif action == "LOCK": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_lock_event + ): + f_name_msg = msg.messages[0] + f_name = get_file_name(f_name_msg) + if f_name in locked: + raise PyCOMPSsException( + "Cache coherence issue: " + "tried to lock an already locked file entry" + ) + locked.add(f_name) + logger.debug( + "%s [%s] Locking: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(f_name), + ) + elif action == "UNLOCK": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_unlock_event + ): + f_name_msg = msg.messages[0] + f_name = get_file_name(f_name_msg) + logger.debug( + "%s [%s] Unlocking: %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(f_name), + ) + try: + locked.remove(f_name) + except KeyError as key_error: + raise PyCOMPSsException( + "Cache coherence issue: " + "tried to remove locked but failed" + ) from key_error + elif action == "IS_LOCKED": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_is_locked_event + ): + f_name_msg = msg.messages[0] + f_name = get_file_name(f_name_msg) + is_locked = f_name in locked + out_queue.put(is_locked) + logger.debug( + "%s [%s] Get if is locked: %s : %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(f_name), + str(is_locked), + ) + elif action == "IS_IN_CACHE": + with EventWorkerCache( + TRACING_WORKER_CACHE.cache_msg_is_in_cache_event + ): + f_name_msg = msg.messages[0] + f_name = get_file_name(f_name_msg) + is_in_cache = f_name in cache_ids + out_queue.put(is_in_cache) + logger.debug( + "%s [%s] Get if is in cache: %s : %s", + CACHE_MANAGER_HEADER, + str(process_name), + str(f_name), + str(is_in_cache), + ) + + except Exception as general_exception: + logger.exception( + "%s - Exception %s", + str(process_name), + str(general_exception), + ) + alive = False + + +def __remove_from_cache__(f_name, cache_ids, cache_hits, gpu_arr_ptr=None): + """Remove object from cache. + + :param f_name: File name (object identifier). + :param cache_ids: Cache identifiers. + :param cache_hits: Cache hits structure. + :param gpu_arr_ptr: Gpu array pointer. + :return: Type, size and device identifier. + """ + (cache_id, _, _, size, current_hits, shared_type) = cache_ids.pop(f_name) + + if __is_shared_type_cuda__(shared_type): + gpu_ptr, device_id = gpu_arr_ptr.pop(cache_id) + with CP.cuda.Device(device_id): + CP.cuda.runtime.free(gpu_ptr) + else: + device_id = None + + cache_hits[current_hits].pop(f_name) + + return shared_type, size, device_id + + +def __free_cache_space__( + conf: CacheTrackerConf, used_size: int, requested_size: int +) -> int: + """Check the cache status looking into the shared dictionary. + + :param conf: configuration of the cache tracker. + :param used_size: Current used size of the cache. + :param requested_size: Size needed to fit the new object. + :return: New used size. + """ + logger = conf.logger # noqa + max_size = int(conf.size) + cache_ids = conf.cache_ids + cache_hits = conf.cache_hits + + if __debug__: + logger.debug( + "%s Checking cache status: Requested %s bytes", + CACHE_MANAGER_HEADER, + str(requested_size), + ) + + sorted_hits = sorted(cache_hits.keys()) + + # Calculate size to recover + size_to_recover = used_size + requested_size - max_size + # Select how many to evict + evicted, recovered_size = __cpu_evict__( + sorted_hits, cache_ids, cache_hits, size_to_recover + ) + if __debug__: + logger.debug("%s Evicting %d entries", CACHE_MANAGER_HEADER, (evicted)) + + return used_size - recovered_size + + +def __cpu_evict__( + sorted_hits: typing.List[int], + cache_ids, + cache_hits: typing.Dict[int, typing.Dict[str, int]], + size_to_recover: int, +) -> typing.Tuple[int, int]: + """Select how many entries to evict. + + :param sorted_hits: List of current hits sorted from lower to higher. + :param cache_hits: Cache hits structure. + :param size_to_recover: Amount of size to recover. + :return: List of f_names to evict. + """ + to_evict = [] + total_recovered_size = 0 + for hits in sorted_hits: + # Does not check the order by size of the objects since they have + # the same amount of hits + files = list(cache_hits[hits]) + for f_name in files: + _, recovered_size, _ = __remove_from_cache__( + f_name, cache_ids, cache_hits + ) + to_evict.append(f_name) + size_to_recover -= recovered_size + total_recovered_size += recovered_size + if size_to_recover <= 0: + return len(to_evict), total_recovered_size + return len(to_evict), total_recovered_size + + +def __free_gpu_cache_space__( + conf: CacheTrackerConf, + gpu_used_size: int, + requested_size: int, + device_id: int, +) -> int: + """Check the cache status looking into the shared dictionary. + + :param conf: configuration of the cache tracker. + :param used_size: Current used size of the cache. + :param requested_size: Size needed to fit the new object. + :return: New used size. + """ + logger = conf.logger # noqa + max_size = int(conf.size) + cache_ids = conf.cache_ids + cache_hits = conf.cache_hits + + if __debug__: + logger.debug( + "%s Checking cache status: Requested %s bytes", + CACHE_MANAGER_HEADER, + str(requested_size), + ) + + sorted_hits = sorted(cache_hits.keys()) + + # Calculate size to recover + size_to_recover = gpu_used_size + requested_size - max_size + # Select how many to evict + evicted, recovered_size = __gpu_evict__( + sorted_hits, + cache_ids, + cache_hits, + conf.gpu_arr_ptr, + size_to_recover, + device_id, + ) + if __debug__: + logger.debug( + "%s Evicting %d entries from GPU", CACHE_MANAGER_HEADER, (evicted) + ) + + return gpu_used_size - recovered_size + + +def __gpu_evict__( + sorted_hits: typing.List[int], + cache_ids, + cache_hits: typing.Dict[int, typing.Dict[str, int]], + gpu_arr_ptr, + size_to_recover: int, + device_id: int, +) -> typing.Tuple[int, int]: + """Select how many entries to evict. + + :param sorted_hits: List of current hits sorted from lower to higher. + :param cache_hits: Cache hits structure. + :param size_to_recover: Amount of size to recover. + :return: List of f_names to evict. + """ + to_evict = [] + total_recovered_size = 0 + + for hits in sorted_hits: + # Does not check the order by size of the objects since they have + # the same amount of hits + files = list(cache_hits[hits]) + for f_name in files: + obj_id, _, _, _, _, shared_type = cache_ids[f_name] + + if ( + __is_shared_type_cuda__(shared_type) + and device_id == gpu_arr_ptr[obj_id][1] + ): + _, obj_size, _ = __remove_from_cache__( + f_name, cache_ids, cache_hits, gpu_arr_ptr=gpu_arr_ptr + ) + to_evict.append(f_name) + size_to_recover -= obj_size + total_recovered_size += obj_size + if size_to_recover <= 0: + return len(to_evict), total_recovered_size + return len(to_evict), total_recovered_size diff --git a/examples/dds/pycompss/worker/piper/cache/profiler.py b/examples/dds/pycompss/worker/piper/cache/profiler.py new file mode 100644 index 00000000..1ffa8747 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/profiler.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Cache Profiler. + +This file contains the cache object profiler functions. +""" + +import json +from pycompss.util.typing_helper import typing + + +def add_profiler_get_put( + profiler_dict: typing.Dict[ + str, typing.Dict[str, typing.Dict[str, typing.Dict[str, int]]] + ], + function: str, + parameter: str, + filename: str, + parameter_type: str, +) -> None: + """Add get and put entry to the profiler. + + :param profiler_dict: Profiling dictionary. + :param function: Function to include. + :param parameter: Function parameter to include. + :param filename: File name associated to the parameter. + :param parameter_type: Parameter type. + :return: None. + """ + if function not in profiler_dict: + profiler_dict[function] = {} + if parameter not in profiler_dict[function]: + profiler_dict[function][parameter] = {} + if filename not in profiler_dict[function][parameter]: + profiler_dict[function][parameter][filename] = {"PUT": 0, "GET": 0} + profiler_dict[function][parameter][filename][parameter_type] += 1 + + +def add_profiler_get_struct( + profiler_get_struct: list, function: str, parameter: str, filename: str +) -> None: + """Add info to profiling struct entry. + + :param profiler_get_struct: Profiling struct. + :param function: Function to include + :param parameter: Function parameter to include. + :param filename: File name associated to the parameter. + :return: None. + """ + if ( + function not in profiler_get_struct[2] + and parameter not in profiler_get_struct[1] + ): + profiler_get_struct[0].append(filename) + profiler_get_struct[1].append(parameter) + profiler_get_struct[2].append(function) + + +def profiler_print_message( + profiler_dict: typing.Dict[ + str, typing.Dict[str, typing.Dict[str, typing.Dict[str, int]]] + ], + profiler_get_struct: typing.List[typing.List[str]], + analysis_dir: str, +) -> None: + r"""Export profiling information to json. + + for function in profiler_dict: + f.write('\t' + "FUNCTION: " + str(function)) + logger.debug('\t' + "FUNCTION: " + str(function)) + for parameter in profiler_dict[function]: + f.write('\t' + '\t' + '\t' + "PARAMETER: " + str(parameter)) + logger.debug('\t' + '\t' + '\t' + "PARAMETER: " + str(parameter)) + for filename in profiler_dict[function][parameter]: + _put = profiler_dict[function][parameter][filename]['PUT'] + _get = profiler_dict[function][parameter][filename]['GET'] + f.write('\t' + '\t' + '\t' + '\t' + "FILENAME: " + filename + + '\t' + " PUT " + str(_put) + " GET " + str(_get)) + logger.debug('\t' + '\t' + '\t' + '\t' + "FILENAME: " + + filename + '\t' + " PUT " + + str(_put) + " GET " + str(_get)) + f.write("") + logger.debug("") + logger.debug("PROFILER GETS") + for i in range(len(profiler_get_struct[0])): + logger.debug('\t' + "FILENAME: " + profiler_get_struct[0][i] + + ". PARAMETER: " + profiler_get_struct[1][i] + + ". FUNCTION: " + profiler_get_struct[2][i]) + + :param profiler_dict: Profiling dictionary. + :param profiler_get_struct: Profiling struct. + :param analysis_dir: Analysis directory. + :return: None. + """ + f_d_type = typing.Dict[ # noqa # pylint: disable=unused-variable + str, + typing.Dict[ + str, + typing.Dict[str, typing.Union[str, int, bool, typing.List[str]]], + ], + ] + final_dict = {} # type: f_d_type + for function in profiler_dict: + final_dict[function] = {} + for parameter in profiler_dict[function]: + total_get = 0 + total_put = 0 + is_used = [] + filenames = profiler_dict[function][parameter] + final_dict[function][parameter] = {} + for filename in filenames: + puts = filenames[filename]["PUT"] + if puts > 0: + try: + index = profiler_get_struct[0].index(filename) + is_used.append( + profiler_get_struct[2][index] + + "#" + + profiler_get_struct[1][index] + ) + except ValueError: + pass + total_put += puts + total_get += filenames[filename]["GET"] + final_dict[function][parameter]["GET"] = total_get + final_dict[function][parameter]["PUT"] = total_put + + if len(is_used) > 0: + final_dict[function][parameter]["USED"] = is_used + elif total_get > 0: + final_dict[function][parameter]["USED"] = [ + function + "#" + parameter + ] + else: + final_dict[function][parameter]["USED"] = [] + + with open( + analysis_dir + "/cache_profiler.json", "a", encoding="utf-8" + ) as json_file: + json.dump(final_dict, json_file) diff --git a/examples/dds/pycompss/worker/piper/cache/setup.py b/examples/dds/pycompss/worker/piper/cache/setup.py new file mode 100644 index 00000000..e5b4ab21 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/setup.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Cache Setup. + +This file contains the cache setup and instantiation. +IMPORTANT: Only used with python >= 3.8. +""" +import logging +import re +from typing import Union + +from pycompss.util.process.manager import Process +from pycompss.util.process.manager import Queue +from pycompss.util.process.manager import create_process +from pycompss.util.process.manager import create_proxy_dict +from pycompss.util.process.manager import DictProxy # typing only +from pycompss.util.process.manager import new_queue +from pycompss.util.typing_helper import typing +from pycompss.worker.piper.cache.classes import CacheQueueMessage +from pycompss.worker.piper.cache.tracker import CacheTrackerConf +from pycompss.worker.piper.cache.tracker import CACHE_TRACKER +from pycompss.worker.piper.cache.manager import cache_manager + +# Used only for typing shortcut +Dict = typing.Dict +List = typing.List + + +# For calculating cache size +size_units = { + "B": 1, + "KB": 10**3, + "MB": 10**6, + "GB": 10**9, + "TB": 10**12, +} + +# RegEx for parsing ``cpu:25%`` or ``gpu:5.5GB`` ... +CACHE_SIZE_REGEX = r"(\d+(\.\d+)?)\s*([A-Za-z%]+)" + + +def is_cache_enabled(cache_config: str) -> bool: + """Check if the cache is enabled. + + :param cache_config: Cache configuration defined on startup. + :return: True if enabled, False otherwise. And size if enabled. + """ + return cache_config.lower() not in ["false", "null"] + + +def start_cache( + logger: logging.Logger, + cache_config: str, + cache_profiler: bool, + analysis_dir: str, +) -> typing.Tuple[typing.Any, Process, Queue, Queue, DictProxy]: + """Set up the cache process which keeps the consistency of the cache. + + :param logger: Logger. + :param cache_config: Cache configuration defined on startup. + :param cache_profiler: If cache profiling is enabled or not. + :param analysis_dir: Directory where to store the profiling. + :return: Shared memory manager, cache process, cache message queue and + cache ids dictionary. + """ + cache_config = cache_config.replace(" ", "").strip() + main_memory_cache_size = get_cache_size("cpu", cache_config) + gpu_cache_size = get_cache_size("gpu", cache_config) + # Cache can be used - Create proxy dict + cache_ids = create_proxy_dict() + cache_hits = {} # type: Dict[int, Dict[str, int]] + profiler_dict = {} # type: Dict[str, Dict[str, Dict[str, Dict[str, int]]]] + profiler_get_struct = [[], [], []] # type: List[List[str]] + # profiler_get_struct structure: Filename, Parameter, Function + smm = CACHE_TRACKER.start_shared_memory_manager() + conf = CacheTrackerConf( + logger, + main_memory_cache_size, + gpu_cache_size, + "default", + cache_ids, + cache_hits, + profiler_dict, + profiler_get_struct, + analysis_dir, + cache_profiler, + ) + ( + cache_process, + in_cache_queue, + out_cache_queue, + ) = create_cache_manager_process(conf) + return smm, cache_process, in_cache_queue, out_cache_queue, cache_ids + + +def stop_cache( + shared_memory_manager: typing.Any, + in_cache_queue: Queue, + out_cache_queue: Queue, + cache_profiler: bool, + cache_process: Process, +) -> None: + """Stop the cache process and performs the necessary cleanup. + + :param shared_memory_manager: Shared memory manager. + :param in_cache_queue: Cache messaging input queue. + :param out_cache_queue: Cache messaging output queue. + :param cache_profiler: If cache profiling is enabled or not. + :param cache_process: Cache process. + :return: None. + """ + if cache_profiler: + message = CacheQueueMessage(action="END_PROFILING") + in_cache_queue.put(message) + __destroy_cache_tracker_process( + cache_process, in_cache_queue, out_cache_queue + ) + CACHE_TRACKER.stop_shared_memory_manager(shared_memory_manager) + + +def get_cache_size(device: str, cache_config: str) -> Union[float, int]: + """Retrieve the CPU/GPU Cache size for the given config. + + :param device: `cpu` or `gpu`. + :param cache_config: Cache configuration defined on startup. + :return: The cache size. + """ + pattern = device + ":" + CACHE_SIZE_REGEX + + matches = re.findall(pattern, cache_config) + + if not matches: + cache_size = 0.25 # Default value 25% of node memory + else: + num, _, unit = matches[0] + num = float(num) + if unit == "%": + num = num / 100 + else: + num = size_units[unit.upper()] * num + cache_size = num + + return cache_size + + +def create_cache_manager_process( + conf: CacheTrackerConf, +) -> typing.Tuple[Process, Queue, Queue]: + """Start a new cache tracker process. + + :param process_name: Process name. + :param conf: cache config. + :return: Cache tracker process, in queue and out queue. + """ + in_queue = new_queue() + out_queue = new_queue() + process = create_process( + target=cache_manager, args=(in_queue, out_queue, "cache_manager", conf) + ) + process.start() + return process, in_queue, out_queue + + +def __destroy_cache_tracker_process( + cache_process: Process, in_queue: Queue, out_queue: Queue +) -> None: + """Stop the given cache tracker process. + + :param cache_process: Cache process. + :param in_queue: Cache messaging input queue. + :param out_queue: Cache messaging output queue. + :return: None. + """ + message = CacheQueueMessage(action="QUIT") + in_queue.put(message) # noqa + cache_process.join() # noqa + in_queue.close() # noqa + in_queue.join_thread() # noqa + out_queue.close() # noqa + out_queue.join_thread() # noqa diff --git a/examples/dds/pycompss/worker/piper/cache/tracker.py b/examples/dds/pycompss/worker/piper/cache/tracker.py new file mode 100644 index 00000000..0f185045 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/cache/tracker.py @@ -0,0 +1,910 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Cache Tracker. + +This file contains the cache object tracker. +IMPORTANT: Only used with python >= 3.8. +""" +import base64 +import logging +import os +from typing import Union + +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.objects.sizer import total_sizeof +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.util.process.manager import create_shared_memory_manager +from pycompss.util.process.manager import Queue +from pycompss.util.process.manager import DictProxy +from pycompss.worker.piper.cache.classes import CacheQueueMessage + +try: + from pycompss.util.process.manager import SharedMemory + from pycompss.util.process.manager import ShareableList + from pycompss.util.process.manager import SharedMemoryManager +except ImportError: + # Unsupported in python < 3.8 + SharedMemory = None # type: ignore + ShareableList = None # type: ignore + SharedMemoryManager = None # type: ignore + + +# Supported shared objects (remind that nested lists are not supported). +SHARED_MEMORY_TAG = "SharedMemory" +SHARED_MEMORY_CUPY_TAG = "SharedCupyCudaMemory" +SHAREABLE_LIST_TAG = "ShareableList" +SHAREABLE_TUPLE_TAG = "ShareableTuple" +SHARED_MEMORY_TORCH_TAG = "SharedTorchCudaMemory" + +# Try to import numpy +NP = None # type: typing.Any +CP = None # type: typing.Any +TORCH = None # type: typing.Any +try: + import numpy # noqa + + NP = numpy +except ImportError: + print( + "WARNING: No Numpy available. " + "Cache using Numpy objects will not be available" + ) + + +def __set_cuda_libs__(): + global TORCH + try: + import torch # noqa + + TORCH = torch + except ImportError: + pass + + global CP + try: + import cupy + + CP = cupy + except ImportError: + pass + + +class SharedMemoryConfig: + """Shared memory configuration.""" + + __slots__ = ["_auth_key", "_ip_address", "_port"] + + def __init__(self) -> None: + """Initialize a new SharedMemoryConfig instance object. + + All parameters are final. + """ + self._auth_key = b"compss_cache" + self._ip_address = "127.0.0.1" + self._port = 50000 + + def get_auth_key(self) -> bytes: + """Retrieve the authentication key. + + :return: The authentication key. + """ + return self._auth_key + + def get_ip(self) -> str: + """Retrieve the IP address. + + :return: The IP address. + """ + return self._ip_address + + def get_port(self) -> int: + """Retrieve the port. + + :return: The port. + """ + return self._port + + +class CacheTrackerConf: + """Cache tracker configuration class.""" + + __slots__ = [ + "logger", + "size", + "gpu_cache_size", + "policy", + "cache_ids", + "cache_hits", + "profiler_dict", + "profiler_get_struct", + "analysis_dir", + "cache_profiler", + "gpu_arr_ptr", + ] + + def __init__( + self, + logger: logging.Logger, + size: Union[float, int], + gpu_cache_size: Union[float, int], + policy: str, + cache_ids: DictProxy, + cache_hits: typing.Dict[int, typing.Dict[str, int]], + profiler_dict: typing.Dict[ + str, typing.Dict[str, typing.Dict[str, typing.Dict[str, int]]] + ], + profiler_get_struct: typing.List[typing.List[str]], + analysis_dir: str, + cache_profiler: bool, + ) -> None: + """Construct a new cache tracker configuration. + + :param logger: Main logger. + :param size: Total cache size supported. + :param policy: Eviction policy. + :param cache_ids: Shared dictionary proxy where the ids and + (size, hits) as its value are. + :param cache_hits: Dictionary containing size and keys to ease + management. + :param profiler_dict: Profiling dictionary. + :param profiler_get_struct: Profiling get structure. + :param analysis_dir: Analysis directory. + :param cache_profiler: Cache profiler. + """ + self.logger = logger + self.size = size + self.policy = policy # currently no policies defined. + self.cache_ids = ( + cache_ids # key - (id, shape, dtype, size, hits, shared_type) + ) + self.cache_hits = cache_hits # hits - {key1: size1, key2: size2, etc.} + self.profiler_dict = profiler_dict + self.profiler_get_struct = profiler_get_struct + self.analysis_dir = analysis_dir + self.cache_profiler = cache_profiler + + self.gpu_cache_size = gpu_cache_size + self.gpu_arr_ptr: typing.Dict[str, typing.List[int]] = ( + {} + ) # cache_id - (cupy_data_ptr, device_id) + + +class CacheTracker: + """Cache Tracker manager (shared memory).""" + + __slots__ = [ + "header", + "shared_memory_manager", + "config", + "cupy_handlers", + "lock", + ] + + def __init__(self) -> None: + """Initialize a new SharedMemory instance object.""" + self.header = "[PYTHON CACHE TRACKER]" + # Shared memory manager to connect. + self.shared_memory_manager = None # type: typing.Any + # Configuration + self.config = SharedMemoryConfig() + # Others + self.lock = None # type: typing.Any + + self.cupy_handlers: typing.Dict[str, bytes] = dict() + + def set_lock(self, lock: typing.Any) -> None: + """Set lock for coherence. + + :param lock: Multiprocessing lock. + :return: None + """ + self.lock = lock + + def connect_to_shared_memory_manager(self) -> None: + """Connect to the shared memory manager initiated in piper_worker.py. + + :return: None. + """ + self.shared_memory_manager = create_shared_memory_manager( + address=(self.config.get_ip(), self.config.get_port()), + authkey=self.config.get_auth_key(), + ) + self.shared_memory_manager.connect() + + def start_shared_memory_manager(self) -> SharedMemoryManager: + """Start the shared memory manager. + + :return: Shared memory manager instance. + """ + smm = create_shared_memory_manager( + address=("", self.config.get_port()), + authkey=self.config.get_auth_key(), + ) + smm.start() # pylint: disable=consider-using-with + return smm + + @staticmethod + def stop_shared_memory_manager(smm: SharedMemoryManager) -> None: + """Stop the given shared memory manager. + + Releases automatically the objects contained in it. + Only needed to be stopped from the main worker process. + It is not necessary to disconnect each executor. + + :param smm: Shared memory manager. + :return: None. + """ + smm.shutdown() + + def __get_shared_numpy(self, obj_id, obj_shape: typing.Tuple, obj_d_type): + """Get shared numpy array. + + :param obj_id: Object identifier. + :param obj_shape: Numpy array shape. + :param obj_d_type: Numpy array dtype. + :return: Shared memory segment, numpy array and its size. + """ + existing_shm = SharedMemory(name=obj_id) + shm_np = NP.ndarray( + obj_shape, dtype=obj_d_type, buffer=existing_shm.buf + ) + output = NP.empty(obj_shape, dtype=obj_d_type) + NP.copyto(output, shm_np) + object_size = len(existing_shm.buf) + return existing_shm, output, object_size + + def __get_shared_cupy(self, obj_id, obj_shape: typing.Tuple, obj_d_type): + """Get shared cupy array. + + :param obj_id: Object identifier. + :param obj_shape: Cupy array shape. + :param obj_d_type: Cupy array dtype. + :return: None, cupy array and its size. + """ + if obj_id in self.cupy_handlers: + array_open = self.cupy_handlers[obj_id] + else: + handler = base64.b64decode(obj_id) + array_open = CP.cuda.runtime.ipcOpenMemHandle(handler) + self.cupy_handlers[obj_id] = array_open + + mem = CP.cuda.UnownedMemory(array_open, 0, obj_id) + + memptr = CP.cuda.MemoryPointer(mem, 0) + output = CP.ndarray(shape=obj_shape, dtype=obj_d_type, memptr=memptr) + + return None, output, output.nbytes + + def __get_shared_list(self, obj_id, i_type): + """Get iterable object. + + :param obj_id: Object identifier. + :param i_type: Object type (can be tuple or list). + :return: Shared memory segment, iterable object and its size. + """ + existing_shm = ShareableList(name=obj_id) + output = i_type(existing_shm) + object_size = len(existing_shm.shm.buf) + return existing_shm, output, object_size + + def close_cupy_mem_handles(self): + """Close all memory handles from cupy arrays. + + :return: None. + """ + for handle in self.cupy_handlers.values(): + CP.cuda.runtime.ipcCloseMemHandle(handle) + self.cupy_handlers.clear() + + def retrieve_object_from_cache( + self, + logger: logging.Logger, + cache_ids: typing.Any, + in_cache_queue: Queue, + out_cache_queue: Queue, + identifier: str, + parameter_name: str, + user_function: typing.Callable, + cache_profiler: bool, + ) -> typing.Any: + """Retrieve an object from the given cache proxy dict. + + :param logger: Logger where to push messages. + :param cache_ids: Cache proxy dictionary. + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :param identifier: Object identifier. + :param parameter_name: Parameter name. + :param user_function: Function name. + :param cache_profiler: If cache profiling is enabled. + :return: The object from cache. + """ + __set_cuda_libs__() + + emit_manual_event_explicit( + TRACING_WORKER.deserialization_cache_size_type, 0 + ) + f_name = get_file_name(identifier) + if __debug__: + logger.debug("%s Retrieving: %s", self.header, str(f_name)) + obj_id, obj_shape, obj_d_type, _, _, shared_type = cache_ids[f_name] + output = None # type: typing.Any + existing_shm = None # type: typing.Any + object_size = 0 + + if shared_type == SHARED_MEMORY_TAG: + with EventInsideWorker( + TRACING_WORKER.retrieve_object_from_cache_event + ): + existing_shm, output, object_size = self.__get_shared_numpy( + obj_id, obj_shape, obj_d_type + ) + elif shared_type == SHARED_MEMORY_TORCH_TAG: + with EventInsideWorker( + TRACING_WORKER.retrieve_object_from_gpu_cache_event + ): + existing_shm, output, object_size = self.__get_shared_cupy( + obj_id, obj_shape, obj_d_type + ) + output = TORCH.as_tensor(output, device="cuda") + elif shared_type == SHARED_MEMORY_CUPY_TAG: + with EventInsideWorker( + TRACING_WORKER.retrieve_object_from_gpu_cache_event + ): + existing_shm, output, object_size = self.__get_shared_cupy( + obj_id, obj_shape, obj_d_type + ) + elif shared_type == SHAREABLE_LIST_TAG: + with EventInsideWorker( + TRACING_WORKER.retrieve_object_from_cache_event + ): + existing_shm, output, object_size = self.__get_shared_list( + obj_id, list + ) + elif shared_type == SHAREABLE_TUPLE_TAG: + with EventInsideWorker( + TRACING_WORKER.retrieve_object_from_cache_event + ): + existing_shm, output, object_size = self.__get_shared_list( + obj_id, tuple + ) + else: + raise PyCOMPSsException("Unknown cacheable type.") + if __debug__: + logger.debug( + "%s Retrieved: %s as %s", self.header, str(f_name), obj_id + ) + emit_manual_event_explicit( + TRACING_WORKER.deserialization_cache_size_type, object_size + ) + + # Profiling + filename = get_file_name_clean(f_name) + function_name = __function_clean__(user_function) + + message = CacheQueueMessage( + action="GET", messages=[filename, parameter_name, function_name] + ) + in_cache_queue.put(message) + + return output, existing_shm + + def insert_object_into_cache( + self, + logger: logging.Logger, + in_cache_queue: Queue, + out_cache_queue: Queue, + obj: typing.Any, + f_name: str, + parameter: str, + user_function: typing.Callable, + ) -> None: + """Put an object into cache filtering supported types. + + :param logger: Logger where to push messages. + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :param obj: Object to store. + :param f_name: File name that corresponds to the object (used as id). + :param parameter: Parameter name. + :param user_function: Function. + :return: None. + """ + __set_cuda_libs__() + + if ( + NP + and in_cache_queue is not None + and out_cache_queue is not None + and ( + (isinstance(obj, NP.ndarray) and not obj.dtype == object) + or ( + CP + and isinstance(obj, CP.ndarray) + and not obj.dtype == object + ) + or ( + TORCH + and isinstance(obj, TORCH.Tensor) + and obj.device.type == "cuda" + ) + or isinstance(obj, (list, tuple)) + ) + ): + self.__insert_object_into_cache( + logger, + in_cache_queue, + out_cache_queue, + obj, + f_name, + parameter, + user_function, + ) + + def __insert_numpy_cache( + self, obj, f_name, parameter, function, in_cache_queue + ): + emit_manual_event_explicit( + TRACING_WORKER.serialization_cache_size_type, 0 + ) + shape = obj.shape + d_type = obj.dtype + size = obj.nbytes + new_cache_id = None + + if size > 0: + with EventInsideWorker( + TRACING_WORKER.insert_object_into_cache_event + ): + # This line takes most of the time to put into cache + shm = self.shared_memory_manager.SharedMemory(size=size) + within_cache = NP.ndarray(shape, dtype=d_type, buffer=shm.buf) + within_cache[:] = obj[:] # Copy contents + new_cache_id = shm.name + message = CacheQueueMessage( + action="PUT", + messages=[ + f_name, + new_cache_id, + SHARED_MEMORY_TAG, + parameter, + function, + ], + size=size, + d_type=d_type, + shape=shape, + ) + in_cache_queue.put(message) + + return size, new_cache_id + + def __insert_cupy_cache( + self, + obj, + f_name, + parameter, + function, + shared_tag, + in_cache_queue: Queue, + out_cache_queue: Queue, + ): + """Insert cupy array into cache. + + :param f_name: File name that corresponds to the object (used as id). + :param parameter: Parameter name. + :param function: Function. + :param shared_tag: Tag of the object (CUPY or TORCH) + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :return: Size and identifier. + """ + emit_manual_event_explicit( + TRACING_WORKER.serialization_cache_size_type, 0 + ) + shape = obj.shape + d_type = obj.dtype + size = obj.nbytes + new_cache_id = None + + if size > 0: + with EventInsideWorker( + TRACING_WORKER.insert_object_into_gpu_cache_event + ): + ipc_handle = CP.cuda.runtime.ipcGetMemHandle(obj.data.ptr) + + new_cache_id = base64.b64encode(ipc_handle) + message = CacheQueueMessage( + action="PUT_GPU", + messages=[ + f_name, + new_cache_id.decode("ascii"), + shared_tag, + parameter, + function, + obj.device.pci_bus_id, + ], + size=size, + d_type=d_type, + shape=shape, + ) + in_cache_queue.put(message) + out_cache_queue.get() + return size, new_cache_id + + def __insert_iterable_cache( + self, + obj: Union[list, tuple], + f_name, + parameter, + function, + in_cache_queue, + type, + ): + """Insert iterable object into cache. + + :param f_name: File name that corresponds to the object (used as id). + :param parameter: Parameter name. + :param function: Function. + :param in_cache_queue: Cache notification input queue. + :param type: Iterable type (can be list or tuple). + :return: Size and identifier. + """ + size = total_sizeof(obj) + new_cache_id = None + + if size > 0: + with EventInsideWorker( + TRACING_WORKER.insert_object_into_cache_event + ): + emit_manual_event_explicit( + TRACING_WORKER.serialization_cache_size_type, 0 + ) + shareable_list = self.shared_memory_manager.ShareableList( + obj + ) # noqa + new_cache_id = shareable_list.shm.name + if isinstance(obj, list): + tag = SHAREABLE_LIST_TAG + else: + tag = SHAREABLE_TUPLE_TAG + message = CacheQueueMessage( + action="PUT", + messages=[ + f_name, + new_cache_id, + tag, + parameter, + function, + ], + size=size, + d_type=type, + shape=(), # only used with numpy + ) + in_cache_queue.put(message) + + return size, new_cache_id + + def __insert_object_into_cache( + self, + logger: logging.Logger, + in_cache_queue: Queue, + out_cache_queue: Queue, + obj: typing.Any, + f_name: str, + parameter: str, + user_function: typing.Callable, + ) -> None: + """Put an object into cache. + + :param logger: Logger where to push messages. + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :param obj: Object to store. + :param f_name: File name that corresponds to the object (used as id). + :param parameter: Parameter name. + :param user_function: Function. + :return: None. + """ + __set_cuda_libs__() + + function = __function_clean__(user_function) + f_name = get_file_name(f_name) + # Exclusion in locking to avoid multiple insertions + # If no lock is defined may lead to unstable behaviour. + if self.lock is not None: + self.lock.acquire() + message = CacheQueueMessage(action="IS_LOCKED", messages=[f_name]) + in_cache_queue.put(message) + is_locked = out_cache_queue.get() + message = CacheQueueMessage(action="IS_IN_CACHE", messages=[f_name]) + in_cache_queue.put(message) + is_in_cache = out_cache_queue.get() + if not is_locked and not is_in_cache: + message = CacheQueueMessage(action="LOCK", messages=[f_name]) + in_cache_queue.put(message) + if self.lock is not None: + self.lock.release() + if is_locked: + if __debug__: + logger.debug( + "%s Not inserting into cache due to it is being " + "inserted by other process: %s", + self.header, + str(f_name), + ) + elif is_in_cache: + if __debug__: + logger.debug( + "%s Not inserting into cache due already exists " + "in cache: %s", + self.header, + str(f_name), + ) + else: + # Not locked and not in cache + if __debug__: + logger.debug( + "%s Inserting into cache (%s): %s", + self.header, + str(type(obj)), + str(f_name), + ) + try: + inserted = True + size = 0 + if isinstance(obj, NP.ndarray): + size, new_cache_id = self.__insert_numpy_cache( + obj, f_name, parameter, function, in_cache_queue + ) + elif TORCH and isinstance(obj, TORCH.Tensor) and CP: + cupy_tensor = CP.asarray(obj) + shared_tag = SHARED_MEMORY_TORCH_TAG + size, new_cache_id = self.__insert_cupy_cache( + cupy_tensor, + f_name, + parameter, + function, + shared_tag, + in_cache_queue, + out_cache_queue, + ) + elif CP and isinstance(obj, CP.ndarray): + shared_tag = SHARED_MEMORY_CUPY_TAG + size, new_cache_id = self.__insert_cupy_cache( + obj, + f_name, + parameter, + function, + shared_tag, + in_cache_queue, + out_cache_queue, + ) + elif isinstance(obj, list) or isinstance(obj, tuple): + size, new_cache_id = self.__insert_iterable_cache( + obj, + f_name, + parameter, + function, + in_cache_queue, + type(obj), + ) + else: + inserted = False + if __debug__: + logger.debug( + "%s Can not put into cache: " + "Not a [NP.ndarray | list | tuple ] object", + self.header, + ) + + if size == 0: + inserted = False + + if inserted: + emit_manual_event_explicit( + TRACING_WORKER.serialization_cache_size_type, + size, + ) + if __debug__ and inserted: + logger.debug( + "%s Inserted into cache: %s as %s", + self.header, + str(f_name), + str(new_cache_id), + ) + except KeyError as key_error: # noqa + if __debug__: + logger.debug( + "%s Can not put into cache. It may be a " + "[NP.ndarray | list | tuple ] object containing " + "an unsupported type", + self.header, + ) + logger.debug(str(key_error)) + message = CacheQueueMessage(action="UNLOCK", messages=[f_name]) + in_cache_queue.put(message) + + def __remove_object_from_cache( + self, + logger: logging.Logger, + in_cache_queue: Queue, + out_cache_queue: Queue, + f_name: str, + ) -> None: + """Remove an object from cache. + + :param logger: Logger where to push messages. + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :param f_name: File name that corresponds to the object (used as id). + :return: None. + """ + with EventInsideWorker(TRACING_WORKER.remove_object_from_cache_event): + f_name = get_file_name(f_name) + if __debug__: + logger.debug( + "%s Removing from cache: %s", self.header, str(f_name) + ) + message = CacheQueueMessage(action="REMOVE", messages=[f_name]) + in_cache_queue.put(message) + if __debug__: + logger.debug( + "%s Removed from cache: %s", self.header, str(f_name) + ) + + def replace_object_into_cache( + self, + logger: logging.Logger, + in_cache_queue: Queue, + out_cache_queue: Queue, + obj: typing.Any, + f_name: str, + parameter: str, + user_function: typing.Callable, + ) -> None: + """Put an object into cache. + + :param logger: Logger where to push messages. + :param in_cache_queue: Cache notification input queue. + :param out_cache_queue: Cache notification output queue. + :param obj: Object to store. + :param f_name: File name that corresponds to the object (used as id). + :param parameter: Parameter name. + :param user_function: Function. + :return: None. + """ + f_name = get_file_name(f_name) + if __debug__: + logger.debug( + "%s Replacing from cache: %s", self.header, str(f_name) + ) + self.__remove_object_from_cache( + logger, in_cache_queue, out_cache_queue, f_name + ) + self.__insert_object_into_cache( + logger, + in_cache_queue, + out_cache_queue, + obj, + f_name, + parameter, + user_function, + ) + if __debug__: + logger.debug( + "%s Replaced from cache: %s", self.header, str(f_name) + ) + + def in_cache( + self, logger: logging.Logger, f_name: str, cache: typing.Any + ) -> bool: + """Check if the given file name is in the cache. + + :param logger: Logger where to push messages. + :param f_name: Absolute file name. + :param cache: Proxy dictionary cache. + :return: True if in. False otherwise. + """ + # It can be checked if it is locked and wait for it to be unlocked + if cache: + f_name = get_file_name(f_name) + if __debug__: + logger.debug("%s Is in cache? %s", self.header, str(f_name)) + + if f_name in cache: + obj_id, _, _, _, _, shared_type = cache[f_name] + + if shared_type == SHARED_MEMORY_CUPY_TAG: + with EventInsideWorker( + TRACING_WORKER.check_access_gpu_event + ): + is_in_gpu = self.__check_cupy_access__(logger, obj_id) + if is_in_gpu: + event = EventInsideWorker( + TRACING_WORKER.cache_hit_gpu_event + ) + else: + event = EventInsideWorker( + TRACING_WORKER.cache_miss_gpu_event + ) + with event: + return is_in_gpu + return True + + return False + + def __check_cupy_access__(self, logger, obj_id): + """Check if object accessible by cupy. + + Check if current GPU has access to the memory of the gpu where the + handle is pointing. + """ + if obj_id in self.cupy_handlers: + return True + + import cupy + from cupy_backends.cuda.api.runtime import CUDARuntimeError + + try: + handler = base64.b64decode(obj_id) + array_open = cupy.cuda.runtime.ipcOpenMemHandle(handler) + cupy.cuda.runtime.ipcCloseMemHandle(array_open) + return True + except CUDARuntimeError as e: + logger.debug( + "%s CHECK CUPY ACCESS CUDARuntimeError %s", self.header, str(e) + ) + return False + + +CACHE_TRACKER = CacheTracker() + + +def get_file_name(f_name: str) -> str: + """Convert a full path with file name to the file name (removes the path). + + Example: /a/b/c.py -> c.py + + :param f_name: Absolute file name path. + :return: File name. + """ + return os.path.basename(f_name) + + +def get_file_name_clean(f_name: str) -> str: + """Retrieve filename given the absolute path. + + :param f_name: Absolute file path. + :return: File name. + """ + return f_name.rsplit("/", 1)[-1] + + +def __function_clean__(function: typing.Callable) -> str: + """Retrieve the clean function name. + + :param function: Function. + :return: Function name. + """ + return str(function)[10:].rsplit(" ", 3)[0] diff --git a/examples/dds/pycompss/worker/piper/commons/__init__.py b/examples/dds/pycompss/worker/piper/commons/__init__.py new file mode 100644 index 00000000..db5257dc --- /dev/null +++ b/examples/dds/pycompss/worker/piper/commons/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +This package contains the piper worker common helpers. + +Where helpers are functions, constants and classes. +""" diff --git a/examples/dds/pycompss/worker/piper/commons/constants.py b/examples/dds/pycompss/worker/piper/commons/constants.py new file mode 100644 index 00000000..997f99ea --- /dev/null +++ b/examples/dds/pycompss/worker/piper/commons/constants.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Commons - Constants. + +This file contains a set of constants used when communication through pipes. + +--------------------- +TAGS EXPECTED FORMAT: +--------------------- +- EXECUTE_TASK_TAG........... "task" taskId jobOut jobErr task_params +- END_TASK_TAG............... "endTask" taskId endStatus +- CANCEL_TASK_TAG............ "cancelTask" in_pipe out_pipe +- COMPSS_EXCEPTION_TAG....... "compssException" taskId exception_message +- ERROR_TASK_TAG............. TBD +- ERROR_TAG.................. "error" [MESSAGE EXPECTED] +- PING_TAG................... "ping" +- PONG_TAG................... "pong" +- ADD_EXECUTOR_TAG........... "addExecutor" in_pipe out_pipe +- ADD_EXECUTOR_FAILED_TAG.... "addExecutorFailed" +- ADDED_EXECUTOR_TAG......... "addedExecutor" +- QUERY_EXECUTOR_ID_TAG...... "query" in_pipe out_pipe +- REPLY_EXECUTOR_ID_TAG...... "reply" executor_id +- REMOVE_EXECUTOR_TAG........ "removeExecutor" in_pipe out_pipe +- REMOVED_EXECUTOR_TAG....... "removedExecutor" +- QUIT_TAG................... "quit" +- REMOVE_TAG................. TBD +- SERIALIZE_TAG.............. TBD +""" + + +class _Tags: + """Currently supported tags in piper workers.""" + + __slots__ = ( + "execute_task", + "end_task", + "cancel_task", + "compss_exception", + "error_task", + "error", + "ping", + "pong", + "add_executor", + "add_executor_failed", + "added_executor", + "query_executor_id", + "reply_executor_id", + "remove_executor", + "removed_executor", + "quit", + "remove", + "serialize", + ) + + def __init__(self) -> None: # pylint: disable=too-many-statements + """Define supported Tags.""" + self.execute_task = "EXECUTE_TASK" + self.end_task = "END_TASK" + self.cancel_task = "CANCEL_TASK" + self.compss_exception = "COMPSS_EXCEPTION" + self.error_task = "ERROR_TASK" + self.error = "ERROR" + self.ping = "PING" + self.pong = "PONG" + self.add_executor = "ADD_EXECUTOR" + self.add_executor_failed = "ADD_EXECUTOR_FAILED" + self.added_executor = "ADDED_EXECUTOR" + self.query_executor_id = "QUERY_EXECUTOR_ID" + self.reply_executor_id = "REPLY_EXECUTOR_ID" + self.remove_executor = "REMOVE_EXECUTOR" + self.removed_executor = "REMOVED_EXECUTOR" + self.quit = "QUIT" + self.remove = "REMOVE" + self.serialize = "SERIALIZE" + + +TAGS = _Tags() + +# ################# # +# Other variables # +# ################# # +HEADER = "[PYTHON WORKER] " diff --git a/examples/dds/pycompss/worker/piper/commons/executor.py b/examples/dds/pycompss/worker/piper/commons/executor.py new file mode 100644 index 00000000..ec60f05f --- /dev/null +++ b/examples/dds/pycompss/worker/piper/commons/executor.py @@ -0,0 +1,1167 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Commons - Executor. + +This file contains the code of an executor running the commands that it +reads from a pipe (for mpi and multiprocessing piped workers). +""" + +import copy +import logging +import os +import signal +import sys +import time +import traceback +import gc +import contextlib +from pycompss.util.process.manager import Queue +from pycompss.util.process.manager import DictProxy +from pycompss.runtime.management.object_tracker import OT + +from pycompss.util.typing_helper import typing + +try: + THREAD_AFFINITY = True + import process_affinity # noqa +except ImportError: + from pycompss.worker.piper.commons.constants import HEADER as MAIN_HEADER + + print( + "".join( + ( + MAIN_HEADER, + "WARNING: Could not import process affinity library: ", + "CPU AFFINITY NOT SUPPORTED!", + ) + ) + ) + THREAD_AFFINITY = False + +from pycompss.runtime.management.COMPSs import COMPSs +from pycompss.util.context import CONTEXT +from pycompss.runtime.commons import GLOBALS +from pycompss.worker.piper.commons.constants import TAGS +from pycompss.worker.piper.commons.utils_logger import load_loggers +from pycompss.worker.commons.executor import build_return_params_message +from pycompss.worker.commons.worker import execute_task +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.tracing.helpers import emit_manual_event +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.helpers import EventInsideWorker +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.worker.piper.cache.tracker import CACHE_TRACKER + +# Streaming imports +from pycompss.streams.components.distro_stream_client import ( + DistroStreamClientHandler, +) + +try: + from threadpoolctl import threadpool_limits + + THREADPOOLCTL_AVAILABLE = True +except ImportError: + THREADPOOLCTL_AVAILABLE = False + +COMPSS_WITH_DLB = False +if int(os.getenv("COMPSS_WITH_DLB", 0)) >= 1: + COMPSS_WITH_DLB = True + import dlb_affinity + + +HEADER = "*[PYTHON EXECUTOR] " +EARING = False +if "EAR_INITIALIZATION_TIME" in os.environ: + EAR_INITIALIZATION = int(os.environ["EAR_INITIALIZATION_TIME"]) +else: + EAR_INITIALIZATION = 40 # seconds minimum before doing any finalize. + + +def shutdown_handler( + signal: int, frame: typing.Any # pylint: disable=redefined-outer-name +) -> None: + """Handle shutdown - Shutdown handler. + + CAUTION! Do not remove the parameters. + + :param signal: shutdown signal. + :param frame: Frame. + :return: None + :raises PyCOMPSsException: Received signal. + """ + process_id = os.getpid() + process_name = os.environ["EAR_APP_NAME"] + sys.stderr.write( + f"[shutdown_handler] executor.py Received SIGTERM - " + f"PID: {process_id} " + f"PROCESS NAME: {process_name} " + f"TIMESTAMP: {time.time()}\n" + ) + sys.stderr.write(f"SIGNAL: {signal}\n") + sys.stderr.write(f"FRAME: %{str(frame)}\n") + traceback.print_stack(frame) + sys.stderr.flush() + sys.stdout.flush() + raise PyCOMPSsException("Received SIGTERM") + + +class Pipe: + """Bi-directional communication channel class.""" + + __slots__ = ["input_pipe", "input_pipe_open", "output_pipe"] + + def __init__(self, input_pipe: str = "", output_pipe: str = "") -> None: + """Construct a new Pipe. + + :param input_pipe: Input pipe for the thread. To receive messages from + the runtime. + :param output_pipe: Output pipe for the thread. To send messages to + the runtime. + """ + self.input_pipe = input_pipe + self.input_pipe_open = None # type: typing.Optional[typing.TextIO] + self.output_pipe = output_pipe + + def read_command(self, retry_period: float = 0.5) -> str: + """Return the first command on the pipe. + + :param retry_period: Time (ms) that the thread sleeps if EOF is read + from pipe. + :return: The first command available on the pipe. + """ + if self.input_pipe == "": + raise PyCOMPSsException("Undefined input pipe in Pipe object") + if self.input_pipe_open is None: + self.input_pipe_open = open( # pylint: disable=consider-using-with + self.input_pipe, "r" + ) + # Non blocking open: + # fd = os.open(self.input_pipe, os.O_RDWR) + # self.input_pipe_open = os.fdopen(fd, "r") + + line = self.input_pipe_open.readline() + if line == "": + time.sleep(0.001 * retry_period) + line = self.input_pipe_open.readline() + + return line + + def write(self, message: str) -> None: + """Write a message through the pipe. + + :param message: Message sent through the pipe. + :return: None. + """ + if self.output_pipe == "": + raise PyCOMPSsException("Undefined output pipe in Pipe object") + with open(self.output_pipe, "w") as out_pipe: + out_pipe.write("".join((message, "\n"))) + + def close(self) -> None: + """Close the pipe, if open. + + :return: None. + """ + if self.input_pipe_open: + self.input_pipe_open.close() + self.input_pipe_open = None + + def __str__(self) -> str: + """Representation of the Pipe object. + + :return: String representing the Pipe object. + """ + return " ".join( + ("PIPE IN", self.input_pipe, "PIPE OUT", self.output_pipe) + ) + + +class ExecutorConf: + """Executor configuration class.""" + + __slots__ = [ + "debug", + "tmp_dir", + "tracing", + "storage_conf", + "logger", + "persistent_storage", + "storage_loggers", + "stream_backend", + "stream_master_ip", + "stream_master_port", + "cache_ids", + "in_cache_queue", + "out_cache_queue", + "cache_profiler", + "ear", + ] + + def __init__( + self, + debug: bool, + tmp_dir: str, + tracing: bool, + storage_conf: str, + logger: logging.Logger, + persistent_storage: bool, + storage_loggers: typing.List[logging.Logger], + stream_backend: str, + stream_master_ip: str, + stream_master_port: str, + cache_ids: typing.Optional[DictProxy] = None, + in_cache_queue: typing.Optional[Queue] = None, + out_cache_queue: typing.Optional[Queue] = None, + cache_profiler: bool = False, + ear: bool = False, + ) -> None: + """Construct a new executor configuration. + + :param debug: If debug is enabled. + :param tmp_dir: Temporary directory for logging purposes. + :param tracing: Enable tracing for the executor. + :param storage_conf: Storage configuration file. + :param logger: Main logger. + :param persistent_storage: If persistent storage is enabled + :param storage_loggers: List of supported storage loggers + (empty if running w/o storage). + :param stream_backend: Streaming backend type. + :param stream_master_ip: Streaming master IP. + :param stream_master_port: Streaming master port. + :param cache_ids: Proxy cache dictionary. + :param in_cache_queue: Cache queue where to submit to add new entries + to cache_ids. + :param out_cache_queue: Cache queue where to the cache returns info. + :param ear: Ear energy metering. + """ + self.debug = debug + self.tmp_dir = tmp_dir + self.tracing = tracing + self.storage_conf = storage_conf + self.logger = logger + self.persistent_storage = persistent_storage + self.storage_loggers = storage_loggers + self.stream_backend = stream_backend + self.stream_master_ip = stream_master_ip + self.stream_master_port = stream_master_port + self.cache_ids = cache_ids # Read-only + self.in_cache_queue = in_cache_queue + self.out_cache_queue = out_cache_queue + self.cache_profiler = cache_profiler + self.ear = ear + + +###################### +# Processes body +###################### + + +def executor( + lock: typing.Any, + queue: typing.Union[None, Queue], + event: typing.Any, + process_id: int, + process_name: str, + pipe: Pipe, + conf: ExecutorConf, +) -> None: + """Thread main body - Overrides Threading run method. + + Iterates over the input pipe in order to receive tasks (with their + parameters) and process them. + Notifies the runtime when each task has finished with the + corresponding output value. + Finishes when the "quit" message is received. + + :param lock: Lock to ensure mutual exclusion. + :param queue: Queue where to put exception messages. + :param process_id: Process identifier (number that matches the java + processes). + :param process_name: Process name (Thread-X, where X is the thread id). + :param pipe: Pipe to receive and send messages from/to the runtime. + :param conf: Executor configuration. + :return: None. + """ + global EARING + start_time = time.time() + + try: + # First thing to do is to emit the process identifier event + emit_manual_event_explicit( + TRACING_WORKER.process_identifier, + TRACING_WORKER.process_worker_executor_event, + ) + # Second thing to do is to emit the executor process identifier event + emit_manual_event_explicit( + TRACING_WORKER.executor_identifier, process_id + ) + + if COMPSS_WITH_DLB: + dlb_affinity.init() + dlb_affinity.setaffinity([], os.getpid()) + dlb_affinity.lend() + + # Replace Python Worker's SIGTERM handler. + signal.signal(signal.SIGTERM, shutdown_handler) + + if len(conf.logger.handlers) == 0: + # Logger has not been inherited correctly. Happens in MacOS. + tmp_dir = os.path.join(conf.tmp_dir, "..") + GLOBALS.set_temporary_directory(tmp_dir) + # Reload logger + ( + conf.logger, + conf.storage_loggers, + _, + ) = load_loggers(conf.debug, conf.persistent_storage) + # Set the binding in worker mode too + CONTEXT.set_worker() + logger = conf.logger + + tracing = conf.tracing + storage_conf = conf.storage_conf + storage_loggers = conf.storage_loggers + + # Get a copy of the necessary information from the logger to + # re-establish after each task + logger_handlers = copy.copy(logger.handlers) + logger_level = logger.getEffectiveLevel() + _fmt = logger_handlers[0].formatter._fmt # type: ignore # pylint:W0212 + # disable=protected-access + logger_formatter = logging.Formatter(_fmt) + storage_loggers_handlers = [] + for storage_logger in storage_loggers: + storage_loggers_handlers.append(copy.copy(storage_logger.handlers)) + + # Establish link with the binding-commons to enable task nesting + if __debug__: + logger.debug( + HEADER + + "Establishing link with runtime in process " + + str(process_name) + ) + COMPSs.load_runtime(external_process=False, _logger=logger) + COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe) + + if storage_conf != "null": + try: + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + initWorkerPostFork, + ) + + with EventWorker(TRACING_WORKER.init_worker_postfork_event): + initWorkerPostFork() + except (ImportError, AttributeError): + if __debug__: + logger.info( + "%s[%s] Could not find initWorkerPostFork " + "storage call. Ignoring it.", + HEADER, + str(process_name), + ) + + # Start the streaming backend if necessary + streaming = False + if conf.stream_backend not in [None, "null", "NONE"]: + streaming = True + + if streaming: + # Initialize streaming + if __debug__: + logger.debug( + "%s[%s] Starting streaming for process", + HEADER, + str(process_name), + ) + try: + DistroStreamClientHandler.init_and_start( + master_ip=conf.stream_master_ip, + master_port=conf.stream_master_port, + ) + except ( + Exception + ) as general_exception: # pylint: disable=broad-except + logger.error(general_exception) + raise general_exception from general_exception + + # Load ear if necessary + if conf.ear: + # Initialize ear + if __debug__: + logger.debug( + "%s[%s] Loading ear", + HEADER, + str(process_name), + ) + with EventWorker(TRACING_WORKER.executor_load_ear_event): + import ear + EARING = True + + # Connect to Shared memory manager + if conf.in_cache_queue and conf.out_cache_queue: + CACHE_TRACKER.set_lock(lock) + CACHE_TRACKER.connect_to_shared_memory_manager() + + # Process properties + alive = True + + if __debug__: + logger.debug("%s[%s] Starting process", HEADER, str(process_name)) + + # MAIN EXECUTOR LOOP + while alive and not event.is_set(): + # Runtime -> pipe - Read command from pipe + command = COMPSs.read_pipes() + if command != "": + if __debug__: + logger.debug( + "%s[%s] Received command %s", + HEADER, + str(process_name), + str(command), + ) + # Process the command + alive = process_message( + command, + process_name, + pipe, + queue, + tracing, + logger, + logger_handlers, + logger_level, + logger_formatter, + storage_conf, + storage_loggers, + storage_loggers_handlers, + conf.in_cache_queue, + conf.out_cache_queue, + conf.cache_ids, + conf.cache_profiler, + ) + if __debug__: + logger.debug( + "%s[%s] Ending process (alive: %s - event: %s)", + HEADER, + str(process_name), + str(alive), + str(event.is_set()), + ) + # Stop storage + if storage_conf != "null": + try: + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + finishWorkerPostFork, + ) + + with EventWorker(TRACING_WORKER.finish_worker_postfork_event): + finishWorkerPostFork() + except (ImportError, AttributeError): + if __debug__: + logger.info( + "%s[%s] Could not find finishWorkerPostFork " + "storage call. Ignoring it.", + HEADER, + str(process_name), + ) + + # Stop streaming + if streaming: + if __debug__: + logger.debug( + "%s Stopping streaming for process ", + HEADER, + str(process_name), + ) + DistroStreamClientHandler.set_stop() + + sys.stdout.flush() + sys.stderr.flush() + if EARING: + with EventWorker(TRACING_WORKER.executor_finalize_ear_event): + elapsed_time = time.time() - start_time + if __debug__: + logger.debug( + "%s[%s] Stopping EAR (elapsed %s)", + HEADER, + str(process_name), + str(elapsed_time), + ) + if elapsed_time < EAR_INITIALIZATION: + logger.debug( + "%s[%s] Waiting to finalize EAR: %s seconds", + HEADER, + str(process_name), + str(elapsed_time), + ) + time.sleep(elapsed_time) + ear.finalize() + EARING = False + if __debug__: + logger.debug( + "%s[%s] Stopped EAR at %s", + HEADER, + str(process_name), + str(time.time()), + ) + if __debug__: + logger.debug("%s[%s] Exiting process ", HEADER, str(process_name)) + # Send quit message back to the runtime + pipe.write(TAGS.quit) + pipe.close() + except Exception as general_exception: # pylint: disable=broad-except + sys.stderr.write("\nGENERAL EXCEPTION:\n") + sys.stderr.write(f"\n{str(general_exception)}\n") + sys.stderr.flush() + raise general_exception from general_exception + + +def process_message( + current_line: str, + process_name: str, + pipe: Pipe, + queue: typing.Optional[Queue], + tracing: bool, + logger: logging.Logger, + logger_handlers: list, + logger_level: int, + logger_formatter: typing.Any, + storage_conf: str, + storage_loggers: typing.List[logging.Logger], + storage_loggers_handlers: list, + in_cache_queue: typing.Optional[Queue] = None, + out_cache_queue: typing.Optional[Queue] = None, + cache_ids: typing.Any = None, + cache_profiler: bool = False, +) -> bool: + """Process command received from the runtime through a pipe. + + :param current_line: Current command (line) to process. + :param process_name: Process name for logger messages. + :param pipe: Pipe where to write the result. + :param queue: Queue where to drop the process exceptions. + :param tracing: Tracing. + :param logger: Logger. + :param logger_handlers: Logger handlers. + :param logger_level: Logger level. + :param logger_formatter: Logger formatter. + :param storage_conf: Storage configuration. + :param storage_loggers: Storage loggers. + :param storage_loggers_handlers: Storage loggers handlers. + :param in_cache_queue: Cache tracker input communication queue. + :param out_cache_queue: Cache tracker output communication queue. + :param cache_ids: Cache proxy dictionary (read-only). + :param cache_profiler: Cache profiler. + :return: True if processed successfully, False otherwise. + """ + if __debug__: + logger.debug( + "%s[%s] Processing message: %s", + HEADER, + str(process_name), + str(current_line), + ) + + current_line_split = current_line.split() + if current_line_split[0] == TAGS.execute_task: + # Process task + return process_task( + current_line_split, + process_name, + pipe, + queue, + tracing, + logger, + logger_handlers, + logger_level, + logger_formatter, + storage_conf, + storage_loggers, + storage_loggers_handlers, + in_cache_queue, + out_cache_queue, + cache_ids, + cache_profiler, + ) + + if current_line_split[0] == TAGS.ping: + # Response -> Pong + return process_ping(pipe, logger, process_name) + + if current_line_split[0] == TAGS.quit: + # Received quit message -> Suicide + return process_quit(logger, process_name) + + if __debug__: + logger.debug( + "%s[%s] Unexpected message: %s", + HEADER, + str(process_name), + str(current_line_split), + ) + raise PyCOMPSsException(f"Unexpected message: {str(current_line_split)}") + + +def process_task( + current_line: typing.List[str], + process_name: str, + pipe: Pipe, + queue: typing.Optional[Queue], + tracing: bool, + logger: logging.Logger, + logger_handlers: list, + logger_level: int, + logger_formatter: typing.Any, + storage_conf: str, + storage_loggers: typing.List[logging.Logger], + storage_loggers_handlers: list, + in_cache_queue: typing.Optional[Queue], + out_cache_queue: typing.Optional[Queue], + cache_ids: typing.Any, + cache_profiler: bool, +) -> bool: + """Process command received from the runtime through a pipe. + + :param current_line: Current command (line) to process. + :param process_name: Process name for logger messages. + :param pipe: Pipe where to write the result. + :param queue: Queue where to drop the process exceptions. + :param tracing: Tracing. + :param logger: Logger. + :param logger_handlers: Logger handlers. + :param logger_level: Logger level. + :param logger_formatter: Logger formatter. + :param storage_conf: Storage configuration. + :param storage_loggers: Storage loggers. + :param storage_loggers_handlers: Storage loggers handlers. + :param in_cache_queue: Cache tracker input communication queue. + :param out_cache_queue: Cache tracker output communication queue. + :param cache_ids: Cache proxy dictionary (read-only). + :param cache_profiler: Cache profiler. + :return: True if processed successfully, False otherwise. + """ + with EventWorker(TRACING_WORKER.process_task_event): + affinity_event_emit = False + binded_cpus = False + binded_gpus = False + current_working_dir = os.getcwd() + + # CPU binding + cpus = current_line[-3] + if cpus != "-" and THREAD_AFFINITY: + # The cpu affinity event is already emitted in Java. + # Instead of emitting what we receive, we are emitting what we + # check after setting the affinity. + binded_cpus = bind_cpus(cpus, process_name, logger) + + # GPU binding + gpus = current_line[-2] + if gpus != "-": + # Emit a mask event to identify which gpu has been assigned + # (add always 1 since 0 can not be emitted). + # Single gpu assigned: + # - 0 = gpu 1 assigned => 0001 == 1 (decimal) + # - 1 = gpu 2 assigned => 0010 == 2 (decimal) + # - 2 = gpu 3 assigned => 0100 == 4 (decimal) + # - 3 = gpu 4 assigned => 1000 == 8 (decimal) + # Multiple gpu assigned: + # - 0,1 = gpu 1 and 2 assigned => 0011 == 3 (decimal) + # - 2,3 = gpu 3 and 4 assigned => 1100 == 12 (decimal) + # - 0,1,3,4 = gpu 1, 2,3 and 4 assigned => 1111 == 15 (decimal) + gpu_mask_list = [0] * 4 + if "," in gpus: + # There are more than one gpus assigned. + gpus_list = gpus.split(",") + for i in range(len(gpus_list)): + gpu_mask_list[int(i)] = 1 + else: + # It is a single gpu where gpus identify which gpu has been + # assigned (0, 1, 2, or 3). + gpu_mask_list[int(gpus)] = 1 + gpu_mask_list.reverse() + gpu_mask = "".join(map(str, gpu_mask_list)) + assigned_gpus = int(gpu_mask, 2) # convert base 2 to decimal + emit_manual_event(assigned_gpus, inside=True, gpu_affinity=True) + bind_gpus(gpus, process_name, logger) + binded_gpus = True + + # Remove the last elements: cpu and gpu bindings + current_line = current_line[0:-3] + + # task jobId command + job_id, working_dir, job_out, job_err = current_line[ + 1:5 + ] # 5th is not taken + # current_line[5] = = tracing + # current_line[6] = = task id + # current_line[7] = = debug + # current_line[8] = = storage conf. + # current_line[9] = = operation type (e.g. METHOD) + # current_line[10] = = module + # current_line[11]= = method + # current_line[12]= = time out + # current_line[13]= = Number of slaves (worker nodes)==#nodes + # <> + # current_line[14 + #nodes] = = computing units + # current_line[15 + #nodes] = = has target + # current_line[16 + #nodes] = = has return (always "null") + # current_line[17 + #nodes] = = Number of parameters + # <> + # !---> type, stream, prefix , value + + # Setting working directory + os.chdir(working_dir) + GLOBALS.set_temporary_directory(working_dir) + + if __debug__: + logger.debug( + "%s[%s] Received task with id: %s", + HEADER, + str(process_name), + str(job_id), + ) + logger.debug( + "%s[%s] Setting working directory: %s", + HEADER, + str(process_name), + str(working_dir), + ) + logger.debug( + "%s[%s] - TASK CMD: %s", + HEADER, + str(process_name), + str(current_line), + ) + + # Swap logger from stream handler to file handler + # All task output will be redirected to job.out/err + for log_handler in logger_handlers: + logger.removeHandler(log_handler) + for storage_logger in storage_loggers: + for log_handler in storage_logger.handlers: + storage_logger.removeHandler(log_handler) + out_file_handler = logging.FileHandler(job_out) + out_file_handler.setLevel(logger_level) + out_file_handler.setFormatter(logger_formatter) + err_file_handler = logging.FileHandler(job_err) + err_file_handler.setLevel("ERROR") + err_file_handler.setFormatter(logger_formatter) + logger.addHandler(out_file_handler) + logger.addHandler(err_file_handler) + for storage_logger in storage_loggers: + storage_logger.addHandler(out_file_handler) + storage_logger.addHandler(err_file_handler) + + if __debug__: + # From now onwards the log is in the job out and err files + logger.debug("-" * 100) + logger.debug("Received task in process: %s", str(process_name)) + logger.debug("TASK CMD: %s", str(current_line)) + + try: + # Check thread affinity + if THREAD_AFFINITY: + # The cpu affinity can be long if multiple cores have been + # assigned. To avoid issues, we get just the first id. + real_affinity = process_affinity.getaffinity() + cpus = str(real_affinity[0]) + num_cpus = len(real_affinity) + emit_manual_event( + int(cpus) + 1, inside=True, cpu_affinity=True + ) + emit_manual_event(int(num_cpus), inside=True, cpu_number=True) + affinity_event_emit = True + if not binded_cpus: + logger.warning( + "This task is going to be executed with default " + "thread affinity %s", + str(real_affinity), + ) + + # Setup process environment + compss_procs = int(current_line[13]) + compss_nodes = int(current_line[14]) + compss_nodes_names = ",".join( + current_line[15 : 15 + compss_nodes] # noqa: E203 + ) + computing_units = current_line[15 + compss_nodes] + if __debug__: + logger.debug("Process environment:") + logger.debug("\t - Number of nodes: %s", (str(compss_nodes))) + logger.debug("\t - Hostnames: %s", str(compss_nodes_names)) + logger.debug( + "\t - Number of threads: %s", (str(computing_units)) + ) + setup_environment( + compss_nodes, compss_procs, compss_nodes_names, computing_units + ) + + # Clean object tracker + OT.clean_object_tracker(hard_stop=False) + + if not COMPSS_WITH_DLB and THREADPOOLCTL_AVAILABLE: + thread_context = threadpool_limits(limits=int(computing_units)) + else: + thread_context = contextlib.nullcontext() + + with thread_context: + # Execute task + result = execute_task( + process_name, + storage_conf, + current_line[10:], + tracing, + logger, + (job_out, job_err), + False, + {}, + in_cache_queue, + out_cache_queue, + cache_ids, + cache_profiler, + ) + + # The ignored variable is timed_out + exit_value, new_types, new_values, _, except_msg = result + + if COMPSS_WITH_DLB: + dlb_affinity.setaffinity([], os.getpid()) + dlb_affinity.lend() + + if exit_value == 0: + # Task has finished without exceptions + # endTask jobId exitValue message + message = build_successful_message( + new_types, new_values, job_id, exit_value + ) + if __debug__: + logger.debug( + "%s - Pipe %s END TASK MESSAGE: %s", + str(process_name), + str(pipe.output_pipe), + str(message), + ) + elif exit_value == 2: + # Task has finished with a COMPSs Exception + # compssExceptionTask jobId exitValue message + except_msg, message = build_compss_exception_message( + except_msg, job_id + ) + if __debug__: + logger.debug( + "%s - Pipe %s COMPSS EXCEPTION TASK MESSAGE: %s", + str(process_name), + str(pipe.output_pipe), + str(except_msg), + ) + else: + # An exception other than COMPSsException has been raised + # within the task + message = build_exception_message(job_id, exit_value) + if __debug__: + logger.debug( + "%s - Pipe %s END TASK MESSAGE: %s", + str(process_name), + str(pipe.output_pipe), + str(message), + ) + + # The return message is: + # + # TaskResult ==> jobId exitValue D List + # + # Where List has D * 2 length: + # D = #parameters == #task_parameters + + # (has_target ? 1 : 0) + + # #returns + # And contains a pair of elements per parameter: + # - Parameter new type. + # - Parameter new value: + # - "null" if it is NOT a PSCO + # - PSCOId (String) if is a PSCO + # Example: + # 4 null 9 null 12 + # + # The order of the elements is: parameters + self + returns + # + # This is sent through the pipe with the END_TASK message. + # If the task had an object or file as parameter and the worker + # returns the id, the runtime can change the type (and locations) + # to a EXTERNAL_OBJ_T. + + except Exception as general_exception: # pylint: disable=broad-except + logger.exception( + "%s - Exception %s", str(process_name), str(general_exception) + ) + if queue: + queue.put("EXCEPTION") + # Clean object tracker + OT.clean_object_tracker(hard_stop=True) + # Go back to initial current working directory + os.chdir(current_working_dir) + GLOBALS.set_temporary_directory(current_working_dir) + # Stop the worker process + return False + + # Clean environment variables + if __debug__: + logger.debug("Cleaning environment.") + clean_environment(binded_cpus, binded_gpus) + if affinity_event_emit: + emit_manual_event(0, inside=True, cpu_affinity=True) + emit_manual_event(0, inside=True, cpu_number=True) + if binded_gpus: + emit_manual_event(0, inside=True, gpu_affinity=True) + + # Restore loggers + if __debug__: + logger.debug("Restoring loggers.") + logger.debug("-" * 100) + # No more logs in job out and err files + # Restore worker log + logger.removeHandler(out_file_handler) + logger.removeHandler(err_file_handler) + logger.handlers = [] + for handler in logger_handlers: + logger.addHandler(handler) + i = 0 + for storage_logger in storage_loggers: + storage_logger.removeHandler(out_file_handler) + storage_logger.removeHandler(err_file_handler) + storage_logger.handlers = [] + for handler in storage_loggers_handlers[i]: + storage_logger.addHandler(handler) + i += 1 + + with EventInsideWorker(TRACING_WORKER.cleanup_task_event): + gc.collect() + + if __debug__: + logger.debug( + "%s[%s] Finished task with id: %s", + HEADER, + str(process_name), + str(job_id), + ) + + # Clean object tracker + OT.clean_object_tracker(hard_stop=True) + # Notify the runtime that the task has finished + pipe.write(message) + # Go back to original working directory + os.chdir(current_working_dir) + GLOBALS.set_temporary_directory(current_working_dir) + return True + + +def process_ping( + pipe: Pipe, logger: logging.Logger, process_name: str +) -> bool: + """Process ping message. + + Response: Pong. + + :param pipe: Where to write the ping response. + :param logger: Logger. + :param process_name: Process name. + :return: True if success. False otherwise. + """ + with EventWorker(TRACING_WORKER.process_ping_event): + if __debug__: + logger.debug("%s[%s] Received ping.", HEADER, str(process_name)) + try: + pipe.write(TAGS.pong) + except Exception: # pylint: disable=broad-except + return False + return True + + +def process_quit(logger: logging.Logger, process_name: str) -> bool: + """Process quit message. + + Response: False. + + :param logger: Logger. + :param process_name: Process name. + :return: Always false. + """ + with EventWorker(TRACING_WORKER.process_quit_event): + if __debug__: + logger.debug("%s[%s] Received quit.", HEADER, str(process_name)) + return False + + +def bind_cpus(cpus: str, process_name: str, logger: logging.Logger) -> bool: + """Bind the given CPUs for core affinity to this process. + + :param cpus: Target CPUs. + :param process_name: Process name for logger messages. + :param logger: Logger. + :return: True if success, False otherwise. + """ + import traceback + + with EventInsideWorker(TRACING_WORKER.bind_cpus_event): + if __debug__: + logger.debug( + "%s[%s] Assigning affinity %s", + HEADER, + str(process_name), + str(cpus), + ) + cpus_list = cpus.split(",") + cpus_map = list(map(int, cpus_list)) + try: + if COMPSS_WITH_DLB: + dlb_affinity.setaffinity(cpus_map, os.getpid()) + else: + process_affinity.setaffinity(cpus_map) + except Exception as e: # pylint: disable=broad-except + if __debug__: + logger.error( + "%s[%s] WARNING: could not assign affinity %s", + HEADER, + str(process_name), + str(cpus_map), + ) + traceback.print_exc() + logger.error(str(e)) + return False + # Export only if success + os.environ["COMPSS_BINDED_CPUS"] = cpus + os.environ["COMPSS_NUM_CPUS"] = str(len(cpus_list)) + return True + + +def bind_gpus(gpus: str, process_name: str, logger: logging.Logger) -> None: + """Bind the given GPUs to this process. + + :param gpus: Target GPUs. + :param process_name: Process name for logger messages. + :param logger: Logger. + :return: None. + """ + with EventInsideWorker(TRACING_WORKER.bind_gpus_event): + os.environ["COMPSS_BINDED_GPUS"] = gpus + os.environ["CUDA_VISIBLE_DEVICES"] = gpus + os.environ["GPU_DEVICE_ORDINAL"] = gpus + if __debug__: + logger.debug( + "%s[%s] Assigning GPU %s", HEADER, str(process_name), str(gpus) + ) + + +def setup_environment( + compss_nodes: int, + compss_ppn: int, + compss_nodes_names: str, + computing_units: str, +) -> None: + """Set the environment (mainly environment variables). + + :param compss_nodes: Number of COMPSs nodes. + :param compss_nodes_names: COMPSs hostnames. + :param computing_units: Number of COMPSs threads. + :return: None. + """ + with EventInsideWorker(TRACING_WORKER.setup_environment_event): + os.environ["COMPSS_NUM_NODES"] = str(compss_nodes) + os.environ["COMPSS_NUM_PROCS"] = str(compss_ppn * compss_nodes) + os.environ["COMPSS_HOSTNAMES"] = compss_nodes_names + os.environ["COMPSS_NUM_THREADS"] = computing_units + os.environ["OMP_NUM_THREADS"] = computing_units + + +def build_successful_message( + new_types: list, new_values: list, job_id: str, exit_value: int +) -> str: + """Generate a successful message. + + :param new_types: New types (can change if INOUT). + :param new_values: New values (can change if INOUT). + :param job_id: Job identifier. + :param exit_value: Exit value. + :return: Successful message. + """ + with EventInsideWorker(TRACING_WORKER.build_successful_message_event): + # Task has finished without exceptions + # endTask jobId exitValue message + params = build_return_params_message(new_types, new_values) + message = " ".join( + (TAGS.end_task, str(job_id), str(exit_value), str(params) + "\n") + ) + return message + + +def build_compss_exception_message( + except_msg: str, job_id: str +) -> typing.Tuple[str, str]: + """Generate a COMPSs exception message. + + :param except_msg: Exception stacktrace. + :param job_id: Job identifier. + :return: Exception message and message. + """ + with EventInsideWorker( + TRACING_WORKER.build_compss_exception_message_event + ): + except_msg = except_msg.replace(" ", "_") + message = " ".join( + (TAGS.compss_exception, str(job_id), str(except_msg) + "\n") + ) + return except_msg, message + + +def build_exception_message(job_id: str, exit_value: int) -> str: + """Generate an exception message. + + :param job_id: Job identifier. + :param exit_value: Exit value. + :return: Exception message. + """ + with EventInsideWorker(TRACING_WORKER.build_exception_message_event): + message = " ".join( + (TAGS.end_task, str(job_id), str(exit_value) + "\n") + ) + return message + + +def clean_environment(cpus: bool, gpus: bool) -> None: + """Clean the environment. + + Mainly unset environment variables. + + :param cpus: If binded cpus. + :param gpus: If binded gpus. + :return: None + """ + with EventInsideWorker(TRACING_WORKER.clean_environment_event): + if cpus: + del os.environ["COMPSS_BINDED_CPUS"] + if gpus: + del os.environ["COMPSS_BINDED_GPUS"] + del os.environ["CUDA_VISIBLE_DEVICES"] + del os.environ["GPU_DEVICE_ORDINAL"] + del os.environ["COMPSS_HOSTNAMES"] diff --git a/examples/dds/pycompss/worker/piper/commons/utils.py b/examples/dds/pycompss/worker/piper/commons/utils.py new file mode 100644 index 00000000..f1ac4ba7 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/commons/utils.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Commons - Utils. + +This file contains the common pipers methods. +""" + +import logging + +from pycompss.util.context import CONTEXT +from pycompss.runtime.commons import GLOBALS +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.serialization.serializer import get_available_libraries +from pycompss.util.typing_helper import typing +from pycompss.worker.piper.commons.constants import HEADER +from pycompss.worker.piper.commons.executor import Pipe + + +class PiperWorkerConfiguration: + """Configuration parameters for the Piper Worker class.""" + + __slots__ = [ + "nesting", + "debug", + "tracing", + "storage_conf", + "stream_backend", + "stream_master_name", + "stream_master_port", + "tasks_x_node", + "exec_ids", + "pipes", + "control_pipe", + "cache", + "cache_profiler", + "ear", + ] + + def __init__(self) -> None: + """Construct an empty configuration description for the piper worker. + + :returns: None. + """ + self.nesting = False # type: bool + self.debug = False # type: bool + self.tracing = False # type: bool + self.storage_conf = "" # type: str + self.stream_backend = "" # type: str + self.stream_master_name = "" # type: str + self.stream_master_port = "" # type: str + self.tasks_x_node = 0 # type: int + self.exec_ids = [] # type: typing.List[int] + self.pipes = [] # type: typing.List[Pipe] + self.control_pipe = Pipe() # type: Pipe + self.cache = False # type: typing.Union[str, bool] + self.cache_profiler = "" # type: str + self.ear = False # type: bool + + def update_params(self, argv: typing.List[str]) -> None: + """Update the PiperWorkerConfiguration parameters from arguments. + + Construct a configuration description for the piper worker using + the arguments. + + :param argv: Arguments from the command line. + :return: None. + """ + GLOBALS.set_temporary_directory(argv[1]) + GLOBALS.set_log_directory(argv[2]) + GLOBALS.set_analysis_directory(argv[3]) + if argv[4] == "true": + CONTEXT.enable_nesting() + self.nesting = True + self.debug = argv[5] == "true" + self.tracing = argv[6] == "true" + self.storage_conf = argv[7] + self.stream_backend = argv[8] + self.stream_master_name = argv[9] + self.stream_master_port = argv[10] + self.cache = argv[11] + self.cache_profiler = argv[12] + self.ear = argv[13] == "true" + self.tasks_x_node = int(argv[14]) + exec_ids = argv[15 : 15 + self.tasks_x_node] # noqa: E203 + self.exec_ids = [int(exec_id) for exec_id in exec_ids] + in_pipes = argv[ + 15 + self.tasks_x_node : 15 + (self.tasks_x_node * 2) # noqa: E203 + ] + out_pipes = argv[15 + (self.tasks_x_node * 2) : -2] # noqa: E203 + if self.debug: + if self.tasks_x_node != len(in_pipes): + raise PyCOMPSsException( + f"Tasks per node different than input pipes (" + f"{self.tasks_x_node} != {len(in_pipes)})" + ) + if self.tasks_x_node != len(out_pipes): + raise PyCOMPSsException( + f"Tasks per node different than output pipes (" + f"{self.tasks_x_node} != {len(out_pipes)})" + ) + self.pipes = [] + for i in range(0, self.tasks_x_node): + self.pipes.append(Pipe(in_pipes[i], out_pipes[i])) + self.control_pipe = Pipe(argv[-2], argv[-1]) + + def print_on_logger(self, logger: logging.Logger) -> None: + """Print the configuration through the given logger. + + :param logger: Logger to output the configuration. + :return: None. + """ + logger.debug(HEADER + "-----------------------------") + logger.debug(HEADER + "Persistent worker parameters:") + logger.debug(HEADER + "-----------------------------") + logger.debug( + HEADER + + "working_dir : " + + str(GLOBALS.get_temporary_directory()) + ) + logger.debug( + HEADER + "log_dir : " + str(GLOBALS.get_log_directory()) + ) + logger.debug( + HEADER + + "analysis_dir : " + + str(GLOBALS.get_analysis_directory()) + ) + logger.debug(HEADER + "Nesting : " + str(self.nesting)) + logger.debug(HEADER + "Debug : " + str(self.debug)) + logger.debug(HEADER + "Tracing : " + str(self.tracing)) + logger.debug(HEADER + "Cache : " + str(self.cache)) + logger.debug(HEADER + "Cache profiler : " + str(self.cache_profiler)) + logger.debug(HEADER + "Ear : " + str(self.ear)) + logger.debug(HEADER + "Tasks per node : " + str(self.tasks_x_node)) + logger.debug(HEADER + "Exec ids : ") + for exec_id in self.exec_ids: + logger.debug(HEADER + " * " + str(exec_id)) + logger.debug(HEADER + "Pipe Pairs : ") + for pipe in self.pipes: + logger.debug(HEADER + " * " + str(pipe)) + logger.debug(HEADER + "Storage conf. : " + str(self.storage_conf)) + logger.debug(HEADER + "Stream backend : " + str(self.stream_backend)) + logger.debug( + HEADER + "Stream master : " + str(self.stream_master_name) + ) + logger.debug( + HEADER + "Stream port : " + str(self.stream_master_port) + ) + logger.debug(HEADER + "-----------------------------") + + available_libs = get_available_libraries() + + logger.debug( + HEADER + "Available serialization/deserialization libraries:" + ) + for priority, lib, lib_file in available_libs: + logger.debug( + HEADER + " - %s : %s : %s" % (str(priority), lib, lib_file) + ) + + logger.debug(HEADER + "-----------------------------") diff --git a/examples/dds/pycompss/worker/piper/commons/utils_logger.py b/examples/dds/pycompss/worker/piper/commons/utils_logger.py new file mode 100644 index 00000000..9165e3d8 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/commons/utils_logger.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Commons - Utils logger. + +This file contains the common pipers methods related to logging. +""" + +import logging + +from pycompss.runtime.commons import GLOBALS +from pycompss.util.logger.helpers import init_logging_worker_piper +from pycompss.util.logger.remittent import LOG_REMITTENT +from pycompss.util.logger.level import LOG_LEVEL +from pycompss.util.typing_helper import typing + + +def load_loggers( + debug: bool, persistent_storage: bool +) -> typing.Tuple[logging.Logger, typing.List[logging.Logger], str]: + """Load all loggers. + + :param debug: Is Debug enabled. + :param persistent_storage: Is persistent storage enabled. + :return: Main logger of the application, the log config file (json), + a list of loggers for the persistent data framework, and + the temporary log directory. + """ + # log_dir is of the form: + # With agents or worker in master: + # /path/to/working_directory/tmpFiles/pycompssID/../../log + # Normal master-worker execution : + # /path/to/working_directory/machine_name/pycompssID/../log + # With normal master-worker execution, it transfers the err and out + # files in the expected folder to the master. + # With agents or worker in master it does not, so keep it in previous + # two folders: + log_dir = GLOBALS.get_log_directory() + if not log_dir: + if __debug__: + print( + "WARNING: Log dir not set, " + + "using temporary directory as log dir." + ) + log_dir = GLOBALS.get_temporary_directory() + + # Load log level configuration file + if debug: + # Debug + init_logging_worker_piper( + LOG_REMITTENT.WORKER, LOG_LEVEL.DEBUG, log_dir + ) + else: + # Default + init_logging_worker_piper(LOG_REMITTENT.WORKER, LOG_LEVEL.OFF, log_dir) + + # Define logger facilities + logger = logging.getLogger("pycompss.worker.piper.piper_worker") + storage_loggers = [] + if persistent_storage: + storage_loggers.append(logging.getLogger("dataclay")) + storage_loggers.append(logging.getLogger("hecuba")) + storage_loggers.append(logging.getLogger("redis")) + storage_loggers.append(logging.getLogger("storage")) + return logger, storage_loggers, log_dir diff --git a/examples/dds/pycompss/worker/piper/mpi_piper_worker.py b/examples/dds/pycompss/worker/piper/mpi_piper_worker.py new file mode 100644 index 00000000..81641915 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/mpi_piper_worker.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - MPI Worker. + +This file contains the mpi piper worker code. +""" + +import os +import signal +import sys + +from mpi4py import MPI +from pycompss.runtime.commons import GLOBALS +from pycompss.util.context import CONTEXT +from pycompss.util.exceptions import PyCOMPSsException +from pycompss.util.process.manager import Queue +from pycompss.util.tracing.helpers import dummy_context +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.helpers import trace_mpi_executor +from pycompss.util.tracing.helpers import trace_mpi_worker +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.worker.piper.cache.setup import is_cache_enabled +from pycompss.worker.piper.cache.setup import start_cache +from pycompss.worker.piper.cache.setup import stop_cache +from pycompss.worker.piper.commons.constants import HEADER +from pycompss.worker.piper.commons.constants import TAGS +from pycompss.worker.piper.commons.executor import ExecutorConf +from pycompss.worker.piper.commons.executor import executor +from pycompss.worker.piper.commons.utils import PiperWorkerConfiguration +from pycompss.worker.piper.commons.utils_logger import load_loggers + +# Persistent worker global variables +COMM = MPI.COMM_WORLD +SIZE = COMM.Get_size() +RANK = COMM.Get_rank() +PROCESSES = {} # IN_PIPE -> PROCESS ID + + +def is_worker() -> bool: + """Return whether the process should act as a worker. + + :return: The process should act as a worker. + """ + return RANK == 0 + + +def shutdown_handler( + signal: int, # pylint: disable=redefined-outer-name, unused-argument + frame: typing.Any, # pylint: disable=unused-argument +) -> None: + """Handle shutdown - Shutdown handler. + + CAUTION! Do not remove the parameters. + + :param signal: Shutdown signal. + :param frame: Frame. + :return: None. + """ + if is_worker(): + print(f"{HEADER}Shutdown signal handler") + else: + print(f"[PYTHON EXECUTOR {RANK}] Shutdown signal handler") + + +def user_signal_handler( + signal: int, # pylint: disable=redefined-outer-name, unused-argument + frame: typing.Any, # pylint: disable=unused-argument +) -> None: + """Handle user signal - User signal handler. + + CAUTION! Do not remove the parameters. + + :param signal: Shutdown signal. + :param frame: Frame. + :return: None. + """ + if is_worker(): + print(f"{HEADER}Default user signal handler") + else: + print(f"[PYTHON EXECUTOR {RANK}] Default user signal handler") + + +###################### +# Main method +###################### + + +def compss_persistent_worker(config: PiperWorkerConfiguration) -> None: + """Create persistent worker main function. + + Retrieve the initial configuration and represents the main worker process. + + :param config: Piper Worker Configuration description. + :return: None. + """ + # First thing to do is to emit the process identifier event + emit_manual_event_explicit( + TRACING_WORKER.process_identifier, TRACING_WORKER.process_worker_event + ) + + pids = COMM.gather(str(os.getpid()), root=0) + if not pids: + raise PyCOMPSsException("Could not gather MPI COMM.") + + # Catch SIGTERM sent by bindings_piper + signal.signal(signal.SIGTERM, shutdown_handler) + # Catch SIGUSER2 to solve strange behaviour with mpi4py + signal.signal(signal.SIGUSR2, user_signal_handler) + + # Set the binding in worker mode + CONTEXT.set_worker() + + persistent_storage = config.storage_conf != "null" + + logger, _, _ = load_loggers(config.debug, persistent_storage) + + if __debug__: + logger.debug( + "%s[mpi_piper_worker.py] rank: %s wake up", HEADER, str(RANK) + ) + config.print_on_logger(logger) + + # Start storage + if persistent_storage: + # Initialize storage + if __debug__: + logger.debug("%sStarting persistent storage", HEADER) + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + initWorker as initStorageAtWorker, + ) + + initStorageAtWorker(config_file_path=config.storage_conf) + + for i in range(0, config.tasks_x_node): + child_in_pipe = config.pipes[i].input_pipe + try: + child_pid = pids[i + 1] + except IndexError: + child_pid = pids[i] + PROCESSES[child_in_pipe] = child_pid + + if __debug__: + logger.debug("%sStarting alive", HEADER) + logger.debug("%sControl pipe: %s", HEADER, str(config.control_pipe)) + # Read command from control pipe + alive = True + control_pipe = config.control_pipe + while alive: + command = control_pipe.read_command() + if command != "": + line = command.split() + if line[0] == TAGS.add_executor: + in_pipe = line[1] + out_pipe = line[2] + control_pipe.write( + " ".join( + (TAGS.add_executor_failed, out_pipe, in_pipe, str(0)) + ) + ) + + elif line[0] == TAGS.remove_executor: + in_pipe = line[1] + out_pipe = line[2] + PROCESSES.pop(in_pipe, None) + control_pipe.write( + " ".join((TAGS.removed_executor, out_pipe, in_pipe)) + ) + + elif line[0] == TAGS.query_executor_id: + in_pipe = line[1] + out_pipe = line[2] + pid = PROCESSES.get(in_pipe) + control_pipe.write( + " ".join( + (TAGS.reply_executor_id, out_pipe, in_pipe, str(pid)) + ) + ) + + elif line[0] == TAGS.cancel_task: + in_pipe = line[1] + cancel_pid = str(PROCESSES.get(in_pipe)) + if __debug__: + logger.debug( + "%sSignaling process with PID %s to cancel a task", + HEADER, + cancel_pid, + ) + # Cancellation produced by COMPSs + os.kill(int(cancel_pid), signal.SIGUSR2) + + elif line[0] == TAGS.ping: + control_pipe.write(TAGS.pong) + + elif line[0] == TAGS.quit: + alive = False + else: + if __debug__: + logger.debug( + "%sERROR: UNKNOWN COMMAND: %s", HEADER, command + ) + alive = False + + # Stop storage + if persistent_storage: + # Finish storage + if __debug__: + logger.debug("%sStopping persistent storage", HEADER) + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + finishWorker as finishStorageAtWorker, + ) + + finishStorageAtWorker() + + if __debug__: + logger.debug("%sFinished", HEADER) + + control_pipe.write(TAGS.quit) + control_pipe.close() + + +def compss_persistent_executor( + config: PiperWorkerConfiguration, + tracing: bool, + in_cache_queue: typing.Optional[Queue], + out_cache_queue: typing.Optional[Queue], + cache_ids: typing.Any, +) -> None: + """Create persistent MPI executor main function. + + Retrieve the initial configuration and performs executor process + functionality. + + :param config: Piper Worker Configuration description. + :param tracing: If tracing is activated. + :param in_cache_queue: Cache input queue. + :param out_cache_queue: Cache output queue. + :param cache_ids: Cache identifiers. + :return: None. + """ + COMM.gather(str(os.getpid()), root=0) + + # Catch SIGTERM sent by bindings_piper + signal.signal(signal.SIGTERM, shutdown_handler) + # Catch SIGUSER2 to solve strange behaviour with mpi4py + signal.signal(signal.SIGUSR2, user_signal_handler) + + # Set the binding in worker mode + CONTEXT.set_worker() + + persistent_storage = config.storage_conf != "null" + + logger, storage_loggers, _ = load_loggers(config.debug, persistent_storage) + + cache_profiler = False + if config.cache_profiler.lower() == "true": + cache_profiler = True + + if persistent_storage: + # Initialize storage + with EventWorker(TRACING_WORKER.init_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + initWorker as initStorageAtWorker, + ) + + initStorageAtWorker(config_file_path=config.storage_conf) + + executor_id = config.exec_ids[RANK - 1] + executor_name = "".join(("Rank-", str(RANK))) + conf = ExecutorConf( + config.debug, + GLOBALS.get_temporary_directory(), + tracing, + config.storage_conf, + logger, + persistent_storage, + storage_loggers, + config.stream_backend, + config.stream_master_name, + config.stream_master_port, + cache_ids, + in_cache_queue, + out_cache_queue, + cache_profiler, + ) + executor( + None, + None, + None, + executor_id, + executor_name, + config.pipes[RANK - 1], + conf, + ) + + if persistent_storage: + # Finish storage + if __debug__: + logger.debug("%sStopping persistent storage", HEADER) + with EventWorker(TRACING_WORKER.finish_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + finishWorker as finishStorageAtWorker, + ) + + finishStorageAtWorker() + + +############################ +# Main -> Calls main method +############################ + + +def main() -> None: + """Start the MPI piper worker. + + :return: None. + """ + tracing = sys.argv[6] == "true" + + # Enable coverage if performed + if "COVERAGE_PROCESS_START" in os.environ: + import coverage # pylint: disable=import-outside-toplevel + + coverage.process_startup() + + # Configure the piper worker with the arguments + worker_conf = PiperWorkerConfiguration() + worker_conf.update_params(sys.argv) + + persistent_storage = worker_conf.storage_conf != "null" + logger, _, log_dir = load_loggers(worker_conf.debug, persistent_storage) + analysis_dir = GLOBALS.get_analysis_directory() + + cache_profiler = False + if worker_conf.cache_profiler.lower() == "true": + cache_profiler = True + + # No cache or it is an executor + cache = False + in_cache_queue = None # type: typing.Any + out_cache_queue = None # type: typing.Any + cache_ids = None + if is_worker(): + # Setup cache if enabled + if is_cache_enabled(str(worker_conf.cache)): + # Deploy the necessary processes + cache = True + cache_params = start_cache( + logger, str(worker_conf.cache), cache_profiler, analysis_dir + ) + ( + smm, + cache_process, + in_cache_queue, + out_cache_queue, + cache_ids, + ) = cache_params + + if is_worker(): + with trace_mpi_worker() if tracing else dummy_context(): + compss_persistent_worker(worker_conf) + else: + with trace_mpi_executor() if tracing else dummy_context(): + compss_persistent_executor( + worker_conf, + tracing, + in_cache_queue, + out_cache_queue, + cache_ids, + ) + + if cache and is_worker(): + # Beware of smm, in_cache_queue, out_cache_queue and cache_process + # variables, since they are only initialized when is_worker() and + # cache is enabled.# Reason for noqa. + stop_cache( + smm, in_cache_queue, out_cache_queue, cache_profiler, cache_process + ) + + +if __name__ == "__main__": + main() diff --git a/examples/dds/pycompss/worker/piper/piper_worker.py b/examples/dds/pycompss/worker/piper/piper_worker.py new file mode 100644 index 00000000..41180e31 --- /dev/null +++ b/examples/dds/pycompss/worker/piper/piper_worker.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2024 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs Worker - Piper - Multiprocessing worker. + +This file contains the multiprocessing piper worker code. +""" +import os +import signal +import sys +import traceback + +# Used only for typing +from multiprocessing import Process # noqa: F401 + +from pycompss.util.context import CONTEXT +from pycompss.runtime.commons import GLOBALS +from pycompss.util.process.manager import Queue # just typing +from pycompss.util.process.manager import create_process +from pycompss.util.process.manager import initialize_multiprocessing +from pycompss.util.process.manager import new_queue +from pycompss.util.process.manager import new_event +from pycompss.util.process.preloader import preimports +from pycompss.util.process.preloader import preload_imports +from pycompss.util.tracing.helpers import dummy_context +from pycompss.util.tracing.helpers import EventWorker +from pycompss.util.tracing.helpers import trace_multiprocessing_worker +from pycompss.util.tracing.helpers import emit_manual_event_explicit +from pycompss.util.tracing.types_events_worker import TRACING_WORKER +from pycompss.util.typing_helper import typing +from pycompss.worker.piper.cache.setup import is_cache_enabled +from pycompss.worker.piper.cache.setup import start_cache +from pycompss.worker.piper.cache.setup import stop_cache +from pycompss.worker.piper.commons.constants import HEADER +from pycompss.worker.piper.commons.constants import TAGS +from pycompss.worker.piper.commons.executor import ExecutorConf +from pycompss.worker.piper.commons.executor import Pipe +from pycompss.worker.piper.commons.executor import executor +from pycompss.worker.piper.commons.utils import PiperWorkerConfiguration +from pycompss.worker.piper.commons.utils_logger import load_loggers + +# Persistent worker global variables +# PROCESSES = IN_PIPE -> (PROCESS, EVENT) +PROCESSES = {} # type: typing.Dict[str, typing.Tuple[Process, typing.Any]] +CACHE = None +CACHE_PROCESS = None +SUBHEADER = "[piper_worker.py]" +EARING = False + + +def shutdown_handler( + signal: int, # pylint: disable=redefined-outer-name, unused-argument + frame: typing.Any, # pylint: disable=unused-argument +) -> None: + """Handle shutdown - Shutdown handler. + + CAUTION! Do not remove the parameters. + + :param signal: Shutdown signal. + :param frame: Frame. + :return: None + """ + process_id = os.getpid() + process_name = os.environ["EAR_APP_NAME"] + sys.stderr.write( + f"[shutdown_handler] piper_worker.py Received SIGTERM - " + f"PID: {process_id} PROCESS NAME: {process_name}\n" + ) + sys.stderr.write(f"SIGNAL: {signal}\n") + sys.stderr.write(f"FRAME: %{str(frame)}\n") + traceback.print_stack(frame) + sys.stderr.write( + "[shutdown_handler] piper_worker.py SIGTERM - " + "Checking executor processes.\n" + ) + for proc, event in PROCESSES.values(): + if proc.is_alive(): + sys.stderr.write( + f"[shutdown_handler] Process id: {proc.pid} " + f"is alive, terminating. \n" + ) + # proc.terminate() # Too hard + event.set() + else: + sys.stderr.write( + f"[shutdown_handler] Process id: {proc.pid} " + f"is not alive.\n" + ) + if CACHE and CACHE_PROCESS.is_alive(): # noqa + sys.stderr.write( + f"[shutdown_handler] Cache Process id: " + f"{CACHE_PROCESS.pid} is alive, terminating.\n" + ) + CACHE_PROCESS.terminate() + if EARING: + sys.stderr.write( + f"[shutdown_handler] Stopping EAR - " + f"PID: {process_id} PROCESS NAME: {process_name}\n" + ) + sys.stderr.flush() + import ear + + ear.finalize() + sys.stderr.write( + "[shutdown_handler] piper_worker.py SIGTERM - " "Flushing\n" + ) + sys.stderr.flush() + sys.stdout.flush() + + +###################### +# Main method +###################### + + +def compss_persistent_worker( + config: PiperWorkerConfiguration, tracing: bool +) -> None: + """Retrieve the initial configuration and spawns the worker processes. + + Persistent worker main function. + + :param config: Piper Worker Configuration description. + :param tracing: If tracing is enabled. + :return: None. + """ + global CACHE + global CACHE_PROCESS + global EARING + + # Catch SIGTERM sent by bindings_piper + signal.signal(signal.SIGTERM, shutdown_handler) + + # Set the binding in worker mode + CONTEXT.set_worker() + + persistent_storage = config.storage_conf != "null" + + logger, storage_loggers, log_dir = load_loggers( + config.debug, persistent_storage + ) + + if __debug__: + logger.debug("%s%s wake up", HEADER, SUBHEADER) + config.print_on_logger(logger) + + if preimports(): + if __debug__: + logger.debug("%s%s Preloading imports", HEADER, SUBHEADER) + with EventWorker(TRACING_WORKER.preload_import_event): + preload_imports(logger, HEADER, SUBHEADER) + + if config.ear: + EARING = True + if __debug__: + logger.debug("%s%s Loading EAR", HEADER, SUBHEADER) + with EventWorker(TRACING_WORKER.load_ear_event): + import ear + + if persistent_storage: + # Initialize storage + logger.debug("%s%s Starting persistent storage", HEADER, SUBHEADER) + with EventWorker(TRACING_WORKER.init_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + initWorker as initStorageAtWorker, + ) + + initStorageAtWorker(config_file_path=config.storage_conf) + + # Create new processes + queues = [] # type: typing.List[Queue] + + cache_profiler = False + if config.cache_profiler.lower() == "true": + cache_profiler = True + + # Setup cache + CACHE = False + cache_ids, in_cache_queue, out_cache_queue = None, None, None + if is_cache_enabled(str(config.cache)): + # Deploy the necessary processes + CACHE = True + cache_params = start_cache( + logger, + str(config.cache), + cache_profiler, + GLOBALS.get_analysis_directory(), + ) + ( + smm, + cache_process, + in_cache_queue_act, + out_cache_queue_act, + cache_ids, + ) = cache_params + in_cache_queue = in_cache_queue_act + out_cache_queue = out_cache_queue_act + CACHE_PROCESS = cache_process + + # Create new executor processes + conf = ExecutorConf( + config.debug, + GLOBALS.get_temporary_directory(), + tracing, + config.storage_conf, + logger, + persistent_storage, + storage_loggers, + config.stream_backend, + config.stream_master_name, + config.stream_master_port, + cache_ids, + in_cache_queue, + out_cache_queue, + cache_profiler, + config.ear, + ) + + for i in range(0, config.tasks_x_node): + exec_id = config.exec_ids[i] + if __debug__: + logger.debug( + "%s%s Launching process %s", HEADER, SUBHEADER, str(exec_id) + ) + process_name = "".join(("Process-", str(exec_id))) + # set name for ear + os.environ["EAR_APP_NAME"] = "python_executor_" + str(i) + pid, queue = create_executor_process( + exec_id, process_name, conf, config.pipes[i] + ) + queues.append(queue) + + # Read command from control pipe + alive = True + error_msgs = [] # type: typing.List[str] + process_counter = config.tasks_x_node + control_pipe = config.control_pipe + while alive: + command = control_pipe.read_command(retry_period=1) + if command != "": + line = command.split() + + if line[0] == TAGS.add_executor: + process_name = "".join(("Process-", str(process_counter))) + process_counter = process_counter + 1 + exec_id = int(line[1]) + in_pipe = line[2] + out_pipe = line[3] + pipe = Pipe(in_pipe, out_pipe) + pid, queue = create_executor_process( + exec_id, process_name, conf, pipe + ) + queues.append(queue) + control_pipe.write( + " ".join( + (TAGS.added_executor, out_pipe, in_pipe, str(pid)) + ) + ) + + elif line[0] == TAGS.query_executor_id: + in_pipe = line[1] + out_pipe = line[2] + query_proc, _ = PROCESSES[in_pipe] + query_pid = query_proc.pid + control_pipe.write( + " ".join( + ( + TAGS.reply_executor_id, + out_pipe, + in_pipe, + str(query_pid), + ) + ) + ) + + elif line[0] == TAGS.cancel_task: + in_pipe = line[1] + cancel_proc, _ = PROCESSES[in_pipe] + cancel_pid = cancel_proc.pid + if cancel_pid is None: + alive = False + error_msgs.append("Cancel pid is None") + else: + cancel_pid_int = int(cancel_pid) + if __debug__: + logger.debug( + "%s%s Signaling process with PID %s to cancel a task", + HEADER, + SUBHEADER, + str(cancel_pid), + ) + # Cancellation produced by COMPSs + os.kill(cancel_pid_int, signal.SIGUSR2) + + elif line[0] == TAGS.remove_executor: + in_pipe = line[1] + out_pipe = line[2] + proc, event = PROCESSES.pop(in_pipe, (None, new_event())) + if proc: + if proc.is_alive(): + logger.warning( + "%s%s Forcing terminate on: %s (pid: %s)", + HEADER, + SUBHEADER, + proc.name, + proc.pid, + ) + # proc.terminate() # Too hard + event.set() + else: + logger.warning( + "%s%s Could not force terminate on: %s (pid: %s)", + HEADER, + SUBHEADER, + proc.name, + proc.pid, + ) + proc.join() + control_pipe.write( + " ".join((TAGS.removed_executor, out_pipe, in_pipe)) + ) + + elif line[0] == TAGS.ping: + control_pipe.write(TAGS.pong) + + elif line[0] == TAGS.quit: + alive = False + + # Wait for all threads + for proc, _ in PROCESSES.values(): + proc.join() + + # Check if there is any exception message from the threads + for i in range(0, config.tasks_x_node): + if not queues[i].empty(): + logger.error( + "%s%s Exception in threads queue: %s", + HEADER, + SUBHEADER, + str(queues[i].get()), + ) + + # Check if there is any exception from the messages + for msg in error_msgs: + logger.error( + "%s%s Exception in piper worker message: %s", + HEADER, + SUBHEADER, + msg, + ) + + for queue in queues: + queue.close() + queue.join_thread() + + if CACHE: + # Beware of smm, in_cache_queue_act, out_cache_queue_act and + # cache_process variables, since they are only initialized when + # cache is enabled. Reason for noqa. + stop_cache( + smm, + in_cache_queue_act, + out_cache_queue_act, + cache_profiler, + cache_process, + ) + + if persistent_storage: + # Finish storage + if __debug__: + logger.debug("%s%s Stopping persistent storage", HEADER, SUBHEADER) + with EventWorker(TRACING_WORKER.finish_storage_at_worker_event): + from storage.api import ( # pylint: disable=E0401, C0415 + # disable=import-error, import-outside-toplevel + finishWorker as finishStorageAtWorker, + ) + + finishStorageAtWorker() + + if EARING: + if __debug__: + logger.debug("%s%s Stopping EAR", HEADER, SUBHEADER) + with EventWorker(TRACING_WORKER.finalize_ear_event): + ear.finalize() + EARING = False + + if __debug__: + logger.debug("%s%s Finished", HEADER, SUBHEADER) + + control_pipe.write(TAGS.quit) + control_pipe.close() + + if EARING: + # Raise SIGTERM to main interpreter so that EAR is noticed of its + # finalization. + signal.raise_signal(signal.SIGTERM) + + +def create_executor_process( + executor_id: int, executor_name: str, conf: ExecutorConf, pipe: Pipe +) -> typing.Tuple[int, Queue]: + """Start a new executor. + + :param executor_id: Executor process identifier. + :param executor_name: Executor process name. + :param conf: executor config. + :param pipe: Communication pipes (in, out). + :return: Process identifier and queue used by the process. + """ + queue = new_queue() + event = new_event() + process = create_process( + target=executor, + args=(queue, event, executor_id, executor_name, pipe, conf), + prepend_lock=True, + ) + PROCESSES[pipe.input_pipe] = (process, event) + process.start() + return int(str(process.pid)), queue + + +############################ +# Main -> Calls main method +############################ + + +def main() -> None: + """Start the multiprocessing worker. + + :return: None. + """ + # Configure the global tracing variable from the argument + tracing = sys.argv[6] == "true" + with trace_multiprocessing_worker() if tracing else dummy_context(): + # First thing to do is to emit the process identifier event + emit_manual_event_explicit( + TRACING_WORKER.process_identifier, + TRACING_WORKER.process_worker_event, + ) + # Configure the piper worker with the arguments + worker_conf = PiperWorkerConfiguration() + worker_conf.update_params(sys.argv) + compss_persistent_worker(worker_conf, tracing) + + +if __name__ == "__main__": + # Enable EAR accounting for subsequent processes + if "EAR_DISABLE_NODE_METRICS" in os.environ: + del os.environ["EAR_DISABLE_NODE_METRICS"] + # Initialize multiprocessing + initialize_multiprocessing() + # Then start the main function + main() diff --git a/noxfile.py b/noxfile.py index b1fb54e5..12bf3d96 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,7 +4,7 @@ PYPROJECT = nox.project.load_toml("pyproject.toml") #PYTHON_VERSIONS = nox.project.python_versions(PYPROJECT) PYTHON_VERSIONS = [ - "3.9", "3.10", "3.11", "3.12" + "3.10", "3.11", "3.12" ] DEFAULT_PYTHON = "3.10" # Modern-ish version compatible with the legacy-deps diff --git a/src/dataclay/contrib/modeltest/compss.py b/src/dataclay/contrib/modeltest/compss.py index 9474ecc8..a9e8af55 100644 --- a/src/dataclay/contrib/modeltest/compss.py +++ b/src/dataclay/contrib/modeltest/compss.py @@ -8,7 +8,7 @@ from pycompss.api.parameter import CONCURRENT, INOUT from pycompss.api.task import task except ImportError: - from dataclay.contrib.dummy_pycompss import task, INOUT, CONCURRENT + from dataclay.contrib.pycompss_dummy.dummy_pycompss import task, INOUT, CONCURRENT from dataclay import DataClayObject, activemethod diff --git a/src/dataclay/contrib/persistent_block.py b/src/dataclay/contrib/persistent_block.py index f6220c64..ea482d34 100644 --- a/src/dataclay/contrib/persistent_block.py +++ b/src/dataclay/contrib/persistent_block.py @@ -10,7 +10,7 @@ from pycompss.api.task import task from pycompss.api.parameter import IN except ImportError: - from dataclay.contrib.dummy_pycompss import task, IN + from dataclay.contrib.pycompss_dummy.dummy_pycompss import task, IN class PersistentBlock(DataClayObject): diff --git a/src/dataclay/contrib/pycompss_dummy/__init__.py b/src/dataclay/contrib/pycompss_dummy/__init__.py new file mode 100644 index 00000000..6071cd1d --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +"""This package contains the API dummy functions, constants and classes.""" diff --git a/src/dataclay/contrib/pycompss_dummy/_decorator.py b/src/dataclay/contrib/pycompss_dummy/_decorator.py new file mode 100644 index 00000000..cd274db6 --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/_decorator.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - decorator. + +This file contains the dummy class task used as decorator. +""" + +from pycompss.util.typing_helper import typing + + +class _Dummy: + """Dummy task class (decorator style).""" + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + """Construct a dummy Task decorator. + + :param args: Task decorator arguments. + :param kwargs: Task decorator keyword arguments. + :returns: None + """ + self.args = args + self.kwargs = kwargs + + def __call__(self, function: typing.Any) -> typing.Any: + """Invoke the dummy decorator. + + :param function: Decorated function. + :returns: Result of executing the given function. + """ + + def wrapped_f(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + # returns may appear in @task decorator + if "returns" in kwargs: + kwargs.pop("returns") + return function(*args, **kwargs) + + return wrapped_f + + def __repr__(self) -> str: + attributes = f"(args: {repr(self.args)}, kwargs: {repr(self.kwargs)})" + return f"Dummy {self.__class__.__name__} decorator {attributes}" diff --git a/src/dataclay/contrib/pycompss_dummy/api.py b/src/dataclay/contrib/pycompss_dummy/api.py new file mode 100644 index 00000000..fae0015a --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/api.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs - dummy - api. + +This file defines the public PyCOMPSs API functions without functionality. +""" + +import os + +from pycompss.util.typing_helper import typing + + +def compss_start( + log_level: str = "off", # pylint: disable=unused-argument + tracing: bool = False, # pylint: disable=unused-argument + interactive: bool = False, # pylint: disable=unused-argument + disable_external: bool = False, # pylint: disable=unused-argument +) -> None: # pylint: disable=unused-argument + """Start runtime dummy. + + Does nothing. + + :param log_level: Log level [ True | False ]. + :param tracing: Activate or disable tracing. + :param interactive: Boolean if interactive (ipython or jupyter). + :param disable_external: To avoid to load compss in external process. + :return: None + """ + + +def compss_stop( + code: int = 0, _hard_stop: bool = False # pylint: disable=unused-argument +) -> None: + """Stop runtime dummy. + + Does nothing. + + :param code: Stop code. + :param _hard_stop: Stop COMPSs when runtime has died. + :return: None + """ + + +def compss_file_exists( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Check if one or more files used in task exists dummy. + + Check if the file/s exists. + + :param file_name: The file/s name to check. + :return: True if exists. False otherwise. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for f_name in file_name: + if isinstance(f_name, (list, tuple)): + ret.append([compss_file_exists(name) for name in f_name]) + else: + ret.append(os.path.exists(f_name)) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_open(file_name: str, mode: str = "r") -> typing.Any: + """Open a file used in task dummy. + + Open the given file with the defined mode (see builtin open). + + :param file_name: The file name to open. + :param mode: Open mode. Options = [w, r+ or a, r or empty]. Default=r. + :return: An object of "file" type. + :raise IOError: If the file can not be opened. + """ + return open(file_name, mode) # pylint: disable=unspecified-encoding + + +def compss_delete_file( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Delete one or more files used in task dummy. + + Does nothing and always return True. + + :param file_name: File/s name. + :return: Always True. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for f_name in file_name: + if isinstance(f_name, (list, tuple)): + ret.append([compss_delete_file(name) for name in f_name]) + else: + ret.append(True) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_wait_on_file( + *file_name: typing.Union[list, tuple, str], +) -> typing.Union[list, tuple, str]: + """Wait on file used in task dummy. + + Does nothing. + + :param file_name: File/s name. + :return: The files/s name. + """ + if len(file_name) == 1: + return file_name[0] + return file_name + + +def compss_wait_on_directory( + *directory_name: typing.Union[list, tuple, str], +) -> typing.Union[list, tuple, str]: + """Wait on directory used in task dummy. + + Does nothing. + + :param directory_name: Directory/ies name. + :return: The directory/ies name. + """ + if len(directory_name) == 1: + return directory_name[0] + return directory_name + + +def compss_delete_object( + *objs: typing.Any, +) -> typing.Union[bool, typing.List[typing.Union[bool, list]]]: + """Delete one or more objects used in task dummy. + + Does nothing and always return True. + + :param objs: Object/s to delete. + :return: Always True. + """ + ret = [] # type: typing.List[typing.Union[bool, list]] + for obj in objs: + if isinstance(obj, (list, tuple)): + ret.append([compss_delete_object(elem) for elem in obj]) + else: + ret.append(True) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_barrier( + no_more_tasks: bool = False, # pylint: disable=unused-argument +) -> None: + """Wait for all submitted tasks dummy. + + Does nothing. + + :param no_more_tasks: No more tasks boolean. + :return: None + """ + + +def compss_barrier_group( + group_name: str, # pylint: disable=unused-argument +) -> None: + """Wait for all submitted tasks of a group dummy. + + Does nothing. + + :param group_name: Name of the group. + :return: None + """ + + +def compss_cancel_group( + group_name: str, # pylint: disable=unused-argument +) -> None: + """Cancel all submitted tasks of a group dummy. + + Does nothing. + + :param group_name: Name of the group. + :return: None + """ + + +def compss_snapshot() -> None: + """Request a snapshot. + + Does nothing. + + :return: None + """ + + +def compss_wait_on( + *args: typing.Any, **kwargs: typing.Any # pylint: disable=unused-argument +) -> typing.Any: + """Synchronize an object used in task dummy. + + Does nothing. + + :param args: Objects to wait on. + :param kwargs: Options dictionary. + :return: The same objects defined as parameter. + """ + ret = list(map(lambda o: o, args)) + if len(ret) == 1: + return ret[0] + return ret + + +def compss_set_wall_clock( + wall_clock_limit: int, # pylint: disable=unused-argument +) -> None: + """Set the application wall_clock_limit dummy. + + Does nothing. + + :param wall_clock_limit: Wall clock limit in seconds. + :return: None + """ + + +class TaskGroup: + """Dummy TaskGroup context manager.""" + + def __init__( + self, group_name: str, implicit_barrier: bool = True + ) -> None: # pylint: disable=unused-argument + """Define a new group of tasks. + + :param group_name: Group name. + :param implicit_barrier: Perform implicit barrier. + """ + + def __enter__(self) -> None: + """Do nothing. + + :return: None + """ + + def __exit__( + self, + type: typing.Any, # pylint: disable=redefined-builtin + value: typing.Any, + traceback: typing.Any, + ) -> None: + """Do nothing. + + :return: None + """ diff --git a/src/dataclay/contrib/pycompss_dummy/constraint.py b/src/dataclay/contrib/pycompss_dummy/constraint.py new file mode 100644 index 00000000..195b16be --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/constraint.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - constraint. + +This file contains the dummy class constraint used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Constraint = Dummy # pylint: disable=invalid-name +constraint = Dummy # pylint: disable=invalid-name diff --git a/src/dataclay/contrib/pycompss_dummy/container.py b/src/dataclay/contrib/pycompss_dummy/container.py new file mode 100644 index 00000000..7e437766 --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/container.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - container. + +This file contains the dummy class container used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Container = Dummy # pylint: disable=invalid-name +container = Dummy # pylint: disable=invalid-name diff --git a/src/dataclay/contrib/dummy_pycompss.py b/src/dataclay/contrib/pycompss_dummy/dummy_pycompss.py similarity index 99% rename from src/dataclay/contrib/dummy_pycompss.py rename to src/dataclay/contrib/pycompss_dummy/dummy_pycompss.py index bd68b069..fcbd8c16 100644 --- a/src/dataclay/contrib/dummy_pycompss.py +++ b/src/dataclay/contrib/pycompss_dummy/dummy_pycompss.py @@ -31,3 +31,4 @@ def task(*args, **kwargs): def constraint(*args, **kwargs): return lambda f: f + diff --git a/src/dataclay/contrib/pycompss_dummy/on_failure.py b/src/dataclay/contrib/pycompss_dummy/on_failure.py new file mode 100644 index 00000000..fc5ba7d8 --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/on_failure.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - on failure. + +This file contains the dummy class on failure used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +OnFailure = Dummy # pylint: disable=invalid-name +on_failure = Dummy # pylint: disable=invalid-name +onFailure = Dummy # noqa: N816 # pylint: disable=invalid-name diff --git a/src/dataclay/contrib/pycompss_dummy/reduction.py b/src/dataclay/contrib/pycompss_dummy/reduction.py new file mode 100644 index 00000000..2a86d451 --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/reduction.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - reduction. + +This file contains the dummy class reduction used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Reduction = Dummy # pylint: disable=invalid-name +reduction = Dummy # pylint: disable=invalid-name diff --git a/src/dataclay/contrib/pycompss_dummy/task.py b/src/dataclay/contrib/pycompss_dummy/task.py new file mode 100644 index 00000000..4d860adb --- /dev/null +++ b/src/dataclay/contrib/pycompss_dummy/task.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2002-2026 Barcelona Supercomputing Center (www.bsc.es) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- + +""" +PyCOMPSs API - dummy - task. + +This file contains the dummy class task used as decorator. +""" + +from pycompss.api.dummy._decorator import _Dummy as Dummy + +Task = Dummy # pylint: disable=invalid-name +task = Dummy # pylint: disable=invalid-name diff --git a/src/dataclay/paraver/__init__.py b/src/dataclay/paraver/__init__.py index 314c35af..5699e154 100644 --- a/src/dataclay/paraver/__init__.py +++ b/src/dataclay/paraver/__init__.py @@ -21,7 +21,7 @@ from functools import wraps from dataclay.config import settings -from dataclay.contrib.dummy_pycompss import task +from dataclay.contrib.pycompss_dummy.dummy_pycompss import task # Explicit and manually crafted list of CLASSES to be instrumentated CLASSES_WITH_EXTRAE_DECORATORS = { # Similar to Java paraver/extrae AspectJ file.