From 6dbfc1cf832ad4f7128ed3a8f94cf8f27cfe46f8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 17 Jul 2020 11:22:17 -0400 Subject: [PATCH 01/10] Begin to work on Slurm backend --- caliban/cli.py | 58 ++++++++++++ caliban/main.py | 5 + caliban/platform/slurm/__init__.py | 15 +++ caliban/platform/slurm/cli.py | 146 +++++++++++++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 caliban/platform/slurm/__init__.py create mode 100644 caliban/platform/slurm/cli.py diff --git a/caliban/cli.py b/caliban/cli.py index 443e9f0..509c288 100644 --- a/caliban/cli.py +++ b/caliban/cli.py @@ -31,6 +31,7 @@ import caliban.platform.gke.constants as gke_k import caliban.platform.gke.types as gke_t import caliban.platform.gke.util as gke_u +import caliban.platform.slurm as slurm import caliban.util as u import caliban.util.argparse as ua import caliban.util.schema as us @@ -413,6 +414,7 @@ def caliban_parser(): local_run_parser(subparser) cloud_parser(subparser) cluster_parser(subparser) + slurm_parser(subparser) status_parser(subparser) stop_parser(subparser) resubmit_parser(subparser) @@ -946,3 +948,59 @@ def max_jobs_arg(parser): f'then this specifies the total number of jobs to return, ordered ' f'by creation date, or all jobs if max_jobs==0.'), ) + + +# ---------------------------------------------------------------------------- +def slurm_parser(base): + """cli parser for slurm commands""" + + parser = base.add_parser("slurm", + description="slurm commands", + help="slurm-related commands") + + subparser = parser.add_subparsers(dest="slurm_cmd") + slurm_ls_cmd(subparser) + slurm_job_parser(subparser) + + +# ---------------------------------------------------------------------------- +def slurm_ls_cmd(base): + """caliban slurm ls""" + + parser = base.add_parser( + "ls", + description="list partitions", + help="list partitions", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + +# ---------------------------------------------------------------------------- +def slurm_job_parser(base): + parser = base.add_parser( + "job", + description="job commands", + help="job commands", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + subparser = parser.add_subparsers(dest="job_cmd") + slurm_job_ls_cmd(subparser) + cluster_job_submit_cmd(subparser) + #TODO cluster_job_submit_file_cmd(subparser) + + +# ---------------------------------------------------------------------------- +def slurm_job_ls_cmd(base): + parser = base.add_parser( + "ls", + description="list Slurm jobs", + help="list Slurm jobs", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + +# ---------------------------------------------------------------------------- +def slurm_job_submit_cmd(base): + parser = base.add_parser( + "submit", + description="submit Slurm job(s)", + help="submit Slurm job(s)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) diff --git a/caliban/main.py b/caliban/main.py index 3dada18..7607463 100644 --- a/caliban/main.py +++ b/caliban/main.py @@ -34,6 +34,8 @@ import caliban.platform.notebook as pn import caliban.platform.run as pr import caliban.platform.shell as ps +import caliban.platform.slurm as slurm +import caliban.platform.slurm.cli import caliban.util.schema as cs ll.getLogger('caliban.main').setLevel(logging.ERROR) @@ -53,6 +55,9 @@ def run_app(arg_input): if command == "cluster": return gke.cli.run_cli_command(args) + if command == "slurm": + return slurm.cli.run_cli_command(args) + job_mode = cli.resolve_job_mode(args) docker_args = cli.generate_docker_args(job_mode, args) docker_run_args = args.get("docker_run_args", []) diff --git a/caliban/platform/slurm/__init__.py b/caliban/platform/slurm/__init__.py new file mode 100644 index 0000000..79c6a2f --- /dev/null +++ b/caliban/platform/slurm/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py new file mode 100644 index 0000000..8fd0bbe --- /dev/null +++ b/caliban/platform/slurm/cli.py @@ -0,0 +1,146 @@ +#!/usr/bin/python +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""slurm cli support""" + +import json +import logging +import os +import pprint as pp +import re +import shlex +import subprocess +from datetime import datetime +from typing import Any, Dict, List, Optional + +import caliban.cli as cli +import caliban.config as conf +import caliban.util as u +from caliban.history.util import (create_experiments, generate_container_spec, + get_mem_engine, get_sql_engine, session_scope) + + +# ---------------------------------------------------------------------------- +# Remote access via ssh +_hostname = "symmetry.pi.local" +_username = "eschnetter" + +# Job management via Slurm +_partition = "debugq" +_timelimit = "00:10:00" +_slurm_path = "/cm/shared/apps/slurm/19.05.5" +_sbatch = _slurm_path + "/bin/sbatch" +_sinfo = _slurm_path + "/bin/sinfo" +_squeue = _slurm_path + "/bin/squeue" + + +# ---------------------------------------------------------------------------- +def run_cli_command(args: dict) -> None: + """cli entrypoint for Slurm commands""" + SLURM_CMDS = { + 'ls': _partition_ls, + 'job': _job_commands, + } + SLURM_CMDS[args['slurm_cmd']](args) + + +# ---------------------------------------------------------------------------- +def _partition_ls(args: dict) -> None: + """list Slurm partitions""" + cmd = [_sinfo, '-h', '-o', '%P'] + process = subprocess.run(_with_ssh(cmd), + stdout=subprocess.PIPE, + universal_newlines=True) + partitions = process.stdout.splitlines() + logging.info("{} partitions found".format(len(partitions))) + for p in partitions: + logging.info(" " + p) + + +# ---------------------------------------------------------------------------- +def _job_commands(args: dict) -> None: + """job commands""" + JOB_CMDS = { + 'ls': _job_ls, + 'submit': _job_submit, + #TODO 'submit_file': _job_submit_file + } + JOB_CMDS[args['job_cmd']](args) + + +# ---------------------------------------------------------------------------- +def _job_ls(args: dict) -> None: + """list Slurm jobs""" + # TODO: Look also for jobs that have terminated a long time ago? + cmd = [_squeue, '-h', '-o', "%A", '-t', "all", '-u', _username] + process = subprocess.run(_with_ssh(cmd), + stdout=subprocess.PIPE, + universal_newlines=True) + jobs = process.stdout.splitlines() + logging.info("{} jobs found".format(len(jobs))) + for j in jobs: + # TODO: Look up and output job metadata + logging.info(" " + j) + + +# ---------------------------------------------------------------------------- +def _job_submit(args: dict) -> None: + """submits Slurm job(s) + + Args: + args: argument dictionary + """ + + dt = datetime.now().astimezone() + jobname = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' + outputname = jobname + ".log" + + script = """\ +#!/bin/bash +echo 'Hello, World!' +echo Date: $(date) +echo Hostname: $(hostname) +""" + + logging.info("Submitting job...") + cmd = [_sbatch, + "--job-name", jobname, + "--nodes", "1", + "--output", outputname, + "--partition", _partition, + "--time", _timelimit] + process = subprocess.run(_with_ssh(cmd), + input=script, + stdout=subprocess.PIPE, + universal_newlines=True) + output = process.stdout.rstrip() + logging.info("Received response \"{}\"".format(output)) + # Output looks like "Submitted batch job 133417" + m = re.search(r"Submitted batch job (\d+)", output) + assert m.lastindex == 1 + job_id = m.group(1) + assert job_id != '' + logging.info("Submitted job {}".format(job_id)) + + +# ---------------------------------------------------------------------------- +def _with_ssh(args: [str]) -> [str]: + """adds ssh command to execute command remotely""" + # Quote command and arguments + qargs = [] + for arg in args: + qargs.append(shlex.quote(arg)) + # Add ssh command prefix + return ['ssh', '-l', _username, _hostname] + qargs From 570bf6b2d37eee79b7c3607406b78119be98b752 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 17 Jul 2020 11:29:59 -0400 Subject: [PATCH 02/10] Run Docker container on remote host --- caliban/platform/slurm/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 8fd0bbe..5929b5b 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -109,9 +109,10 @@ def _job_submit(args: dict) -> None: script = """\ #!/bin/bash -echo 'Hello, World!' -echo Date: $(date) -echo Hostname: $(hostname) +# echo 'Hello, World!' +# echo Date: $(date) +# echo Hostname: $(hostname) +/cm/shared/apps/singularity/3.3.0/bin/singularity run docker://godlovedc/lolcow """ logging.info("Submitting job...") @@ -143,4 +144,5 @@ def _with_ssh(args: [str]) -> [str]: for arg in args: qargs.append(shlex.quote(arg)) # Add ssh command prefix + assert _hostname[0] != '-' return ['ssh', '-l', _username, _hostname] + qargs From 75480d87a8e9d2d830a45e5d93e4296a6d55610e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 18 Jul 2020 15:43:09 -0400 Subject: [PATCH 03/10] Slurm: Allow specifying a job name --- caliban/cli.py | 3 ++- caliban/platform/slurm/cli.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/caliban/cli.py b/caliban/cli.py index 509c288..1f11d5e 100644 --- a/caliban/cli.py +++ b/caliban/cli.py @@ -985,7 +985,6 @@ def slurm_job_parser(base): subparser = parser.add_subparsers(dest="job_cmd") slurm_job_ls_cmd(subparser) cluster_job_submit_cmd(subparser) - #TODO cluster_job_submit_file_cmd(subparser) # ---------------------------------------------------------------------------- @@ -1004,3 +1003,5 @@ def slurm_job_submit_cmd(base): description="submit Slurm job(s)", help="submit Slurm job(s)", formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + job_name_arg(parser) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 5929b5b..442d344 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -103,9 +103,11 @@ def _job_submit(args: dict) -> None: args: argument dictionary """ - dt = datetime.now().astimezone() - jobname = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' - outputname = jobname + ".log" + job_name = args.get('name') + if job_name is None: + dt = datetime.now().astimezone() + job_name = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' + output_filename = job_name + ".log" script = """\ #!/bin/bash @@ -117,9 +119,9 @@ def _job_submit(args: dict) -> None: logging.info("Submitting job...") cmd = [_sbatch, - "--job-name", jobname, + "--job-name", job_name, "--nodes", "1", - "--output", outputname, + "--output", output_filename, "--partition", _partition, "--time", _timelimit] process = subprocess.run(_with_ssh(cmd), From 4cf2ab782c5d977a38e9db76ccf070a90af00854 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 18 Jul 2020 15:48:05 -0400 Subject: [PATCH 04/10] Slurm: Make output log files unique --- caliban/platform/slurm/cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 442d344..fc8b948 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -103,11 +103,15 @@ def _job_submit(args: dict) -> None: args: argument dictionary """ + dt = datetime.now().astimezone() + job_prefix = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' job_name = args.get('name') + # TODO: Sanitize job_name if job_name is None: - dt = datetime.now().astimezone() - job_name = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' - output_filename = job_name + ".log" + job_name = job_prefix + output_filename = job_name + ".log" + else: + output_filename = job_prefix + "-" + job_name + ".log" script = """\ #!/bin/bash From 57be6c39ed30bc1e05886cab4ed440099af3cfae Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 19 Jul 2020 20:25:02 -0400 Subject: [PATCH 05/10] Finish prototype implementation --- caliban/platform/slurm/cli.py | 162 +++++++++++++++++++++++++++------- 1 file changed, 128 insertions(+), 34 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index fc8b948..31e7c39 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -22,28 +22,40 @@ import re import shlex import subprocess +from blessings import Terminal from datetime import datetime +from pprint import pformat from typing import Any, Dict, List, Optional import caliban.cli as cli import caliban.config as conf +import caliban.docker.build as db import caliban.util as u from caliban.history.util import (create_experiments, generate_container_spec, get_mem_engine, get_sql_engine, session_scope) +from caliban.platform.cloud.core import generate_image_tag + + +t = Terminal() # ---------------------------------------------------------------------------- # Remote access via ssh +_ssh = "ssh" _hostname = "symmetry.pi.local" _username = "eschnetter" +# Environment modules +_setup_cmds = [["source", "/etc/profile"], + ["module", "load", "slurm"]] + # Job management via Slurm +_sbatch = "sbatch" +_sinfo = "sinfo" +_squeue = "squeue" _partition = "debugq" _timelimit = "00:10:00" -_slurm_path = "/cm/shared/apps/slurm/19.05.5" -_sbatch = _slurm_path + "/bin/sbatch" -_sinfo = _slurm_path + "/bin/sinfo" -_squeue = _slurm_path + "/bin/squeue" +_nodes = 1 # ---------------------------------------------------------------------------- @@ -103,9 +115,29 @@ def _job_submit(args: dict) -> None: args: argument dictionary """ + script_args = conf.extract_script_args(args) + job_mode = cli.resolve_job_mode(args) + docker_args = cli.generate_docker_args(job_mode, args) + docker_run_args = args.get('docker_run_args', []) or [] + dry_run = args['dry_run'] + package = args['module'] + job_name = args.get('name') + gpu_spec = args.get('gpu_spec') + preemptible = not args['nonpreemptible'] + min_cpu = args.get('min_cpu') + min_mem = args.get('min_mem') + experiment_config = args.get('experiment_config') or [{}] + xgroup = args.get('xgroup') + image_tag = args.get('image_tag') + export = args.get('export', None) + + labels = args.get('label') + if labels is not None: + labels = dict(cu.sanitize_labels(args.get('label'))) + + # Generate job name dt = datetime.now().astimezone() job_prefix = f'caliban-{dt.strftime("%Y%m%d-%H%M%S")}' - job_name = args.get('name') # TODO: Sanitize job_name if job_name is None: job_name = job_prefix @@ -113,42 +145,104 @@ def _job_submit(args: dict) -> None: else: output_filename = job_prefix + "-" + job_name + ".log" - script = """\ + # Arguments to internally build the image required to submit to Slurm + docker_m = {'job_mode': job_mode, 'package': package, **docker_args} + + # -------------------------------------------------------------------------- + engine = get_mem_engine() if dry_run else get_sql_engine() + + logging.info("*** engine") + with session_scope(engine) as session: + container_spec = generate_container_spec(session, docker_m, image_tag) + + logging.info("*** image_tag") + if image_tag is None: + # Create Docker image, and push it to a Docker repository + logging.info("Generating Docker image with parameters:") + logging.info(t.yellow(pformat(docker_args))) + + if dry_run: + logging.info("Dry run - skipping actual 'docker build' and 'docker push'.") + image_tag = "dry_run_tag" + else: + image_id = db.build_image(**docker_m) + + project_id = args['project_id'] + project_s = project_id.replace(":", "/") + # TODO: Use sub-project "caliban" or similar? + # base = f"docker.io/{project_s}/{image_id}" + base = f"{project_s}/{image_id}" + image_tag = f"{base}:latest" + subprocess.run(["docker", "tag", image_id, image_tag], check=True) + subprocess.run(["docker", "push", image_tag], check=True) + + logging.info("*** experiments") + experiments = create_experiments( + session=session, + container_spec=container_spec, + script_args=script_args, + experiment_config=experiment_config, + xgroup=xgroup, + ) + + for experiment in experiments: + logging.info("*** script") + # TODO: Use srun to start job, then use singularity inside srun + script = f"""\ #!/bin/bash -# echo 'Hello, World!' -# echo Date: $(date) -# echo Hostname: $(hostname) -/cm/shared/apps/singularity/3.3.0/bin/singularity run docker://godlovedc/lolcow +source /etc/profile +set -euxo pipefail +date +hostname +nproc +module load singularity +env PYTHONUNBUFFERED=1 singularity run --tmpdir /gpfs/eschnetter/singularity/tmp --workdir /gpfs/eschnetter/singularity/work --pwd /usr/app docker://{image_tag} exe/cactus_sim arrangements/CactusWave/WaveToyC/par/wavetoyc_rad.par """ - - logging.info("Submitting job...") - cmd = [_sbatch, - "--job-name", job_name, - "--nodes", "1", - "--output", output_filename, - "--partition", _partition, - "--time", _timelimit] - process = subprocess.run(_with_ssh(cmd), - input=script, - stdout=subprocess.PIPE, - universal_newlines=True) - output = process.stdout.rstrip() - logging.info("Received response \"{}\"".format(output)) - # Output looks like "Submitted batch job 133417" - m = re.search(r"Submitted batch job (\d+)", output) - assert m.lastindex == 1 - job_id = m.group(1) - assert job_id != '' - logging.info("Submitted job {}".format(job_id)) + logging.info(f"Script is {script}") + + logging.info("Submitting job...") + cmd = [_sbatch, + "--job-name", job_name, + "--nodes", str(_nodes), + "--output", output_filename, + "--partition", _partition, + "--time", _timelimit] + process = subprocess.run(_with_ssh(cmd), + input=script, + stdout=subprocess.PIPE, + universal_newlines=True, + check=True) + output = process.stdout.rstrip() + logging.info("Received response \"{}\"".format(output)) + # Output looks like "Submitted batch job 133417" + m = re.search(r"Submitted batch job (\d+)", output) + assert m.lastindex == 1 + job_id = m.group(1) + assert job_id != '' + logging.info("Submitted job {}".format(job_id)) # ---------------------------------------------------------------------------- -def _with_ssh(args: [str]) -> [str]: - """adds ssh command to execute command remotely""" - # Quote command and arguments +def _quote_args(args: [str]) -> str: + """converts a command into a shell command string""" qargs = [] for arg in args: qargs.append(shlex.quote(arg)) + return " ".join(qargs) + + +# ---------------------------------------------------------------------------- +def _join_cmds(cmds: [str]) -> str: + """joins several shell commands into a single one""" + return "; ".join(cmds) + + +# ---------------------------------------------------------------------------- +def _with_ssh(cmd: [str]) -> [str]: + """adds ssh command to execute command remotely""" + # Add module load commands + cmds = _setup_cmds + [cmd] + cmd = _join_cmds(map(_quote_args, cmds)) # Add ssh command prefix assert _hostname[0] != '-' - return ['ssh', '-l', _username, _hostname] + qargs + return [_ssh, '-l', _username, _hostname, cmd] From 3595a01e158e4c43942dc9816434df0acc0cc3c6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 19 Jul 2020 21:17:54 -0400 Subject: [PATCH 06/10] Fix white space --- caliban/platform/slurm/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 31e7c39..8545cc3 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -160,7 +160,7 @@ def _job_submit(args: dict) -> None: # Create Docker image, and push it to a Docker repository logging.info("Generating Docker image with parameters:") logging.info(t.yellow(pformat(docker_args))) - + if dry_run: logging.info("Dry run - skipping actual 'docker build' and 'docker push'.") image_tag = "dry_run_tag" From fbdcd03afa0c66c061e8a60a4810a9130b265b69 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 19 Jul 2020 22:18:34 -0400 Subject: [PATCH 07/10] Reformat code --- caliban/platform/slurm/cli.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 8545cc3..2390aed 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -35,10 +35,8 @@ get_mem_engine, get_sql_engine, session_scope) from caliban.platform.cloud.core import generate_image_tag - t = Terminal() - # ---------------------------------------------------------------------------- # Remote access via ssh _ssh = "ssh" @@ -46,8 +44,7 @@ _username = "eschnetter" # Environment modules -_setup_cmds = [["source", "/etc/profile"], - ["module", "load", "slurm"]] +_setup_cmds = [["source", "/etc/profile"], ["module", "load", "slurm"]] # Job management via Slurm _sbatch = "sbatch" @@ -85,9 +82,9 @@ def _partition_ls(args: dict) -> None: def _job_commands(args: dict) -> None: """job commands""" JOB_CMDS = { - 'ls': _job_ls, - 'submit': _job_submit, - #TODO 'submit_file': _job_submit_file + 'ls': _job_ls, + 'submit': _job_submit, + #TODO 'submit_file': _job_submit_file } JOB_CMDS[args['job_cmd']](args) @@ -162,7 +159,8 @@ def _job_submit(args: dict) -> None: logging.info(t.yellow(pformat(docker_args))) if dry_run: - logging.info("Dry run - skipping actual 'docker build' and 'docker push'.") + logging.info( + "Dry run - skipping actual 'docker build' and 'docker push'.") image_tag = "dry_run_tag" else: image_id = db.build_image(**docker_m) @@ -201,12 +199,11 @@ def _job_submit(args: dict) -> None: logging.info(f"Script is {script}") logging.info("Submitting job...") - cmd = [_sbatch, - "--job-name", job_name, - "--nodes", str(_nodes), - "--output", output_filename, - "--partition", _partition, - "--time", _timelimit] + cmd = [ + _sbatch, "--job-name", job_name, "--nodes", + str(_nodes), "--output", output_filename, "--partition", _partition, + "--time", _timelimit + ] process = subprocess.run(_with_ssh(cmd), input=script, stdout=subprocess.PIPE, From e765d7f2dd77a04492a564ee18534168f8fded2b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 21 Jul 2020 22:23:07 -0400 Subject: [PATCH 08/10] Slurm: Make proper job submission work --- caliban/docker/build.py | 3 ++- caliban/platform/slurm/cli.py | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/caliban/docker/build.py b/caliban/docker/build.py index be0911b..e44debb 100644 --- a/caliban/docker/build.py +++ b/caliban/docker/build.py @@ -36,7 +36,8 @@ t = Terminal() -DEV_CONTAINER_ROOT = "gcr.io/blueshift-playground/blueshift" +# DEV_CONTAINER_ROOT = "gcr.io/blueshift-playground/blueshift" +DEV_CONTAINER_ROOT = "docker.io/eschnett/carpetx-caliban" TF_VERSIONS = {"2.2.0", "1.12.3", "1.14.0", "1.15.0"} DEFAULT_WORKDIR = "/usr/app" CREDS_DIR = "/.creds" diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 2390aed..b145b60 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -42,6 +42,9 @@ _ssh = "ssh" _hostname = "symmetry.pi.local" _username = "eschnetter" +_logdir = "/home/eschnetter/caliban" +_resultdir_host = "/gpfs/eschnetter/caliban-simulations" +_resultdir_container = "/caliban-simulations" # Environment modules _setup_cmds = [["source", "/etc/profile"], ["module", "load", "slurm"]] @@ -138,9 +141,10 @@ def _job_submit(args: dict) -> None: # TODO: Sanitize job_name if job_name is None: job_name = job_prefix - output_filename = job_name + ".log" + log_filename = job_name + ".log" else: - output_filename = job_prefix + "-" + job_name + ".log" + log_filename = job_prefix + "-" + job_name + ".log" + log_filename = _logdir + "/" + log_filename # Arguments to internally build the image required to submit to Slurm docker_m = {'job_mode': job_mode, 'package': package, **docker_args} @@ -166,6 +170,7 @@ def _job_submit(args: dict) -> None: image_id = db.build_image(**docker_m) project_id = args['project_id'] + assert project_id is not None project_s = project_id.replace(":", "/") # TODO: Use sub-project "caliban" or similar? # base = f"docker.io/{project_s}/{image_id}" @@ -186,6 +191,7 @@ def _job_submit(args: dict) -> None: for experiment in experiments: logging.info("*** script") # TODO: Use srun to start job, then use singularity inside srun + qargs = _quote_args(script_args) script = f"""\ #!/bin/bash source /etc/profile @@ -194,17 +200,22 @@ def _job_submit(args: dict) -> None: hostname nproc module load singularity -env PYTHONUNBUFFERED=1 singularity run --tmpdir /gpfs/eschnetter/singularity/tmp --workdir /gpfs/eschnetter/singularity/work --pwd /usr/app docker://{image_tag} exe/cactus_sim arrangements/CactusWave/WaveToyC/par/wavetoyc_rad.par +env PYTHONUNBUFFERED=1 singularity run --tmpdir /gpfs/eschnetter/singularity/tmp --no-home --bind {_resultdir_host}:{_resultdir_container} --pwd /usr/app docker://{image_tag} {qargs} """ logging.info(f"Script is {script}") logging.info("Submitting job...") + prepare_cmd = ["mkdir", "-p", _logdir, _resultdir_host] cmd = [ - _sbatch, "--job-name", job_name, "--nodes", - str(_nodes), "--output", output_filename, "--partition", _partition, + _sbatch, + "--job-name", job_name, + "--nodes", str(_nodes), + "--output", log_filename, + "--partition", _partition, "--time", _timelimit ] - process = subprocess.run(_with_ssh(cmd), + cmds = [prepare_cmd, cmd] + process = subprocess.run(_with_ssh(cmds), input=script, stdout=subprocess.PIPE, universal_newlines=True, @@ -235,10 +246,10 @@ def _join_cmds(cmds: [str]) -> str: # ---------------------------------------------------------------------------- -def _with_ssh(cmd: [str]) -> [str]: +def _with_ssh(cmds: [[str]]) -> [str]: """adds ssh command to execute command remotely""" # Add module load commands - cmds = _setup_cmds + [cmd] + cmds = _setup_cmds + cmds cmd = _join_cmds(map(_quote_args, cmds)) # Add ssh command prefix assert _hostname[0] != '-' From 82f67cd9db1352563da44ab46c6e51c8799787ca Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 22 Jul 2020 11:07:34 -0400 Subject: [PATCH 09/10] Reformat code --- caliban/platform/slurm/cli.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index b145b60..2f2f967 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -207,11 +207,8 @@ def _job_submit(args: dict) -> None: logging.info("Submitting job...") prepare_cmd = ["mkdir", "-p", _logdir, _resultdir_host] cmd = [ - _sbatch, - "--job-name", job_name, - "--nodes", str(_nodes), - "--output", log_filename, - "--partition", _partition, + _sbatch, "--job-name", job_name, "--nodes", + str(_nodes), "--output", log_filename, "--partition", _partition, "--time", _timelimit ] cmds = [prepare_cmd, cmd] From 316d1a973dadcd33f028acb93397112e30d44c6f Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 18 Aug 2020 13:14:46 -0400 Subject: [PATCH 10/10] Slurm: Specify tasks and threads --- caliban/platform/slurm/cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/caliban/platform/slurm/cli.py b/caliban/platform/slurm/cli.py index 2f2f967..31f5222 100644 --- a/caliban/platform/slurm/cli.py +++ b/caliban/platform/slurm/cli.py @@ -56,6 +56,8 @@ _partition = "debugq" _timelimit = "00:10:00" _nodes = 1 +_ntasks_per_node = 40 +_nthreads_per_task = 1 # ---------------------------------------------------------------------------- @@ -206,12 +208,17 @@ def _job_submit(args: dict) -> None: logging.info("Submitting job...") prepare_cmd = ["mkdir", "-p", _logdir, _resultdir_host] + env_cmd = [ + "export", "CALIBAN_NPROCS=" + str(_nodes * _ntasks_per_node), + "CALIBAN_NTHREADS_PER_PROC=" + str(_nthreads_per_task) + ] cmd = [ _sbatch, "--job-name", job_name, "--nodes", - str(_nodes), "--output", log_filename, "--partition", _partition, - "--time", _timelimit + str(_nodes), "--ntasks-per-node", + str(_ntasks_per_node), "--output", log_filename, "--partition", + _partition, "--time", _timelimit ] - cmds = [prepare_cmd, cmd] + cmds = [prepare_cmd, env_cmd, cmd] process = subprocess.run(_with_ssh(cmds), input=script, stdout=subprocess.PIPE,