diff --git a/caliban/cli.py b/caliban/cli.py index ff26d9c..b5fbe3b 100644 --- a/caliban/cli.py +++ b/caliban/cli.py @@ -209,6 +209,15 @@ def image_tag_arg(parser): "Caliban will skip the build and push steps and use this image tag.") +def dlvm_arg(parser): + dlvm_types = [key for key in sorted(conf._dlvm_config('CPU').keys())] + parser.add_argument( + "--dlvm", + help="DLVM base image type. Must be one of " + "{}".format(dlvm_types) + ". If supplied, " + "Caliban will skip the build and push steps and use this image tag.") + + def machine_type_arg(parser): machine_types = u.enum_vals(ct.MachineType) cpu_default = conf.DEFAULT_MACHINE_TYPE[conf.JobMode.CPU].value @@ -259,6 +268,7 @@ def shell_parser(base): parser = base.add_parser( "shell", help="Start an interactive shell with this dir mounted.") base_parser(parser) + dlvm_arg(parser) image_id_arg(parser) docker_run_arg(parser) parser.add_argument( @@ -281,6 +291,7 @@ def notebook_parser(base): help="Run a local Jupyter notebook instance.") base_parser(parser) docker_run_arg(parser) + dlvm_arg(parser) # Custom notebook arguments. parser.add_argument( @@ -313,6 +324,7 @@ def local_run_parser(base): """Configure the subparser for `caliban run`.""" parser = base.add_parser("run", help="Run a job inside a Docker container.") executing_parser(parser) + dlvm_arg(parser) image_id_arg(parser) docker_run_arg(parser) xgroup_submit_arg(parser) @@ -379,6 +391,7 @@ def container_parser(parser): image_tag_arg(parser) project_id_arg(parser) region_arg(parser) + dlvm_arg(parser) machine_type_arg(parser) gpu_spec_arg(parser) tpu_spec_arg(parser) diff --git a/caliban/cloud/core.py b/caliban/cloud/core.py index b14743c..d895fee 100644 --- a/caliban/cloud/core.py +++ b/caliban/cloud/core.py @@ -486,7 +486,7 @@ def build_job_specs( experiments=experiments) -def generate_image_tag(project_id, docker_args, dry_run: bool = False): +def generate_image_tag(project_id, docker_args, dlvm: str = None, dry_run: bool = False): """Generates a new Docker image and pushes an image to the user's GCloud Container Repository, tagged using the UUID of the generated image. @@ -501,7 +501,7 @@ def generate_image_tag(project_id, docker_args, dry_run: bool = False): logging.info("Dry run - skipping actual 'docker build' and 'docker push'.") image_tag = "dry_run_tag" else: - image_id = d.build_image(**docker_args) + image_id = d.build_image(dlvm=dlvm, **docker_args) image_tag = d.push_uuid_tag(project_id, image_id) return image_tag @@ -551,6 +551,7 @@ def submit_ml_job(job_mode: conf.JobMode, credentials_path: Optional[str] = None, dry_run: bool = False, job_name: Optional[str] = None, + dlvm: Optional[str] = None, machine_type: Optional[ct.MachineType] = None, gpu_spec: Optional[ct.GPUSpec] = None, tpu_spec: Optional[ct.TPUSpec] = None, @@ -633,7 +634,9 @@ def submit_ml_job(job_mode: conf.JobMode, with session_scope(engine) as session: container_spec = generate_container_spec(session, docker_args, image_tag) - if image_tag is None: + if dlvm is not None: + image_tag = generate_image_tag(project_id, docker_args, dlvm=dlvm, dry_run=dry_run) + elif image_tag is None: image_tag = generate_image_tag(project_id, docker_args, dry_run=dry_run) experiments = create_experiments( diff --git a/caliban/config.py b/caliban/config.py index a602c40..76a7142 100644 --- a/caliban/config.py +++ b/caliban/config.py @@ -78,6 +78,33 @@ class JobMode(str, Enum): } +def extract_dlvm_image(job_mode: JobMode, dlvm_arg: str) -> str: + """Returns the DLVM image url for the job model and the comand line arg + or returns None if the key doesn't exist in the config. + + """ + return _dlvm_config(job_mode).get(dlvm_arg) + + +def _dlvm_config(job_mode: JobMode) -> Dict[str, str]: + job_mode_str = job_mode.lower() + return { + "pytorch": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}", + "pytorch-1.0": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}.1-0", + "pytorch-1.1": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}.1-1", + "pytorch-1.2": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}.1-2", + "pytorch-1.3": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}.1-3", + "pytorch-1.4": f"gcr.io/deeplearning-platform-release/pytorch-{job_mode_str}.1-4", + "tf": f"gcr.io/deeplearning-platform-release/tf-{job_mode_str}", + "tf-1.13": f"gcr.io/deeplearning-platform-release/tf-{job_mode_str}.1-13", + "tf-1.14": f"gcr.io/deeplearning-platform-release/tf-{job_mode_str}.1-14", + "tf-1.15": f"gcr.io/deeplearning-platform-release/tf-{job_mode_str}.1-15", + "tf2": f"gcr.io/deeplearning-platform-release/tf2-{job_mode_str}", + "tf-2.0": f"gcr.io/deeplearning-platform-release/tf2-{job_mode_str}.2-0", + "tf-2.1": f"gcr.io/deeplearning-platform-release/tf2-{job_mode_str}.2-1", + "tf-2.2": f"gcr.io/deeplearning-platform-release/tf2-{job_mode_str}.2-2" + } + def gpu(job_mode: JobMode) -> bool: """Returns True if the supplied JobMode is JobMode.GPU, False otherwise. diff --git a/caliban/docker.py b/caliban/docker.py index e17ef24..81b8577 100644 --- a/caliban/docker.py +++ b/caliban/docker.py @@ -378,7 +378,7 @@ def _credentials_entries(user_id: int, return ret -def _notebook_entries(lab: bool = False, version: Optional[str] = None) -> str: +def _notebook_entries(lab: bool = False, version: Optional[str] = None, dlvm: bool = False) -> str: """Returns the Dockerfile entries necessary to install Jupyter{lab}. Optionally takes a version string. @@ -391,9 +391,17 @@ def _notebook_entries(lab: bool = False, version: Optional[str] = None) -> str: library = "jupyterlab" if lab else "jupyter" - return """ + if not dlvm: + return """ RUN pip install {}{} """.format(library, version_suffix) + else: + return """ +RUN /opt/conda/bin/pip install \ + https://storage.googleapis.com/deeplearning-platform-ui-public/jupyterlab_gcpscheduler-1.0.0.tar.gz + +RUN /opt/conda/bin/jupyter lab build --minimize=False + """ def _custom_packages( @@ -461,9 +469,17 @@ def _extra_dir_entries(workdir: str, user_id: int, user_group: int, return ret +def _dlvm_id(job_mode: c.JobMode, dlvm_arg: str) -> str: + """Returns the DLVM image url for job mode and the command line parameter + + """ + return c.extract_dlvm_image(job_mode, dlvm_arg) + + def _dockerfile_template( job_mode: c.JobMode, workdir: Optional[str] = None, + dlvm: Optional[str] = None, base_image_fn: Optional[Callable[[c.JobMode], str]] = None, package: Optional[Union[List, u.Package]] = None, requirements_path: Optional[str] = None, @@ -480,9 +496,8 @@ def _dockerfile_template( on the value of job_mode) to create a container that: - installs any dependency specified in a requirements.txt file living at - requirements_path, a conda environment at conda_env_path, or any - dependencies in a setup.py file, including extra dependencies, if - setup_extras isn't None + requirements_path, or any dependencies in a setup.py file, including extra + dependencies, if setup_extras isn't None - injects gcloud credentials into the container, so Cloud interaction works just like it does locally - potentially installs a custom shell, or jupyterlab for notebook support @@ -511,7 +526,10 @@ def _dockerfile_template( if base_image_fn is None: base_image_fn = base_image_id - base_image = base_image_fn(job_mode) + if dlvm is None: + base_image = base_image_fn(job_mode) + else: + base_image = _dlvm_id(job_mode, dlvm) dockerfile = """ FROM {base_image} @@ -530,8 +548,6 @@ def _dockerfile_template( ENV HOME={c_home} WORKDIR {workdir} - -USER {uid}:{gid} """.format_map({ "base_image": base_image, "username": username, @@ -545,6 +561,19 @@ def _dockerfile_template( gid, adc_path=adc_path, credentials_path=credentials_path) + if inject_notebook.value != 'none': + install_lab = inject_notebook == NotebookInstall.lab + if dlvm is None: + dockerfile += _notebook_entries(lab=install_lab, version=jupyter_version, dlvm=False) + else: + dockerfile += _notebook_entries(lab=install_lab, version=jupyter_version, dlvm=True) + + dockerfile += """ + +USER {uid}:{gid} +""".format_map({"uid": uid, + "gid": gid + }) dockerfile += _dependency_entries(workdir, uid, @@ -589,6 +618,7 @@ def build_image(job_mode: c.JobMode, credentials_path: Optional[str] = None, adc_path: Optional[str] = None, no_cache: bool = False, + dlvm: str = None, **kwargs) -> str: """Builds a Docker image by generating a Dockerfile and passing it to `docker build` via stdin. All output from the `docker build` process prints to @@ -608,6 +638,7 @@ def build_image(job_mode: c.JobMode, dockerfile = _dockerfile_template(job_mode, credentials_path=creds, adc_path=adc, + dlvm=dlvm, **kwargs) joined_cmd = " ".join(cmd) @@ -889,6 +920,7 @@ def run(job_mode: c.JobMode, run_args: Optional[List[str]] = None, script_args: Optional[List[str]] = None, image_id: Optional[str] = None, + dlvm: Optional[str] = None, **build_image_kwargs) -> None: """Builds an image using the supplied **build_image_kwargs and calls `docker run` on the resulting image using sensible defaults. @@ -907,7 +939,9 @@ def run(job_mode: c.JobMode, if script_args is None: script_args = [] - if image_id is None: + if dlvm is not None: + image_id = build_image(job_mode, dlvm=dlvm, **build_image_kwargs) + elif image_id is None: image_id = build_image(job_mode, **build_image_kwargs) base_cmd = _run_cmd(job_mode, run_args) @@ -922,6 +956,7 @@ def run(job_mode: c.JobMode, def run_interactive(job_mode: c.JobMode, workdir: Optional[str] = None, image_id: Optional[str] = None, + dlvm: Optional[str] = None, run_args: Optional[List[str]] = None, mount_home: Optional[bool] = None, shell: Optional[Shell] = None, @@ -936,6 +971,8 @@ def run_interactive(job_mode: c.JobMode, - job_mode: c.JobMode. - image_id: ID of the image to run. Supplying this will skip an image build. - run_args: extra arguments to supply to `docker run`. + - dlvm: key of the base DLVM image to run. Supplying this perform an image + build using DLVM as the base image instead of blueshift - mount_home: if true, mounts the user's $HOME directory into the container to `/home/$USERNAME`. If False, nothing. - shell: name of the shell to install into the container. Also configures the @@ -967,15 +1004,101 @@ def run_interactive(job_mode: c.JobMode, if entrypoint is None: entrypoint = SHELL_DICT[shell].executable - interactive_run_args = _interactive_opts(workdir) + [ + if dlvm is None: + # Pass in the entrypoint if not using DLVM + # Otherwise set the DLVM entrypoint from the shell argument + interactive_run_args = _interactive_opts(workdir) + [ "-it", \ "--entrypoint", entrypoint - ] + _home_mount_cmds(mount_home) + run_args + ] + _home_mount_cmds(mount_home) + run_args + else: + entrypoint = SHELL_DICT[shell].executable + interactive_run_args = _interactive_opts(workdir) + [ + "-it", \ + "--entrypoint", entrypoint + ] + _home_mount_cmds(mount_home) + run_args + entrypoint_args = [] + + run(job_mode=job_mode, + run_args=interactive_run_args, + script_args=entrypoint_args, + image_id=image_id, + dlvm=dlvm, + shell=shell, + workdir=workdir, + **build_image_kwargs) + + +def run_notebook_interactive(job_mode: c.JobMode, + workdir: Optional[str] = None, + image_id: Optional[str] = None, + dlvm: Optional[str] = None, + run_args: Optional[List[str]] = None, + mount_home: Optional[bool] = None, + shell: Optional[Shell] = None, + entrypoint: Optional[str] = None, + entrypoint_args: Optional[List[str]] = None, + **build_image_kwargs) -> None: + """Start a live shell in the terminal, with all dependencies installed and the + current working directory (and optionally the user's home directory) mounted. + + Keyword args: + + - job_mode: c.JobMode. + - image_id: ID of the image to run. Supplying this will skip an image build. + - dlvm: key of the base DLVM image to run. Supplying this perform an image + build using DLVM as the base image instead of blueshift + - run_args: extra arguments to supply to `docker run`. + - mount_home: if true, mounts the user's $HOME directory into the container + to `/home/$USERNAME`. If False, nothing. + - shell: name of the shell to install into the container. Also configures the + entrypoint if that's not supplied. + - entrypoint: command to run. Defaults to the executable command for the + supplied shell. + - entrypoint_args: extra arguments to supply to the entrypoint. + + any extra kwargs supplied are passed through to build_image. + + """ + if workdir is None: + workdir = DEFAULT_WORKDIR + + if run_args is None: + run_args = [] + + if entrypoint_args is None: + entrypoint_args = [] + + if mount_home is None: + mount_home = True + + if shell is None: + # Only set a default shell if we're also mounting the home volume. + # Otherwise a custom shell won't have access to the user's profile. + shell = default_shell() if mount_home else Shell.bash + + if entrypoint is None: + entrypoint = SHELL_DICT[shell].executable + + if dlvm is None: + # Run vanilla jupyter/jupyterlab notebook if not using DLVM + # Otherwise use the jupyterlab notebook from DLVM with the scheduler + # extension. + interactive_run_args = _interactive_opts(workdir) + [ + "-it", \ + "--entrypoint", entrypoint + ] + _home_mount_cmds(mount_home) + run_args + else: + interactive_run_args = _interactive_opts(workdir) + [ + "-it", \ + ] + _home_mount_cmds(mount_home) + run_args + entrypoint_args = [] run(job_mode=job_mode, run_args=interactive_run_args, script_args=entrypoint_args, image_id=image_id, + dlvm=dlvm, shell=shell, workdir=workdir, **build_image_kwargs) @@ -986,6 +1109,7 @@ def run_notebook(job_mode: c.JobMode, lab: Optional[bool] = None, version: Optional[bool] = None, run_args: Optional[List[str]] = None, + dlvm: Optional[str] = None, **run_interactive_kwargs) -> None: """Start a notebook in the current working directory; the process will run inside of a Docker container that's identical to the environment available to @@ -1024,7 +1148,8 @@ def run_notebook(job_mode: c.JobMode, ] docker_args = ["-p", "{}:{}".format(port, port)] + run_args - run_interactive(job_mode, + run_notebook_interactive(job_mode, + dlvm=dlvm, entrypoint="/opt/conda/envs/caliban/bin/python", entrypoint_args=jupyter_args, run_args=docker_args, diff --git a/caliban/history/utils.py b/caliban/history/utils.py index 5b7737c..543e83f 100644 --- a/caliban/history/utils.py +++ b/caliban/history/utils.py @@ -180,6 +180,7 @@ def generate_container_spec( session: Session, docker_args: Dict[str, Any], image_tag: Optional[str] = None, + dlvm_tag: Optional[str] = None, ) -> ContainerSpec: '''generates a container spec @@ -192,7 +193,9 @@ def generate_container_spec( ContainerSpec instance ''' - if image_tag is None: + if dlvm_tag is not None: + spec = {'image_id': dlvm_tag} + elif image_tag is None: spec = docker_args else: spec = {'image_id': image_tag} diff --git a/caliban/main.py b/caliban/main.py index d1120ff..3ee0cc7 100644 --- a/caliban/main.py +++ b/caliban/main.py @@ -57,8 +57,10 @@ def run_app(arg_input): if command == "shell": mount_home = not args['bare'] image_id = args.get("image_id") + dlvm = args.get("dlvm") shell = args['shell'] docker.run_interactive(job_mode, + dlvm=dlvm, image_id=image_id, run_args=docker_run_args, mount_home=mount_home, @@ -68,9 +70,11 @@ def run_app(arg_input): elif command == "notebook": port = args.get("port") lab = args.get("lab") + dlvm = args.get("dlvm") version = args.get("jupyter_version") mount_home = not args['bare'] docker.run_notebook(job_mode, + dlvm=dlvm, port=port, lab=lab, version=version, @@ -95,6 +99,7 @@ def run_app(arg_input): dry_run = args["dry_run"] package = args["module"] image_id = args.get("image_id") + dlvm = args.get("dlvm") exp_config = args.get("experiment_config") xgroup = args.get('xgroup') @@ -102,6 +107,7 @@ def run_app(arg_input): run_args=docker_run_args, script_args=script_args, image_id=image_id, + dlvm=dlvm, experiment_config=exp_config, dry_run=dry_run, package=package, @@ -120,6 +126,7 @@ def run_app(arg_input): tpu_spec = args.get("tpu_spec") image_tag = args.get("image_tag") machine_type = args.get("machine_type") + dlvm = args.get("dlvm") exp_config = args.get("experiment_config") labels = u.sanitize_labels(args.get("label") or []) xgroup = args.get('xgroup') @@ -135,6 +142,7 @@ def run_app(arg_input): credentials_path=cloud_key, dry_run=dry_run, job_name=job_name, + dlvm=dlvm, machine_type=machine_type, gpu_spec=gpu_spec, tpu_spec=tpu_spec,