diff --git a/.gitignore b/.gitignore
index fc9e1f65..da6114bf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
# Radical EnTK
re.session*
+conda-env/
+
# Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,visualstudiocode,visualstudio,pycharm,pycharm+all,pycharm+iml,pydev,macos,emacs,windows,linux
# Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,visualstudiocode,visualstudio,pycharm,pycharm+all,pycharm+iml,pydev,macos,emacs,windows,linux
diff --git a/bin/powerai.sh b/bin/powerai.sh
new file mode 100644
index 00000000..e2d21a58
--- /dev/null
+++ b/bin/powerai.sh
@@ -0,0 +1,78 @@
+module load gcc/7.3.1
+. /etc/profile.d/conda.sh
+conda activate /usr/workspace/cv_ddmd/conda3/powerai
+
+export IBM_POWERAI_LICENSE_ACCEPT=yes
+module use /usr/workspace/cv_ddmd/software1/modules
+module load adios2/2.8.1a
+
+export WORKSPACE=/usr/workspace/$USER
+export SHARED=/usr/workspace/cv_ddmd/$USER
+export GPFS1=/p/gpfs1/$USER
+export SB=$GPFS1/radical.pilot.sandbox
+
+export MyDir=$1
+export MyDir1=$GPFS1/DDMD/$MyDir/entk_cvae_md
+export MyDir2=$MyDir1/analysis
+
+export PYTHONPATH=$MyDir1/Outlier_search:$PYTHONPATH
+
+export PATH=$HOME/bin:$HOME/spack/bin:$PATH
+
+shopt -s direxpand
+
+alias wks='cd $WORKSPACE'
+alias gpfs='cd $GPFS1'
+
+alias sb='cd $SB'
+alias sb1='cd $SB/$(ls -d $MyDir1/re.* | xargs basename)/pilot.0000'
+
+alias cd1='cd $MyDir1'
+alias cd2='cd $MyDir2'
+alias cd3='cd $SHARED'
+
+alias bu='bkill -u $USER'
+
+alias topU='top -U $USER'
+alias bj='bjobs'
+alias current='cd /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/'
+alias sim='cd /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream'
+alias agg='cd /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream'
+alias ml='cd /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream'
+alias outliers='cd /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream'
+alias outputs_ls='echo "/p/gpfs1/yakushin/Outputs"; ls -ltr /p/gpfs1/yakushin/Outputs'
+alias outputs='cd /p/gpfs1/yakushin/Outputs'
+
+function mcd1()
+{
+ cd $GPFS1/DDMD/$1/entk_cvae_md
+}
+
+function mcd2()
+{
+ cd $GPFS1/DDMD/$1/entk_cvae_md/analysis
+}
+
+function msb1()
+{
+ cd $SB/$(ls -d $GPFS1/DDMD/$1/entk_cvae_md/re.* | xargs basename)/pilot.0000
+}
+
+
+function em
+{
+ emacs -nw +$2:1 $1
+}
+
+function logs()
+{
+ cd /p/gpfs1/yakushin/radical.pilot.sandbox
+ cd $(ls -tr | tail -1)
+ cd pilot*
+}
+
+function logs1()
+{
+ cd /p/gpfs1/yakushin/radical.pilot.sandbox
+ cd $1/pilot*
+}
diff --git a/deepdrivemd/agents/stream/config.py b/deepdrivemd/agents/stream/config.py
index 51959eee..822305cd 100644
--- a/deepdrivemd/agents/stream/config.py
+++ b/deepdrivemd/agents/stream/config.py
@@ -76,18 +76,6 @@ class OutlierDetectionConfig(AgentTaskConfig):
model: str = "cvae"
num_points: int = 539
num_features: int = 0
- """
- encoder_bias: bool = True
- encoder_relu_slope: float = 0.0
- encoder_filters: List[int] = [64, 128, 256, 256, 512]
- encoder_kernels: List[int] = [5, 5, 3, 1, 1]
- decoder_bias: bool = True
- decoder_relu_slope: float = 0.0
- decoder_affine_widths: List[int] = [64, 128, 512, 1024]
- discriminator_bias: bool = True
- discriminator_relu_slope: float = 0.0
- discriminator_affine_widths: List[int] = [512, 128, 64]
- """
if __name__ == "__main__":
diff --git a/deepdrivemd/agents/stream/dbscan.py b/deepdrivemd/agents/stream/dbscan.py
index 1d242c0b..5b61ae0c 100644
--- a/deepdrivemd/agents/stream/dbscan.py
+++ b/deepdrivemd/agents/stream/dbscan.py
@@ -20,7 +20,7 @@
from mdlearn.nn.models.aae.point_3d_aae import AAE3d
from numba import cuda
from pathos.multiprocessing import ProcessingPool as Pool
-from simtk.openmm.app.pdbfile import PDBFile
+from openmm.app.pdbfile import PDBFile
from sklearn.neighbors import LocalOutlierFactor
from torchsummary import summary
@@ -116,9 +116,10 @@ def wait_for_model(cfg: OutlierDetectionConfig) -> str:
while True:
if os.path.exists(cfg.best_model):
break
- print(f"No model {cfg.best_model}, sleeping")
- sys.stdout.flush()
- time.sleep(cfg.timeout2)
+ if(os.getenv('DDMD_DEBUG') == None):
+ print(f"No model {cfg.best_model}, sleeping")
+ sys.stdout.flush()
+ time.sleep(cfg.timeout2)
return cfg.best_model
@@ -251,7 +252,6 @@ def outliers_from_latent(
db_label = db.labels_.to_array()
print("unique labels = ", np.unique(db_label))
outlier_list = np.where(db_label == -1)
- # clear_gpu()
return outlier_list
@@ -334,19 +334,8 @@ def write_pdb_frame(
)
sys.stdout.flush()
- """
- import MDAnalysis as mda
- with Timer("universe"):
- mda_u = mda.Universe(str(original_pdb))
- print(np.max(frame, axis=0) - np.min(frame, axis=0)); sys.stdout.flush()
- mda_u.atoms.positions = frame
-
- with Timer("write_pdb"):
- mda_u.atoms.write(str(output_pdb_fn))
- """
np.save(str(output_pdb_fn), frame)
-
def write_pdb_frame_2(
frame: np.ndarray, original_pdb: Path, output_pdb_fn: str, ligand: int
):
@@ -611,15 +600,6 @@ def top_lof(
print(e)
outlier_list = list(filter(lambda x: x < len(cm_predict), outlier_list))
projections = cm_predict[outlier_list]
- """
- print("type(projections)=", type(projections))
- print("dir(projections)=", dir(projections))
- print("projections=", projections)
- projections = projections.cpu()
- print("type(projections) = ", type(projections))
- projections = projections.detach().numpy()
- print("type(projections) = ", type(projections))
- """
lof_scores = run_lof(projections)
print("lof_scores = ", lof_scores)
@@ -715,7 +695,6 @@ def main(cfg: OutlierDetectionConfig):
adios_files_list = list(map(lambda x: x.replace(".sst", ""), adios_files_list))
variable_list = [
- # StreamContactMapVariable("contact_map", np.uint8, DataStructure.array),
StreamVariable("positions", np.float32, DataStructure.array),
StreamVariable("md5", str, DataStructure.string),
StreamVariable("velocities", np.float32, DataStructure.array),
@@ -752,11 +731,13 @@ def main(cfg: OutlierDetectionConfig):
try:
agg_input = mystreams.next()
except: # noqa TODO: flake8 - should not have a bar except
- print("Sleeping for input")
- time.sleep(60)
+ if(os.getenv('DDMD_DEBUG') == None):
+ print("Sleeping for input")
+ time.sleep(60)
continue
if len(agg_input[list(agg_input.keys())[0]]) < 10:
- time.sleep(30)
+ if(os.getenv('DDMD_DEBUG') == None):
+ time.sleep(30)
else:
break
@@ -785,7 +766,6 @@ def main(cfg: OutlierDetectionConfig):
or len(outlier_list[0]) < cfg.num_sim
):
print("Not using outliers")
- # clear_gpu()
if cfg.compute_rmsd:
print("Using best rmsd states")
outlier_list = [select_best(cfg, agg_input)]
@@ -823,7 +803,8 @@ def main(cfg: OutlierDetectionConfig):
with Timer("outlier_publish"):
publish(tmp_dir, published_dir)
- time.sleep(random.randint(250, 350))
+ if(os.getenv('DDMD_DEBUG') == None):
+ time.sleep(random.randint(250, 350))
with Timer("outlier_read"):
agg_input = mystreams.next()
@@ -915,17 +896,6 @@ def read_lastN( # noqa
variable_lists[vl] = np.vstack(variable_lists[vl])
print(len(variable_lists[vl]))
- """
- variable_lists["contact_map"] = np.array(
- list(
- map(
- lambda x: t1Dto2D(np.unpackbits(x.astype("uint8"))),
- list(variable_lists["contact_map"]),
- )
- )
- )
- """
-
if not variable_lists:
return {}
@@ -994,62 +964,6 @@ def project_mini(cfg: OutlierDetectionConfig, trajectory: str):
np.save(f, embeddings)
-"""
-
-def project_mini(cfg: OutlierDetectionConfig):
- # with Timer("wait_for_input"):
- # adios_files_list = wait_for_input(cfg)
-
- ddd = os.path.dirname(cfg.agg_dir)
-
- adios_files_list = glob.glob(
- f"{ddd}/molecular_dynamics_runs/stage0000/task0*/*/trajectory.bp"
- )
-
- adios_files_list = glob.glob(f"{ddd}/aggregation_runs/stage0000/task0*/agg.bp")
-
- with Timer("wait_for_model"):
- model_path = str(wait_for_model(cfg))
- print("model_path = ", model_path)
-
- lastN = cfg.project_lastN
-
- # Create output directories
- dirs(cfg)
-
- for i, bp in enumerate(adios_files_list):
- print(f"i={i}, bp={bp}")
- sys.stdout.flush()
- agg_input = read_lastN([bp], lastN)
-
- if hasattr(cfg, "compute_zcentroid") and cfg.compute_zcentroid:
- zcentroid = agg_input["zcentroid"]
- with open(cfg.output_path / f"zcentroid_{i}.npy", "wb") as f:
- np.save(f, zcentroid)
-
- if cfg.compute_rmsd:
- rmsds = agg_input["rmsds"]
- with open(cfg.output_path / f"rmsd_{i}.npy", "wb") as f:
- np.save(f, rmsds)
-
- if hasattr(cfg, "multi_ligand_table") and cfg.multi_ligand_table.is_file():
- ligand = agg_input["ligand"]
- sim = agg_input["dirs"]
- for j in range(len(ligand)):
- print(f"ligand[{j}] = {ligand[j]}")
- if ligand[j] == -1:
- ligand[j] = int(sim[j])
- with open(cfg.output_path / f"ligand_{i}.npy", "wb") as f:
- np.save(f, ligand)
-
- with Timer("project_predict"):
- embeddings_cvae = predict(cfg, model_path, agg_input, batch_size=64)
- with open(cfg.output_path / f"embeddings_cvae_{i}.npy", "wb") as f:
- np.save(f, embeddings_cvae)
-
-"""
-
-
def project(cfg: OutlierDetectionConfig):
"""Postproduction: compute t-SNE embeddings."""
if cfg.project_gpu:
@@ -1120,26 +1034,18 @@ def project_tsne_2D(cfg: OutlierDetectionConfig):
tsne2 = TSNE(n_components=2)
emb = []
rmsds = []
- # ligands = []
embs = glob.glob(str(cfg.output_path) + "/embeddings_cvae_*.npy")
for i in range(len(embs)):
with open(cfg.output_path / f"embeddings_cvae_{i}.npy", "rb") as f:
emb.append(np.load(f))
with open(cfg.output_path / f"rmsd_{i}.npy", "rb") as f:
rmsds.append(np.load(f))
- """
- with open(cfg.output_path / f"ligand_{i}.npy", "rb") as f:
- ligands.append(np.load(f))
- """
+
embeddings_cvae = np.concatenate(emb)
RMSDS = np.concatenate(rmsds)
with open(cfg.output_path / "rmsds.npy", "wb") as f:
np.save(f, RMSDS)
- """
- LIGANDS = np.concatenate(ligands)
- with open(cfg.output_path / "ligands.npy", "wb") as f:
- np.save(f, LIGANDS)
- """
+
with Timer("project_TSNE_2D"):
tsne_embeddings2 = tsne2.fit_transform(embeddings_cvae)
diff --git a/deepdrivemd/aggregation/stream/aggregator.py b/deepdrivemd/aggregation/stream/aggregator.py
index 81c17190..35402d5a 100644
--- a/deepdrivemd/aggregation/stream/aggregator.py
+++ b/deepdrivemd/aggregation/stream/aggregator.py
@@ -175,17 +175,6 @@ def aggregate(
status = ARW.read_step(sim_task_id)
if status:
ARW.d_md5 = intarray2hash(ARW.d_md5)
- """
- print("dir(ARW) = ", dir(ARW))
- sys.stdout.flush()
- print("contact_map.shape = ", ARW.d_contact_map.shape)
- print("contact_map.dtype = ", ARW.d_contact_map.dtype)
- print(ARW.d_contact_map)
- print("positions.shape = ", ARW.d_positions.shape)
- print("positions.dtype = ", ARW.d_positions.dtype)
- sys.stdout.flush()
- """
-
ARW.write_step(aggregator_stream_4ml, variablesW_4ml, end_step=True)
ARW.write_step(aggregator_stream, variablesW, end_step=False)
aggregator_stream.write("dir", str(sim_task_id), end_step=True)
diff --git a/deepdrivemd/data/stream/adios_utils.py b/deepdrivemd/data/stream/adios_utils.py
index e10907f2..2aa5d390 100644
--- a/deepdrivemd/data/stream/adios_utils.py
+++ b/deepdrivemd/data/stream/adios_utils.py
@@ -9,7 +9,11 @@
class AdiosStreamStepRW:
- """Read/Write step by step adios stream using Full API.
+ """Read/Write step by step adios stream using Full API. Full API is needed not to block if some
+ adios stream does not have data yet: check the status of the adios stream, if the next simulation step
+ is not available there yet, go to the next stream and revisit the current stream later.
+ The status can only be checked in Full ADIOS API. High level API can only read in blocking or
+ non-blocking way.
Attributes
----------
@@ -65,13 +69,10 @@ def read_step(self, sim_task_id: int) -> bool:
if not (status == adios2.StepStatus.OK):
return False
- # print(self.variables)
-
for v in self.variables:
vname = "var_" + v
dname = "d_" + v
dtype = self.variables[v][0]
- # print("vname = ", vname); import sys; sys.stdout.flush()
structure_type = self.variables[v][1]
setattr(self, vname, io.InquireVariable(v))
if structure_type == DataStructure.scalar:
diff --git a/deepdrivemd/data/stream/aggregator_reader.py b/deepdrivemd/data/stream/aggregator_reader.py
index 904f9786..e42eedca 100644
--- a/deepdrivemd/data/stream/aggregator_reader.py
+++ b/deepdrivemd/data/stream/aggregator_reader.py
@@ -1,4 +1,3 @@
-# from deepdrivemd.utils import t1Dto2D
from pathlib import Path
from typing import Dict, List, Union
@@ -62,8 +61,6 @@ class StreamContactMapVariable(StreamVariable):
def next(self, ARW):
var = getattr(ARW, "d_" + self.name)
- # var = np.unpackbits(var)
- # var = t1Dto2D(var)
self.total.append(var)
diff --git a/deepdrivemd/deepdrivemd.py b/deepdrivemd/deepdrivemd.py
index d8f77c22..b492c24a 100644
--- a/deepdrivemd/deepdrivemd.py
+++ b/deepdrivemd/deepdrivemd.py
@@ -256,18 +256,25 @@ def generate_agent_stage(self) -> Stage:
"Invalid RMQ environment. Please see README.md for configuring environment."
)
- # Calculate total number of nodes required. Assumes 1 MD job per GPU
- # TODO: fix this assumption for NAMD
- num_full_nodes, extra_gpus = divmod(
- cfg.molecular_dynamics_stage.num_tasks, cfg.gpus_per_node
- )
- extra_node = int(extra_gpus > 0)
- num_nodes = max(1, num_full_nodes + extra_node)
+ # Calculate total number of nodes required.
+ # If gpus_per_node is 0, then we assume that the CPU is used for
+ # simulation, in which case we request a node per simulation task.
+ # Otherwise, we assume that each simulation task uses a single GPU.
+ if cfg.gpus_per_node == 0:
+ num_nodes = cfg.molecular_dynamics_stage.num_tasks
+ else:
+ num_nodes, extra_gpus = divmod(
+ cfg.molecular_dynamics_stage.num_tasks, cfg.gpus_per_node
+ )
+ # If simulations don't pack evenly onto nodes, add an extra node
+ num_nodes += int(extra_gpus > 0)
+
+ num_nodes = max(1, num_nodes)
appman.resource_desc = {
"resource": cfg.resource,
"queue": cfg.queue,
- "schema": cfg.schema_,
+ "access_schema": cfg.schema_,
"walltime": cfg.walltime_min,
"project": cfg.project,
"cpus": cfg.cpus_per_node * cfg.hardware_threads_per_cpu * num_nodes,
diff --git a/deepdrivemd/deepdrivemd_stream.py b/deepdrivemd/deepdrivemd_stream.py
index 86f26375..3f854d36 100644
--- a/deepdrivemd/deepdrivemd_stream.py
+++ b/deepdrivemd/deepdrivemd_stream.py
@@ -263,7 +263,7 @@ def compute_number_of_nodes(cfg: StreamingExperimentConfig) -> int:
appman.resource_desc = {
"resource": cfg.resource,
"queue": cfg.queue,
- "schema": cfg.schema_,
+ "access_schema": cfg.schema_,
"walltime": cfg.walltime_min,
"project": cfg.project,
"cpus": cfg.cpus_per_node * cfg.hardware_threads_per_cpu * num_nodes,
diff --git a/deepdrivemd/models/aae_stream/train.py b/deepdrivemd/models/aae_stream/train.py
index c766c0cb..51f6c220 100644
--- a/deepdrivemd/models/aae_stream/train.py
+++ b/deepdrivemd/models/aae_stream/train.py
@@ -92,8 +92,6 @@ def next_input(
num_workers=cfg.num_data_workers,
drop_last=True,
pin_memory=True,
- # persistent_workers=True,
- # prefetch_factor=cfg.prefetch_factor,
)
print("len(train_loader) = ", len(train_loader))
@@ -347,7 +345,6 @@ def main(cfg: Point3dAAEConfig):
checkpoint_path = cfg.checkpoint_dir / "best.pt"
if checkpoint_path.exists():
- # shutil.move(str(checkpoint_path), str(cfg.published_model_dir))
subprocess.getstatusoutput(
f"mv {checkpoint_path} {cfg.published_model_dir}/"
)
diff --git a/deepdrivemd/models/keras_cvae_stream/train.py b/deepdrivemd/models/keras_cvae_stream/train.py
index 70b0cb3a..3ec717c2 100644
--- a/deepdrivemd/models/keras_cvae_stream/train.py
+++ b/deepdrivemd/models/keras_cvae_stream/train.py
@@ -32,39 +32,14 @@ def wait_for_input(cfg: KerasCVAEModelConfig) -> List[str]:
sys.stdout.flush()
if len(bpfiles) == cfg.num_agg:
break
- print(f"Waiting for {cfg.num_agg} agg_4ml.bp files")
- time.sleep(cfg.timeout1)
+ if(os.getenv('DDMD_DEBUG') == None):
+ print(f"Waiting for {cfg.num_agg} agg_4ml.bp files")
+ time.sleep(cfg.timeout1)
print(f"bpfiles = {bpfiles}")
- time.sleep(60 * 5)
+ time.sleep(5*60)
- """
- # Wait for enough time steps in each bp file
- while True:
- enough = True
- for bp in bpfiles:
- com = f"bpls {bp}"
- a = subprocess.getstatusoutput(com)
- if a[0] != 0:
- enough = False
- print(f"Waiting, a = {a}, {bp}")
- break
- try:
- steps = int(a[1].split("\n")[0].split("*")[0].split(" ")[-1])
- except Exception as e:
- print("Exception ", e)
- steps = 0
- enough = False
- if steps < cfg.min_step_increment:
- enough = False
- print(f"Waiting, steps = {steps}, {bp}"); sys.stdout.flush()
- break
- if enough:
- break
- else:
- time.sleep(cfg.timeout2)
- """
return bpfiles
@@ -85,9 +60,10 @@ def next_input(
cm_data_input = streams.next()["contact_map"]
break
except: # noqa TODO: flake8 - should not have a bar except
- print("Sleeping for input to become readable")
- sys.stdout.flush()
- time.sleep(60)
+ if(os.getenv('DDMD_DEBUG') == None):
+ print("Sleeping for input to become readable")
+ sys.stdout.flush()
+ time.sleep(60)
continue
cm_data_input = np.expand_dims(cm_data_input, axis=-1)
@@ -150,7 +126,6 @@ def main(cfg: KerasCVAEModelConfig):
config=cfg.adios_xml_agg_4ml,
batch=cfg.read_batch,
stream_name="AggregatorOutput4ml",
- # change for different aggregators
)
# Infinite loop of CVAE training
diff --git a/deepdrivemd/sim/openmm/run_openmm.py b/deepdrivemd/sim/openmm/run_openmm.py
index a42706bf..bb1ef4a9 100644
--- a/deepdrivemd/sim/openmm/run_openmm.py
+++ b/deepdrivemd/sim/openmm/run_openmm.py
@@ -2,9 +2,8 @@
from pathlib import Path
from typing import Optional
-import simtk.openmm as omm # type: ignore[import]
-import simtk.openmm.app as app # type: ignore[import]
-import simtk.unit as u # type: ignore[import]
+import openmm.unit as u # type: ignore[import]
+import openmm.app as app # type: ignore[import]
from mdtools.openmm.reporter import OfflineReporter # type: ignore[import]
from mdtools.openmm.sim import configure_simulation # type: ignore[import]
@@ -115,7 +114,7 @@ def move_results(self) -> None:
def configure_reporters(
- sim: omm.app.Simulation,
+ sim: "app.Simulation",
ctx: SimulationContext,
cfg: OpenMMConfig,
report_steps: int,
@@ -158,12 +157,6 @@ def configure_reporters(
def run_simulation(cfg: OpenMMConfig) -> None:
- # openmm typed variables
- dt_ps = cfg.dt_ps * u.picoseconds
- report_interval_ps = cfg.report_interval_ps * u.picoseconds
- simulation_length_ns = cfg.simulation_length_ns * u.nanoseconds
- temperature_kelvin = cfg.temperature_kelvin * u.kelvin
-
# Handle files
with Timer("molecular_dynamics_SimulationContext"):
ctx = SimulationContext(cfg)
@@ -175,11 +168,16 @@ def run_simulation(cfg: OpenMMConfig) -> None:
top_file=ctx.top_file,
solvent_type=cfg.solvent_type,
gpu_index=0,
- dt_ps=dt_ps,
- temperature_kelvin=temperature_kelvin,
+ dt_ps=cfg.dt_ps,
+ temperature_kelvin=cfg.temperature_kelvin,
heat_bath_friction_coef=cfg.heat_bath_friction_coef,
)
+ # openmm typed variables
+ dt_ps = cfg.dt_ps * u.picoseconds
+ report_interval_ps = cfg.report_interval_ps * u.picoseconds
+ simulation_length_ns = cfg.simulation_length_ns * u.nanoseconds
+
# Write all frames to a single HDF5 file
frames_per_h5 = int(simulation_length_ns / report_interval_ps)
# Steps between reporting DCD frames and logs
diff --git a/deepdrivemd/sim/openmm_stream/openmm_reporter.py b/deepdrivemd/sim/openmm_stream/openmm_reporter.py
index d0e6858e..c6eb7b0b 100644
--- a/deepdrivemd/sim/openmm_stream/openmm_reporter.py
+++ b/deepdrivemd/sim/openmm_stream/openmm_reporter.py
@@ -1,14 +1,11 @@
import datetime
import hashlib
import sys
-
-# from deepdrivemd.utils import t2Dto1D
from typing import Dict
import adios2
import MDAnalysis
import numpy as np
-import simtk.unit as u
from MDAnalysis.analysis import distances, rms
from deepdrivemd.utils import hash2intarray, timer
@@ -22,6 +19,14 @@ def __init__(self, reportInterval, cfg):
print(cfg)
print(f"report interval = {reportInterval}")
print("ContactMapRepoter constructor")
+
+ """
+ ADIOS stream is opened outside of this class so that
+ it can be created and destroyed without affecting the connection
+ to the ADIOS stream. Reinitializing this class is necessary for
+ the multi-ligand case.
+ """
+
self._adios_stream = cfg._adios_stream
self.step = 0
@@ -38,6 +43,18 @@ def __init__(self, reportInterval, cfg):
self.cfg.mda_selection
).positions.copy()
+
+ """
+ This ADIOS file stores trajectory for the given simulation run from
+ a particular initial conditions.
+ There is one ADIOS file for each simulation run and therefore
+ opening and closing this file in this class is OK since
+ this class is created for each simulation run as well.
+ The data from this file is not used during the run by other
+ components (which get their data via network), this file is only needed
+ if one wants to get positions in postproduction.
+ """
+
self._adios_file = adios2.open(
name=str(self.cfg.current_dir) + "/trajectory.bp",
mode="w",
@@ -70,9 +87,8 @@ def report(self, simulation, state):
if atom.name == self.cfg.openmm_selection[0]:
ca_indices.append(atom.index)
- positions = np.array(state.getPositions().value_in_unit(u.angstrom)).astype(
- np.float32
- )
+ positions = state.getPositions(asNumpy=True).astype(np.float32)
+
if self.cfg.compute_zcentroid:
centroid = np.array(self.zcentroid(positions), dtype=np.float32)
@@ -105,8 +121,6 @@ def report(self, simulation, state):
contact_map = distances.contact_matrix(
positions_ca, cutoff=self.cfg.threshold, returntype="numpy", box=None
).astype("uint8")
- # contact_map = t2Dto1D(contact_map)
- # contact_map = np.packbits(contact_map)
step = np.array([step], dtype=np.int32)
gpstime = np.array([int(datetime.datetime.now().timestamp())], dtype=np.int32)
@@ -154,6 +168,8 @@ def write_adios_step(self, output: Dict[str, np.ndarray]):
representing one step
"""
+
+ # Sending data via network to the aggregator
for k, v in output.items():
if k == "gpstime":
continue
@@ -162,6 +178,7 @@ def write_adios_step(self, output: Dict[str, np.ndarray]):
)
self._adios_stream.end_step()
+ # Saving trajectories to BP file
for k, v in output.items():
self._adios_file.write(
k, v, list(v.shape), [0] * len(v.shape), list(v.shape)
diff --git a/deepdrivemd/sim/openmm_stream/run_openmm.py b/deepdrivemd/sim/openmm_stream/run_openmm.py
index 1d125b18..8221adf9 100644
--- a/deepdrivemd/sim/openmm_stream/run_openmm.py
+++ b/deepdrivemd/sim/openmm_stream/run_openmm.py
@@ -11,8 +11,8 @@
import adios2
import numpy as np
import pandas as pd
-import simtk.openmm as omm
-import simtk.unit as u
+import openmm.unit as u
+import openmm.app as app
from mdtools.openmm.sim import configure_simulation
from deepdrivemd.sim.openmm.run_openmm import SimulationContext
@@ -22,7 +22,7 @@
def configure_reporters(
- sim: omm.app.simulation.Simulation,
+ sim: app.simulation.Simulation,
cfg: OpenMMConfig,
report_steps: int,
iteration: int = 0,
@@ -30,35 +30,15 @@ def configure_reporters(
cfg.reporter = ContactMapReporter(report_steps, cfg)
sim.reporters.append(cfg.reporter)
- print("type(sim) = ", type(sim))
- print("type(cfg) = ", type(cfg))
-
- """
- log_file = os.path.dirname(ctx.log_file) + f"/{iteration}/" + os.path.basename(ctx.log_file)
- sim.reporters.append(
- omm.app.StateDataReporter(
- log_file,
- report_steps,
- step=True,
- time=True,
- speed=True,
- potentialEnergy=True,
- temperature=True,
- totalEnergy=True,
- )
- )
- """
-
-
def next_outlier(
- cfg: OpenMMConfig, sim: omm.app.simulation.Simulation
-) -> Dict[str, Union[int, float, str, np.array]]:
+ cfg: OpenMMConfig, sim: app.simulation.Simulation
+) -> Dict[str, Union[int, float, str, np.ndarray]]:
"""Get the next outlier to use as an initial state.
Parameters
----------
cfg : OpenMMConfig
- sim : omm.app.simulation.Simulation
+ sim : app.simulation.Simulation
Returns
-------
@@ -75,12 +55,10 @@ def next_outlier(
while True:
try:
- # cfg.lock.acquire()
with open(cfg.pickle_db, "rb") as f:
db = pickle.load(f)
md5 = db.sorted_index[cfg.task_idx]
rmsd = db.dictionary[md5]
- # positions_pdb = cfg.outliers_dir / f"{md5}.pdb"
positions_pdb = cfg.outliers_dir / f"p_{md5}.npy"
velocities_npy = cfg.outliers_dir / f"v_{md5}.npy"
@@ -91,7 +69,6 @@ def next_outlier(
task = cfg.outliers_dir / f"{md5}.txt"
shutil.copy(task, cfg.current_dir)
copied_task = cfg.current_dir / f"{md5}.txt"
- # cfg.lock.release()
except Exception as e:
print("=" * 30)
print(e)
@@ -108,9 +85,6 @@ def next_outlier(
with open(cfg.current_dir / "rmsd.txt", "w") as f:
f.write(f"{rmsd}\n")
- # positions_pdb = cfg.current_dir / f"{md5}.pdb"
- # velocities_npy = cfg.current_dir / f"{md5}.npy"
-
positions_pdb = cfg.current_dir / f"p_{md5}.npy"
velocities_npy = cfg.current_dir / f"v_{md5}.npy"
@@ -132,7 +106,7 @@ def next_outlier(
# TODO: flake8 says this function is too complex. Needs refactor.
def prepare_simulation( # noqa
- cfg: OpenMMConfig, iteration: int, sim: omm.app.simulation.Simulation
+ cfg: OpenMMConfig, iteration: int, sim: app.simulation.Simulation
) -> bool:
"""Replace positions and, with `cfg.copy_velocities_p` probability, velocities
of the current simulation state from an outlier
@@ -141,7 +115,7 @@ def prepare_simulation( # noqa
----------
cfg : OpenMMConfig
iteration : int
- sim: omm.app.simulation.Simulation
+ sim: app.simulation.Simulation
Returns
-------
@@ -187,12 +161,9 @@ def prepare_simulation( # noqa
while True:
try:
- # import parmed as pmd (would go at the top)
- # positions = pmd.load_file(str(positions_pdb)).positions
positions = np.load(str(positions_pdb))
print("positions.shape = ", positions.shape)
print("positions = ", positions)
- # positions = pmd.load_file(ctx.top_file, xyz=str(positions_pdb)).positions
velocities = np.load(str(velocities_npy))
break
except Exception as e:
@@ -202,23 +173,20 @@ def prepare_simulation( # noqa
if hasattr(cfg, "multi_ligand_table") and cfg.multi_ligand_table.is_file():
with Timer("molecular_dynamics_configure_simulation"):
- dt_ps = cfg.dt_ps * u.picoseconds
- temperature_kelvin = cfg.temperature_kelvin * u.kelvin
print("positions_pdb = ", positions_pdb)
print("ctx.top_file = ", ctx.top_file)
- # cfg.pdb_file = str(positions_pdb)
try:
del sim
except Exception as e:
print(e)
sim = configure_simulation(
- pdb_file=ctx.pdb_file, # str(positions_pdb), # ctx.pdb_file,
+ pdb_file=ctx.pdb_file,
top_file=ctx.top_file,
solvent_type=cfg.solvent_type,
gpu_index=0,
- dt_ps=dt_ps,
- temperature_kelvin=temperature_kelvin,
+ dt_ps=cfg.dt_ps,
+ temperature_kelvin=cfg.temperature_kelvin,
heat_bath_friction_coef=cfg.heat_bath_friction_coef,
)
else:
@@ -230,8 +198,6 @@ def prepare_simulation( # noqa
with Timer("molecular_dynamics_configure_reporters"):
configure_reporters(sim, cfg, cfg.report_steps, iteration)
- sim.context.setPositions(positions / 10)
-
if random.random() < cfg.copy_velocities_p:
print("Copying velocities from outliers")
sim.context.setVelocities(velocities)
@@ -256,8 +222,6 @@ def prepare_simulation( # noqa
print("dir(ctx) = ", dir(ctx))
with Timer("molecular_dynamics_configure_simulation"):
- dt_ps = cfg.dt_ps * u.picoseconds
- temperature_kelvin = cfg.temperature_kelvin * u.kelvin
try:
del sim
except Exception as e:
@@ -269,9 +233,9 @@ def prepare_simulation( # noqa
top_file=ctx.top_file,
solvent_type=cfg.solvent_type,
gpu_index=0,
- dt_ps=dt_ps,
- temperature_kelvin=temperature_kelvin,
- heat_bath_friction_coef=cfg.heat_bath_friction_coef,
+ dt_ps=cfg.dt_ps,
+ temperature_kelvin=cfg.temperature_kelvin,
+ heat_bath_friction_coef=cfg.heat_bath_friction_coef
# explicit_barostat="MonteCarloAnisotropicBarostat", ### ifs
)
sim.context.setVelocitiesToTemperature(
diff --git a/docs/source/conf.py b/docs/source/conf.py
index a1c4443d..0c6cc9c4 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -83,12 +83,20 @@
"adios2",
"tensorflow",
"simtk.openmm",
+ "openmm",
"cupy",
"cuml",
"numba",
+ "pandas",
+ "h5py",
+ "sklearn",
+ "wandb",
"torch",
"torchsummary",
"mdlearn",
+ "mdtools",
+ "molecules",
+ "MDAnalysis",
]
html_context = {
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 8df26ad1..cc87c623 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -61,6 +61,22 @@ Installation with `pip` and a *minimal set of dependencies*:
cd deepdrivemd
pip install -e .
+Running DeepDriveMD-S
+---------------------
+We have tested support for DeepDriveMD-S on the `Lassen`_ supercomputer at `Lawrence Livermore National Laboratory`_.
+Please follow the instructions linked below for installation and usage.
+
+.. _Lassen: https://hpc.llnl.gov/hardware/compute-platforms/lassen
+.. _Lawrence Livermore National Laboratory: https://www.llnl.gov
+
+.. toctree::
+ :maxdepth: 2
+
+ pages/ddmd-s-install
+ pages/ddmd-s-usage
+
+API Documentation
+-----------------
.. toctree::
:maxdepth: 3
diff --git a/docs/source/pages/_autosummary/deepdrivemd.models.keras_cvae.inference.rst b/docs/source/pages/_autosummary/deepdrivemd.models.keras_cvae.inference.rst
index 4b050154..75722087 100644
--- a/docs/source/pages/_autosummary/deepdrivemd.models.keras_cvae.inference.rst
+++ b/docs/source/pages/_autosummary/deepdrivemd.models.keras_cvae.inference.rst
@@ -1,4 +1,4 @@
-deepdrivemd.models.keras\_cvae.inference
+deepdrivemd.models.keras\_cvae.inference
========================================
.. automodule:: deepdrivemd.models.keras_cvae.inference
diff --git a/docs/source/pages/ddmd-s-install.rst b/docs/source/pages/ddmd-s-install.rst
new file mode 100644
index 00000000..3787db8d
--- /dev/null
+++ b/docs/source/pages/ddmd-s-install.rst
@@ -0,0 +1,92 @@
+=====================================================================
+How to install software dependencies for DeepDriveMD-S on Lassen@LLNL
+=====================================================================
+
+#. Set up the environment for Anaconda::
+
+ module load gcc/7.3.1
+ . /etc/profile.d/conda.sh
+ export IBM_POWERAI_LICENSE_ACCEPT=yes
+
+#. Add the following lines to ``$HOME/.condarc`` so that the packages are downloaded not into ``$HOME`` but into ``$CONDA2``::
+
+ pkgs_dirs:
+ - /usr/workspace/cv_ddmd/conda2/pkgs
+
+
+#. Follow IBM's documentation to install powerai environment (``https://www.ibm.com/docs/en/wmlce/1.7.0?topic=installing-mldl-frameworks``) with some modifications::
+
+ conda config --prepend channels https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/
+ export CONDA2=/usr/workspace/cv_ddmd/conda2
+ mkdir $CONDA2
+ conda create -p $CONDA2/powerai
+ conda activate $CONDA2/powerai
+ conda install powerai
+ conda install powerai-rapids
+
+ The last two steps can easily take hours before you are even asked to answer y/n.
+
+
+
+#. Install ADIOS::
+
+ git clone git@github.com:ornladios/ADIOS2.git
+ cd ADIOS2
+ mkdir build; cd build
+ cmake -DADIOS2_USE_MPI=OFF -DADIOS2_USE_CUDA=OFF -DCMAKE_INSTALL_PREFIX=/usr/workspace/cv_ddmd/software1/ADIOS2a ..
+ make -j20
+ make install
+ export PYTHONPATH=/usr/workspace/cv_ddmd/software1/ADIOS2a/lib/python3.7/site-packages:$PYTHONPATH
+ export PATH=/usr/workspace/cv_ddmd/software1/ADIOS2_060522/bin:$PATH
+ export LD_LIBRARY_PATH=/usr/workspace/cv_ddmd/software1/ADIOS2a/lib64:$LD_LIBRARY_PATH
+
+#. Assuming that you have configured `spack `_ , install swig::
+
+ spack install swig
+
+#. Install openmm::
+
+ module load cuda/10.2.89
+ module load cmake/3.23.1
+ spack load swig
+ git clone https://github.com/pandegroup/openmm.git
+ cd openmm
+ mkdir build; cd build
+ cmake -DCMAKE_INSTALL_PREFIX=/usr/workspace/cv_ddmd/software1/OPENMMa -DSWIG_EXECUTABLE=`which swig` ..
+ make -j20
+ make install
+ make PythonInstall
+ python -m openmm.testInstallation
+
+ The last statement runs tests.
+
+#. Miscellaneous dependencies::
+
+ pip install pathos
+ pip install mdanalysis==1.0.0
+ pip install opentsne
+ pip install radical.entk==1.6.7 radical.gtod==1.6.7 radical.pilot==1.6.7 radical.saga==1.6.10 radical.utils==1.6.7
+ pip install torchsummary
+
+ I had trouble with the latest versions of Radical.
+
+#. Install mdlearn::
+
+ git clone https://github.com/ramanathanlab/mdlearn
+ cd mdlearn
+ git checkout develop
+ pip install -e .
+
+#. Install MD-Tools::
+
+ git clone git@github.com:braceal/MD-tools.git
+ cd MD-tools
+ pip install -e .
+
+#. Install DeepDriveMD::
+
+ mkdir /usr/workspace/cv_ddmd/$USER
+ cd /usr/workspace/cv_ddmd/$USER
+ git clone git@github.com:DeepDriveMD/DeepDriveMD-pipeline.git
+ cd DeepDriveMD-pipeline
+ pip install -e .
diff --git a/docs/source/pages/ddmd-s-usage.rst b/docs/source/pages/ddmd-s-usage.rst
new file mode 100644
index 00000000..e5baf799
--- /dev/null
+++ b/docs/source/pages/ddmd-s-usage.rst
@@ -0,0 +1,190 @@
+======================================================================================
+How to use DeepDriveMD-S on Lassen@LLNL with the existing installation of dependencies
+======================================================================================
+
+---
+Run
+---
+
+#. Set up the environment::
+
+ module load gcc/7.3.1
+ . /etc/profile.d/conda.sh
+ conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ export IBM_POWERAI_LICENSE_ACCEPT=yes
+ module use /usr/workspace/cv_ddmd/software1/modules
+ module load adios2
+
+ Usually I source something like `powerai.sh `_.
+
+#. For convenience, let us define::
+
+ export DDMD="/usr/workspace/cv_ddmd/$USER/DeepDriveMD-pipeline"
+
+#. To run various preconfigured examples::
+
+ cd $DDMD/test
+ nohup make X > X.log 2>&1 &
+
+ where ``X`` is one of these:
+
+ * ``run1`` - mini BBA run with 12 simulations, 1 aggregator; 30m.
+ * ``run2`` - production BBA run with 120 simulations, 10 aggregators; 12h.
+ * ``run3`` - production BBA run, no outliers, no machine learning is used,
+ states for the next batch of the simulations are selected randomly from the reported states; 120 simulations, 10 aggregators; 12h.
+ * ``run3m`` - same as ``run3``, but 12 simulations, 1 aggregator; 30m.
+ * ``run4`` - production BBA run, random selection of outliers; 120 simulations, 10 aggregators; 12h.
+ * ``run4m`` - same as ``run4`` by 12 simulations, 1 aggregator; 30m.
+ * ``run5`` - production BBA run, greedy selection from the traversed states based on RMSD, no outliers are used; 120 simulations, 10 aggregators; 12h.
+ * ``run5m`` - same as ``run5`` but 12 simulations, 1 aggregator; 30m.
+ * ``run6`` - same as ``run9`` but 12 simulations, 1 aggregator; 3h.
+ * ``run6a`` - same as ``run9a`` but 12 simulations, 1 aggregator, 3h.
+ * ``run7`` - production run for insRec_OM_region; 120 simulations, 10 aggregators; 12h.
+ * ``run7m`` - same as run7 but 12 simulations, 1 aggregator; 4h.
+ * ``run8`` - production run for spike; 120 simulations, 10 aggregators; 12h.
+ * ``run9`` - production run for smoothended_rec, 120 simulations, 10 aggregators; 12h.
+ * ``run9a`` - production run for smoothended_rec where CVAE is replaced by AAE; 120 simulations, 10 aggregators; 12h.
+ * ``run10`` - same as run11 but 12 simulations, 1 aggregator; 6h.
+ * ``run11`` - production run for multi-ligand case; 120 simulations, 10 aggregators; 12h.
+
+#. The configuration files for the above cases can be found by appending ``$DDMD`` with ``test/bba/DIR``
+ where ``DIR`` is listed in the ``Makefile``. For example, for run1, the path is ``$DDMD/test/bba/test1_stream``.
+
+#. DeepDriveMD is configured using a YAML file, we edit ``generate.py`` to generate the YAML file (make sure to activate
+ the conda environment before running this)::
+
+ python generate.py > config.yaml
+
+#. ``adios_file.xml`` controls the output of a complete trajectory including positions, velocities, contact maps from each simulation.
+
+#. ``adios_sim.xml`` controls the communication over the network between a simulation and an aggregator.
+
+#. ``adios_agg_4ml.xml`` controls the communication over the network between an aggregator and training.
+
+#. ``adios_agg.xml`` controls the communication over the network between an aggregator and inference.
+
+#. Notice that ``bin/run.sh`` command sets up authorization in this line::
+
+ source /usr/workspace/cv_ddmd/.radical/auth
+
+ For the content of this file for a particular cluster, ask Radical developers.
+ In this file various environmental variables are set, such as RMQ_HOSTNAME, RMQ_PASSWORD,
+ mongodb_host, RADICAL_PILOT_DBURL, ..., that allow Radical Ensemble Toolkit to
+ communicate with the corresponding servers.
+
+-------
+Results
+-------
+
+#. Radical logs: ``/p/gpfs1/$USER/radical.pilot.sandbox``
+#. The latest best model: ``{experiment_directory}/machine_learning_runs/stage0000/task0000/published_model/best.h5``, where ``{experiment_directory}``
+ is specified in ``generate.py`` (and in the corresponding ``config.yaml``)
+#. ADIOS trajectories from each simulation invocation: ``{experiment_directory}/molecular_dynamics_runs/stage0000/task*/*/trajectory.bp``
+
+ * There is only one stage and as many tasks under ``molecular_dynamics_runs/stage0000`` directory as there are parallel simulations
+ (in our typical production run we use 120 parallel simulations, in mini test runs - 12 simulations).
+ * Under each ``taskXXXX`` directory, there are subdirectories 0, 1, ... corresponding to different restarts of the simulations from the outliers.
+ * The first one or two simulations are started from initial conditions. For the rest, the corresponding outlier files are copied into the directory.
+
+--------------
+Postproduction
+--------------
+
+^^^^^^^^^^^^
+Trajectories
+^^^^^^^^^^^^
+
+#. Trajectories in bp format are saved for each simulation. For example::
+
+ $ bpls /p/gpfs1/yakushin/Outputs/3/molecular_dynamics_runs/stage0000/task0000/4/trajectory.bp
+ char contact_map 100*{28, 28}
+ int32_t gpstime 100*{1}
+ int64_t md5 100*{32}
+ float positions 100*{504, 3}
+ float rmsd 100*{1}
+ int32_t step 100*{1}
+
+#. The above output says that there are 100 time steps saved in ``trajectory.bp``.
+ For each time step, we save 28x28 contact map, gpftime when the time step was reported,
+ md5 sum of positions, positions (in this case, it is x,y,z coordinates for each of the 504 atoms, corresponding
+ velocities, rmsd to the folded state, reporting step in the simulation (here it is from 0 to 99).
+#. To convert those trajectories into npy format, using 4 nodes, 10-minute walltime, do::
+
+ cd $DDMD/postproduction_stream
+ nohup ./run_positions.py 3 4 10 > positions_3.log 2>&1 &
+
+ Here 3 corresponds to the output subdirectory for the run: ``/p/gpfs1/yakushin/Outputs/3``.
+#. Running the above command creates `positions.npy` in each directory where `trajectory.bp` is found.
+#. The script uses Radical-ENTK to create as many independent tasks (that can run in parallel) as there are trajectories.
+#. You might have to edit ``driver_positions.py`` to change the path to your python and to your ``$DDMD``, your file that sets the environment for the jobs.
+
+^^^^^^^^^^^
+Loss curves
+^^^^^^^^^^^
+
+#. To generate loss curves from logs, run, for example::
+
+ python loss_real1.py -s re.session.lassen709.yakushin.019150.0009 -p 0 -t 13
+
+ provided that the log file for the machine learning task is in::
+
+ /p/gpfs1/$USER/radical.pilot.sandbox/re.session.lassen709.yakushin.019150.0009\
+ /pilot.0000/task.0013/task.0013.out
+
+ * The corresponding ``*.csv`` file will be in ``/p/gpfs1/$USER/Outputs/3/postproduction_stream/losses.csv``.
+ * The loss curves can be plotted by with ``$DDMD/postproduction_stream/loss.ipynb``.
+
+#. For AAE case, the logs are written in a different format and can be parsed with::
+
+ python loss_aae.py logfile dir
+
+ where ``dir`` is subdirectory of ``/p/gpfs1/yakushin/Outputs`` where the run files are written.
+
+^^^^^^^^^^^^
+Gantt charts
+^^^^^^^^^^^^
+
+#. To parse log files in order to generate gantt charts, run from postproduction_stream directory::
+
+ nohup ./run_timers.sh output_dir nodes walltime session pilot exclude > timers.log 2>&1 &
+
+ where ``output_dir`` - subdirectory of ``/p/gpfs1/yakushin/Outputs``, ``nodes`` - number of nodes to use (the job is submitted to the cluster),
+ ``walltime`` - maximum walltime, ``session`` - log session like ``re.session.lassen709.yakushin.019150.0009``, ``pilot`` - typically 0, ``exclude`` - what tasks to exclude (for example, it is not interesting to see timing
+ for aggregators and it takes a lot of time to parse the corresponding logs so you can exclude those by using ``120-129`` as exclude (currently it is dumb and just parses one range).
+
+#. An example of the notebook to plot gantt charts: ``postproduction_stream/gantt_rmsd_streaming.ipynb``
+
+
+^^^^^^^^^^
+Embeddings
+^^^^^^^^^^
+
+#. To generate embedding files, run from the postproduction_stream directory::
+
+ nohup ./run_emb.sh > emb.log outputdir nodes walltime zcentroid 2>&1 &
+
+ where ``outputdir`` - subdirectory of ``/p/gpfs1/yakushin/Outputs/`` where job outputs are stored, ``nodes`` - number of nodes to use (the job is submitted to the cluster),
+ ``walltime`` - up to how much time to run the job, ``zcentroid`` - 1 or 0 depending on whether you want to calculate zcentroid or not.
+
+#. An example notebook to plot embeddings: ``postproduction_stream/plot_tsne.ipynb``
+
+^^^^^^^^^
+Positions
+^^^^^^^^^
+
+#. To generate positions in numpy format, run from the postproduction_stream directory::
+
+ nohup ./run_positions.sh output_dir nodes walltime > positions.log 2>&1 &
+
+#. To collect the results into a single file, run from ``/p/gpfs1/yakushin/Outputs`` directory::
+
+ /path/postproduction_stream/archive.sh outputdir
+
+
+
+.. autosummary::
+ :toctree: _autosummary
+ :recursive:
+
+ deepdrivemd
+
diff --git a/examples/bridges2/README.md b/examples/bridges2/README.md
new file mode 100644
index 00000000..d55d5caf
--- /dev/null
+++ b/examples/bridges2/README.md
@@ -0,0 +1,191 @@
+# Software package
+DeepDriveMD consists of multiple stages including MD simulation, ML training
+and each stage require specific software such as OpenMM CUDA, Pytorch, MD-tools
+and molecules. Luckily these requirement can be installed via pip or conda.
+
+```
+module load cuda/10.2.0
+conda install -c omnia-dev/label/cuda102 openmm
+pip install git+https://github.com/braceal/MD-tools.git
+pip install git+https://github.com/braceal/molecules.git
+```
+
+Due to the conflict of dependency libraries, we need to create separate
+environment for pytorch, for example in conda:
+
+```
+conda create -n pytorch python=3.6.12
+conda activate pytorch
+conda install pytorch
+```
+
+On PSC Bridges2, there is a sample conda env for RCT/EnTK and OpenMM 7.5.1,:
+
+```
+conda activate /ocean/projects/mcb110096p/hrlee/conda/1.6.6
+```
+
+or clone (copy of the environment):
+
+```
+conda create --name rct --clone /ocean/projects/mcb110096p/hrlee/conda/1.6.6
+```
+
+
+and for pytorch:
+
+```
+conda activate /ocean/projects/mcb110096p/hrlee/conda/pytorch
+```
+
+or clone:
+```
+conda create --name pytorch --clone /ocean/projects/mcb110096p/hrlee/conda/pytorch
+```
+
+# Example template
+
+DeepDriveMD runs with *user-defined* configuration file, which can be created
+by an empty template, for example:
+
+```
+python -m deepdrivemd.config
+```
+
+this command calls deepdrivemd.config module, and generates
+`deepdrvemd_template.yaml`. Users can start modifying values in the template
+such as HPC resource name, experiment base directory and input pdb path.
+
+There is a sample template for PSC Bridges:
+
+```
+/ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/experiment/deepdrivemd_bridges.yaml
+```
+
+**NOTE** that `experiment_directory` has to be non-exist prior actual run, it
+is a new base directory to be created. Replace the current value with your home
+location:
+
+```
+experiment_directory: /ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/experiment/1st_run
+```
+
+# Test Run
+
+```
+$ python -m deepdrivemd.deepdrivemd -c deepdrivemd_bridges.yaml
+```
+
+The example output looks like:
+
+```
+================================================================================
+ COVID-19 - Workflow2
+================================================================================
+
+EnTK session: re.session.br013.ib.bridges2.psc.edu.hrlee.018784.0008
+Creating AppManagerSetting up RabbitMQ system ok
+ ok
+Validating and assigning resource manager ok
+Setting up RabbitMQ system n/a
+new session: [re.session.br013.ib.bridges2.psc.edu.hrlee.018784.0008] \
+database : [mongodb://singharoy:****@129.114.17.185:27017/mdffentk] ok
+create pilot manager ok
+submit 1 pilot(s)
+ pilot.0000 xsede.bridges2 128 cores 0 gpus ok
+
+```
+
+There will be waiting first (looks like the screen is frozen but actually the
+HPC job is in the queue and trying to get it scheduled on available compute
+nodes in the cluster) and the progress will be visible once HPC job is active,
+like:
+
+```
+...
+Update: DeepDriveMD.MolecularDynamics state: SCHEDULING
+Update: DeepDriveMD.MolecularDynamics.task.0031 state: SCHEDULING
+Update: DeepDriveMD.MolecularDynamics.task.0031 state: SCHEDULED
+submit: ########################################################################
+Update: DeepDriveMD.MolecularDynamics state: SCHEDULED
+Update: DeepDriveMD.MolecularDynamics.task.0031 state: SUBMITTING
+Update: DeepDriveMD.MolecularDynamics.task.0031 state: EXECUTED
+Update: DeepDriveMD.MolecularDynamics state: DONE
+Update: DeepDriveMD.MachineLearning state: SCHEDULING
+Update: DeepDriveMD.MachineLearning.task.0032 state: SCHEDULING
+Update: DeepDriveMD.MachineLearning.task.0032 state: SCHEDULED
+Update: DeepDriveMD.MachineLearning state: SCHEDULED
+submit: ########################################################################
+Update: DeepDriveMD.MachineLearning.task.0032 state: SUBMITTING
+Update: DeepDriveMD.MachineLearning.task.0032 state: EXECUTED
+Update: DeepDriveMD.MachineLearning state: DONE
+Update: DeepDriveMD.ModelSelection state: SCHEDULING
+Update: DeepDriveMD.ModelSelection.task.0033 state: SCHEDULING
+Update: DeepDriveMD.ModelSelection.task.0033 state: SCHEDULED
+Update: DeepDriveMD.ModelSelection state: SCHEDULED
+submit: ########################################################################
+Update: DeepDriveMD.ModelSelection.task.0033 state: SUBMITTING
+Update: DeepDriveMD.ModelSelection.task.0033 state: EXECUTED
+Update: DeepDriveMD.ModelSelection state: DONE
+Update: DeepDriveMD.Agent state: SCHEDULING
+Update: DeepDriveMD.Agent.task.0034 state: SCHEDULING
+Update: DeepDriveMD.Agent.task.0034 state: SCHEDULED
+Update: DeepDriveMD.Agent state: SCHEDULED
+submit: ########################################################################
+Update: DeepDriveMD.Agent.task.0034 state: SUBMITTING
+Update: DeepDriveMD.Agent.task.0034 state: EXECUTED
+Update: DeepDriveMD.Agent state: DONE
+Finishing stage 3 of 4
+...
+```
+
+# bridges2 job commands
+
+* Running jobs to check:
+
+```
+$ squeue -u $USER
+ JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)
+ 1484680 RM pilot.00 hrlee R 1:36 1 r138
+```
+
+* Canceling jobs:
+
+```
+scancel
+scancel 1484680
+```
+# EnTK specific commands
+
+DeepDriveMD is running based on EnTK, ensemble toolkit, which creates an unique
+session id to store necessary data and logs for each run. For example, `python
+-m deepdrivemd.deepdrivemd xx.yaml` will leave a sub-directory like:
+
+* Session directory (temporal, auto-generated):
+
+```
+re.session.br013.ib.bridges2.psc.edu.hrlee.018784.0008
+```
+
+This session data is useful for further analysis, and can be removed if runs are complete.
+
+* Check debugging messages:
+
+```
+more /radical.log
+```
+
+* Check task outputs and error messages:
+Individual has `radical.pilot.sandbox` to keep all the output and error
+messages of DeepDriveMD tasks. For example, OpenMM python script and pytorch
+script may generate stdout/stderr messages and all these output of a specific
+run are stored under the sandbox directory.
+
+```
+/ocean/projects/mcb110096p/hrlee/radical.pilot.sandbox/
+```
+
+# PSC SSH
+
+Please refer to https://www.psc.edu/about-using-ssh/
+
diff --git a/examples/bridges2/deepdrivemd_bridges.yaml b/examples/bridges2/deepdrivemd_bridges.yaml
new file mode 100644
index 00000000..e673876b
--- /dev/null
+++ b/examples/bridges2/deepdrivemd_bridges.yaml
@@ -0,0 +1,222 @@
+title: COVID-19 - Workflow2
+resource: xsede.bridges2
+queue: GPU
+schema_: local
+project: mcb110096p
+walltime_min: 30
+max_iteration: 4
+cpus_per_node: 20
+gpus_per_node: 8
+hardware_threads_per_cpu: 2
+experiment_directory: /ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/examples/brdiges2/2nd_run
+node_local_path: null
+molecular_dynamics_stage:
+ pre_exec:
+ - module load anaconda3/2020.07
+ - module load cuda/10.2.0
+ - module load hdf5
+ - module load openmpi/3.1.6-gcc8.3.1
+ - source activate /ocean/projects/mcb110096p/hrlee/conda/openmm
+ - export HDF5_DISABLE_VERSION_CHECK=1
+ executable: '/ocean/projects/mcb110096p/hrlee/conda/openmm/bin/python'
+ arguments: ['/ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/deepdrivemd/sim/openmm/run_openmm.py']
+ cpu_reqs:
+ processes: 1
+ process_type: null
+ threads_per_process: 3
+ thread_type: OpenMP
+ gpu_reqs:
+ processes: 0
+ process_type: null
+ threads_per_process: 0
+ thread_type: null
+ num_tasks: 16
+ task_config:
+ experiment_directory: set_by_deepdrivemd
+ stage_idx: 0
+ task_idx: 0
+ output_path: set_by_deepdrivemd
+ node_local_path: set_by_deepdrivemd
+ pdb_file: set_by_deepdrivemd
+ initial_pdb_dir: /ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/data/bba
+ reference_pdb_file: /ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/data/bba/1FME-folded.pdb
+ simulation_length_ns: 1.0
+ report_interval_ps: 1.0
+ dt_ps: 0.002
+ temperature_kelvin: 310.0
+ heat_bath_friction_coef: 1.0
+ wrap: false
+ openmm_selection:
+ - CA
+ mda_selection: protein and name CA
+ threshold: 8.0
+ contact_map: true
+ point_cloud: true
+ fraction_of_contacts: true
+ in_memory: false
+aggregation_stage:
+ pre_exec: []
+ executable: ''
+ arguments: []
+ cpu_reqs:
+ processes: 1
+ process_type: null
+ threads_per_process: 1
+ thread_type: null
+ gpu_reqs:
+ processes: 0
+ process_type: null
+ threads_per_process: 0
+ thread_type: null
+ skip_aggregation: true
+ task_config:
+ experiment_directory: set_by_deepdrivemd
+ stage_idx: 0
+ task_idx: 0
+ output_path: set_by_deepdrivemd
+ node_local_path: set_by_deepdrivemd
+machine_learning_stage:
+ pre_exec:
+ - module load anaconda3/2020.07
+ - module load cuda/10.2.0
+ - module load hdf5
+ - module load openmpi/3.1.6-gcc8.3.1
+ - export LANG=en_US.utf-8
+ - export LC_ALL=en_US.utf-8
+ - export HDF5_USE_FILE_LOCKING=FALSE
+ - source activate /ocean/projects/mcb110096p/hrlee/conda/pytorch
+ executable: '/ocean/projects/mcb110096p/hrlee/conda/pytorch/bin/python'
+ arguments: ['/ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/deepdrivemd/models/aae/train.py']
+ cpu_reqs:
+ processes: 1
+ process_type: null
+ threads_per_process: 1
+ thread_type: null
+ gpu_reqs:
+ processes: 0
+ process_type: null
+ threads_per_process: 0
+ thread_type: null
+ retrain_freq: 1
+ task_config:
+ experiment_directory: set_by_deepdrivemd
+ stage_idx: 0
+ task_idx: 0
+ output_path: set_by_deepdrivemd
+ node_local_path: set_by_deepdrivemd
+ model_tag: set_by_deepdrivemd
+ init_weights_path: null
+ dataset_name: point_cloud
+ rmsd_name: rmsd
+ fnc_name: fnc
+ num_points: 28
+ num_features: 0
+ initial_epochs: 10
+ epochs: 10
+ batch_size: 32
+ optimizer_name: Adam
+ optimizer_lr: 0.0001
+ latent_dim: 10
+ encoder_filters:
+ - 64
+ - 128
+ - 256
+ - 256
+ - 512
+ encoder_kernel_sizes:
+ - 5
+ - 3
+ - 3
+ - 1
+ - 1
+ generator_filters:
+ - 64
+ - 128
+ - 512
+ - 1024
+ discriminator_filters:
+ - 512
+ - 512
+ - 128
+ - 64
+ encoder_relu_slope: 0.0
+ generator_relu_slope: 0.0
+ discriminator_relu_slope: 0.0
+ use_encoder_bias: true
+ use_generator_bias: true
+ use_discriminator_bias: true
+ noise_mu: 0.0
+ noise_std: 1.0
+ lambda_rec: 0.5
+ lambda_gp: 10.0
+ embed_interval: 1
+ tsne_interval: 5
+ sample_interval: 20
+ num_data_workers: 0
+ dataset_location: storage
+ wandb_project_name: null
+model_selection_stage:
+ pre_exec:
+ - module load anaconda3/2020.07
+ - module load cuda/10.2.0
+ - module load hdf5
+ - module load openmpi/3.1.6-gcc8.3.1
+ - source activate /ocean/projects/mcb110096p/hrlee/conda/pytorch
+ executable: '/ocean/projects/mcb110096p/hrlee/conda/pytorch/bin/python'
+ arguments: ['/ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/deepdrivemd/selection/latest/select_model.py']
+ cpu_reqs:
+ processes: 1
+ process_type: null
+ threads_per_process: 4
+ thread_type: OpenMP
+ gpu_reqs:
+ processes: 0
+ process_type: null
+ threads_per_process: 0
+ thread_type: null
+ task_config:
+ experiment_directory: set_by_deepdrivemd
+ stage_idx: 0
+ task_idx: 0
+ output_path: set_by_deepdrivemd
+ node_local_path: set_by_deepdrivemd
+ retrain_freq: 1
+
+agent_stage:
+ pre_exec:
+ - module load anaconda3/2020.07
+ - module load cuda/10.2.0
+ - module load hdf5
+ - module load openmpi/3.1.6-gcc8.3.1
+ - export LANG=en_US.utf-8
+ - export LC_ALL=en_US.utf-8
+ - source activate /ocean/projects/mcb110096p/hrlee/conda/pytorch
+
+ executable: '/ocean/projects/mcb110096p/hrlee/conda/pytorch/bin/python'
+ arguments: ['/ocean/projects/mcb110096p/hrlee/git/DeepDriveMD-pipeline/deepdrivemd/agents/lof/lof.py']
+ cpu_reqs:
+ processes: 1
+ process_type: null
+ threads_per_process: 12
+ thread_type: OpenMP
+ gpu_reqs:
+ processes: 0
+ process_type: null
+ threads_per_process: 0
+ thread_type: null
+ task_config:
+ experiment_directory: set_by_deepdrivemd
+ stage_idx: 0
+ task_idx: 0
+ output_path: set_by_deepdrivemd
+ node_local_path: set_by_deepdrivemd
+ num_intrinsic_outliers: 100
+ num_extrinsic_outliers: 12
+ intrinsic_score: lof
+ extrinsic_score: rmsd
+ n_traj_frames: 1000
+ n_most_recent_h5_files: 12
+ k_random_old_h5_files: 12
+ sklearn_num_jobs: -1
+ model_type: "AAE3d"
+ inference_batch_size: 128
diff --git a/examples/ensemble-md/md.py b/examples/ensemble-md/md.py
index 7573dcfa..d78b39e9 100644
--- a/examples/ensemble-md/md.py
+++ b/examples/ensemble-md/md.py
@@ -1,7 +1,7 @@
import random
from typing import Optional
-import simtk.openmm.app as app # type: ignore[import]
-import simtk.unit as u # type: ignore[import]
+import openmm.app as app # type: ignore[import]
+import openmm.unit as u # type: ignore[import]
from mdtools.openmm.sim import configure_simulation # type: ignore[import]
from deepdrivemd.config import BaseSettings
from deepdrivemd.utils import parse_args
@@ -37,7 +37,6 @@ def run_simulation(cfg: OpenMMSimulationConfig) -> None:
# openmm typed variables
dt_ps = cfg.params.dt_ps * u.picoseconds
- temperature_kelvin = cfg.params.temperature_kelvin * u.kelvin
report_interval_ps = cfg.params.report_interval_ps * u.picoseconds
# If the simulation length is None, then we default to 10 and
# implement logic below to run indefinitely.
@@ -56,8 +55,8 @@ def run_simulation(cfg: OpenMMSimulationConfig) -> None:
top_file=cfg.input_top_file,
solvent_type=cfg.params.solvent_type,
gpu_index=0,
- dt_ps=dt_ps,
- temperature_kelvin=temperature_kelvin,
+ dt_ps=cfg.params.dt_ps,
+ temperature_kelvin=cfg.params.temperature_kelvin,
heat_bath_friction_coef=cfg.params.heat_bath_friction_coef,
)
diff --git a/postproduction_stream/archive.sh b/postproduction_stream/archive.sh
new file mode 100755
index 00000000..fe5c6b00
--- /dev/null
+++ b/postproduction_stream/archive.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/bash
+
+
+tar cvf $1.tar $1/postproduction $1/molecular_dynamics_runs/stage0000/task*/*/embeddings $1/molecular_dynamics_runs/stage0000/task*/*/positions.npy $1/molecular_dynamics_runs/stage0000/task*/*/rmsd.csv $1/machine_learning_runs/stage0000/task0000/published_model/best.*
+
+
diff --git a/setup.cfg b/setup.cfg
index 00c4de3c..7a8d7137 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -24,9 +24,7 @@ install_requires =
pydantic[dotenv]>=1.8.2,<2.0.0
radical.entk
MDAnalysis
- numpy>=1.20
tqdm
- h5py==2.10.0
pathos
python_requires = >=3.6
diff --git a/test/Makefile b/test/Makefile
index 7848ae7a..9f2f3af5 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -2,6 +2,8 @@ SHELL=/bin/bash
run1:
../bin/run.sh test1_stream
+run1mm:
+ ../bin/run.sh test1_stream3
run2:
../bin/run.sh lassen-keras-dbscan_stream
run3:
diff --git a/test/bba/lassen-aae_stream_smoothended_rec/config.yaml b/test/bba/lassen-aae_stream_smoothended_rec/config.yaml
index f2957c41..5e441c7b 100644
--- a/test/bba/lassen-aae_stream_smoothended_rec/config.yaml
+++ b/test/bba/lassen-aae_stream_smoothended_rec/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: smoothended_rec mini
walltime_min: 180
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/18m/aggregation_runs
@@ -108,13 +108,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -125,11 +125,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -146,13 +146,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/aae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/aae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -162,11 +162,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -243,13 +243,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -260,11 +260,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-aae_stream_smoothended_rec/generate.py b/test/bba/lassen-aae_stream_smoothended_rec/generate.py
index 69486bab..8afb5e8e 100755
--- a/test/bba/lassen-aae_stream_smoothended_rec/generate.py
+++ b/test/bba/lassen-aae_stream_smoothended_rec/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "smoothended_rec mini"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/18m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/18m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb"
@@ -38,10 +50,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -102,17 +110,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -153,7 +161,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -234,7 +242,7 @@ class TaskConfigML(AAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/aae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -285,7 +293,7 @@ class TaskConfigAgent(AAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-aae_stream_smoothended_rec_120/config.yaml b/test/bba/lassen-aae_stream_smoothended_rec_120/config.yaml
index 73710ddf..b73efed1 100644
--- a/test/bba/lassen-aae_stream_smoothended_rec_120/config.yaml
+++ b/test/bba/lassen-aae_stream_smoothended_rec_120/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: smoothended_rec, aae, 12h
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/405a/aggregation_runs
@@ -109,13 +109,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 64
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -126,11 +126,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -148,13 +148,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/aae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/aae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -164,11 +164,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -245,13 +245,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -262,11 +262,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-aae_stream_smoothended_rec_120/generate.py b/test/bba/lassen-aae_stream_smoothended_rec_120/generate.py
index c14ba7e3..35a384b0 100755
--- a/test/bba/lassen-aae_stream_smoothended_rec_120/generate.py
+++ b/test/bba/lassen-aae_stream_smoothended_rec_120/generate.py
@@ -6,6 +6,17 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "smoothended_rec, aae, 12h"
@@ -18,9 +29,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/405a"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/405a"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb"
@@ -38,10 +49,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -103,17 +110,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -165,7 +172,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_agg.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -245,7 +252,7 @@ class TaskConfigML(AAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/aae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -297,7 +304,7 @@ class TaskConfigAgent(AAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream/config.yaml b/test/bba/lassen-keras-dbscan_stream/config.yaml
index 7c7bb9c0..d5fbbac6 100644
--- a/test/bba/lassen-keras-dbscan_stream/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -113,13 +113,13 @@ agent_stage:
use_random_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -130,11 +130,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -151,13 +151,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -167,11 +167,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -235,13 +235,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -252,11 +252,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream/generate.py b/test/bba/lassen-keras-dbscan_stream/generate.py
index 3ba74923..7eba804e 100755
--- a/test/bba/lassen-keras-dbscan_stream/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/14"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/14"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,10 +53,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -103,17 +111,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
"module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -154,7 +162,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -210,7 +218,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -258,7 +266,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_greedy/config.yaml b/test/bba/lassen-keras-dbscan_stream_greedy/config.yaml
index dc6d16da..2f46be5f 100644
--- a/test/bba/lassen-keras-dbscan_stream_greedy/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_greedy/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/16/aggregation_runs
@@ -110,13 +110,13 @@ agent_stage:
use_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -127,11 +127,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -147,13 +147,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -163,11 +163,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -231,13 +231,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -248,11 +248,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_greedy/generate.py b/test/bba/lassen-keras-dbscan_stream_greedy/generate.py
index 82c20222..490d86ea 100755
--- a/test/bba/lassen-keras-dbscan_stream_greedy/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_greedy/generate.py
@@ -6,6 +6,17 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +29,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/16"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/16"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,10 +52,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -102,17 +109,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +159,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +215,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -253,7 +260,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_greedy_mini/config.yaml b/test/bba/lassen-keras-dbscan_stream_greedy_mini/config.yaml
index 1da53ddb..07e384f1 100644
--- a/test/bba/lassen-keras-dbscan_stream_greedy_mini/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_greedy_mini/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/16m/aggregation_runs
@@ -110,13 +110,13 @@ agent_stage:
use_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -127,11 +127,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -147,13 +147,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -163,11 +163,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -231,13 +231,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -248,11 +248,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_greedy_mini/generate.py b/test/bba/lassen-keras-dbscan_stream_greedy_mini/generate.py
index 9384c770..9d4036f2 100755
--- a/test/bba/lassen-keras-dbscan_stream_greedy_mini/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_greedy_mini/generate.py
@@ -6,6 +6,19 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +31,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/16m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/16m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,9 +54,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
class CPUReqMD(BaseModel):
processes = 1
@@ -102,17 +112,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +162,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +218,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -253,7 +263,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/config.yaml b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/config.yaml
index 054875e8..16f14bf9 100644
--- a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: null
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: '{DDMD}/deepdrivemd'
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - '{DDMD}/deepdrivemd/agents/stream/dbscan.py'
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/19/aggregation_runs
@@ -112,13 +112,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - '{DDMD}/deepdrivemd/aggregation/stream/aggregator.py'
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -129,11 +129,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -150,13 +150,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - '{DDMD}/deepdrivemd/models/keras_cvae_stream/train.py'
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -166,11 +166,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -234,13 +234,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - '{DDMD}/deepdrivemd/sim/openmm_stream/run_openmm.py'
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -251,11 +251,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/generate.py b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/generate.py
index efa3908d..5eae2985 100755
--- a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/19"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/19"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ "{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/insRec_OM_region/system/prot.pdb"
@@ -37,10 +49,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -100,17 +108,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -151,7 +159,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -207,7 +215,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -254,7 +262,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/config.yaml b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/config.yaml
index 9d817742..15603f72 100644
--- a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: null
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 240
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/19m/aggregation_runs
@@ -112,13 +112,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -129,11 +129,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -150,13 +150,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -166,11 +166,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -216,7 +216,7 @@ machine_learning_stage:
- 1024
- 1024
latent_dim: 10
- max_loss: 65000
+ max_loss: 6500000
max_steps: 2000
min_step_increment: 200
model: cvae
@@ -234,13 +234,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -251,11 +251,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/generate.py b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/generate.py
index bdeede25..eea61936 100755
--- a/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_insRec_OM_region_mini/generate.py
@@ -6,6 +6,19 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +31,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/19m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/19m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/insRec_OM_region/system/prot.pdb"
@@ -37,10 +50,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -100,17 +109,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -151,7 +160,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -187,7 +196,7 @@ class TaskConfigML(CVAE):
batch_size = 32 * 2
min_step_increment = 200
max_steps = 2000
- max_loss = 65000
+ max_loss = 6500000
num_agg = agg.num_tasks
timeout1 = 30
timeout2 = 10
@@ -207,7 +216,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -254,7 +263,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_multi-ligand/config.yaml b/test/bba/lassen-keras-dbscan_stream_multi-ligand/config.yaml
index 9b79802b..c4300779 100644
--- a/test/bba/lassen-keras-dbscan_stream_multi-ligand/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_multi-ligand/config.yaml
@@ -17,19 +17,19 @@ queue: pbatch
ref_pdb_file: null
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 360
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -39,11 +39,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/306m/aggregation_runs
@@ -114,13 +114,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -131,11 +131,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -154,13 +154,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -170,11 +170,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -238,13 +238,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -255,11 +255,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_multi-ligand/generate.py b/test/bba/lassen-keras-dbscan_stream_multi-ligand/generate.py
index dc41d508..6baa1a6b 100755
--- a/test/bba/lassen-keras-dbscan_stream_multi-ligand/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_multi-ligand/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "BBA integration test"
@@ -20,7 +32,7 @@ class Header(BaseModel):
hardware_threads_per_cpu = 4
experiment_directory = "/p/gpfs1/yakushin/Outputs/306m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "set_by_deepdrivemd"
@@ -40,10 +52,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -107,17 +115,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -160,7 +168,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -216,7 +224,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -264,7 +272,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/config.yaml b/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/config.yaml
index ac62686d..8390c169 100644
--- a/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/config.yaml
@@ -17,19 +17,19 @@ queue: pbatch
ref_pdb_file: null
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: multi-ligand
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -39,11 +39,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -116,13 +116,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -133,11 +133,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
skip_aggregation: false
task_config:
@@ -157,13 +157,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -173,11 +173,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -242,13 +242,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -259,11 +259,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_file: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/generate.py b/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/generate.py
index fe30d2f4..dd11949b 100755
--- a/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_multi-ligand_120/generate.py
@@ -6,6 +6,21 @@
import yaml
from pydantic import BaseModel
+import yaml
+from pydantic import BaseModel
+
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "multi-ligand"
@@ -18,9 +33,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/306"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/306"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "set_by_deepdrivemd"
@@ -40,10 +55,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -106,18 +117,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
"export OMP_PROC_BIND=false",
]
-
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -160,7 +170,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -216,7 +226,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -265,7 +275,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_noutliers/config.yaml b/test/bba/lassen-keras-dbscan_stream_noutliers/config.yaml
index c21c78cd..56b73ac1 100644
--- a/test/bba/lassen-keras-dbscan_stream_noutliers/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_noutliers/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/13/aggregation_runs
@@ -110,13 +110,13 @@ agent_stage:
use_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -127,11 +127,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -147,13 +147,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -163,11 +163,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -231,13 +231,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -248,11 +248,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_noutliers/generate.py b/test/bba/lassen-keras-dbscan_stream_noutliers/generate.py
index 2fe2997c..3827ac4c 100755
--- a/test/bba/lassen-keras-dbscan_stream_noutliers/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_noutliers/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/13"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/13"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,10 +53,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -102,17 +110,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +160,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +216,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -253,7 +261,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_noutliers_mini/config.yaml b/test/bba/lassen-keras-dbscan_stream_noutliers_mini/config.yaml
index cf99fbeb..951b05e4 100644
--- a/test/bba/lassen-keras-dbscan_stream_noutliers_mini/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_noutliers_mini/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 30
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/13m/aggregation_runs
@@ -110,13 +110,13 @@ agent_stage:
use_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -127,11 +127,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -147,13 +147,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -163,11 +163,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -231,13 +231,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -248,11 +248,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_noutliers_mini/generate.py b/test/bba/lassen-keras-dbscan_stream_noutliers_mini/generate.py
index 9ca7fcdf..e1ae961d 100755
--- a/test/bba/lassen-keras-dbscan_stream_noutliers_mini/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_noutliers_mini/generate.py
@@ -6,6 +6,17 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +29,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/13m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/13m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,8 +52,8 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
+#pythonpath = os.getenv("PYTHONPATH")
+#python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
class CPUReqMD(BaseModel):
@@ -102,17 +113,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +163,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +219,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -253,7 +264,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_random_outliers/config.yaml b/test/bba/lassen-keras-dbscan_stream_random_outliers/config.yaml
index ad018580..42efcda3 100644
--- a/test/bba/lassen-keras-dbscan_stream_random_outliers/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_random_outliers/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/14/aggregation_runs
@@ -111,13 +111,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -128,11 +128,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -148,13 +148,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -164,11 +164,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -232,13 +232,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -249,11 +249,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_random_outliers/generate.py b/test/bba/lassen-keras-dbscan_stream_random_outliers/generate.py
index 7150e82d..75fb6577 100755
--- a/test/bba/lassen-keras-dbscan_stream_random_outliers/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_random_outliers/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/14"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/14"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -102,17 +114,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +164,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +220,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -254,7 +266,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/config.yaml b/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/config.yaml
index 495bec37..946c06bd 100644
--- a/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 30
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/14m/aggregation_runs
@@ -111,13 +111,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -128,11 +128,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -148,13 +148,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -164,11 +164,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -232,13 +232,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -249,11 +249,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/generate.py b/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/generate.py
index 77cab209..630bce45 100755
--- a/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_random_outliers_mini/generate.py
@@ -6,6 +6,20 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +32,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/14m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/14m"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,10 +55,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -102,17 +112,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +162,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +218,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -254,7 +264,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_smoothended_rec/config.yaml b/test/bba/lassen-keras-dbscan_stream_smoothended_rec/config.yaml
index 4b91edf1..d21d5781 100644
--- a/test/bba/lassen-keras-dbscan_stream_smoothended_rec/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_smoothended_rec/config.yaml
@@ -28,7 +28,7 @@ agent_stage:
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/18mc/aggregation_runs
@@ -119,7 +119,7 @@ aggregation_stage:
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -130,11 +130,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -157,7 +157,7 @@ machine_learning_stage:
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -167,11 +167,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -241,7 +241,7 @@ molecular_dynamics_stage:
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -252,11 +252,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_smoothended_rec/generate.py b/test/bba/lassen-keras-dbscan_stream_smoothended_rec/generate.py
index b45f231e..3f091894 100755
--- a/test/bba/lassen-keras-dbscan_stream_smoothended_rec/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_smoothended_rec/generate.py
@@ -6,6 +6,19 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "smoothended_rec mini"
@@ -37,10 +50,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -101,17 +110,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -152,7 +161,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -208,7 +217,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -256,7 +265,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/config.yaml b/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/config.yaml
index cf20ae1f..754bda47 100644
--- a/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/305/aggregation_runs
@@ -114,13 +114,13 @@ agent_stage:
use_random_outliers: true
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -131,11 +131,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -153,13 +153,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -169,11 +169,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -237,13 +237,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -254,11 +254,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/generate.py b/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/generate.py
index 76f59bae..056a8d96 100755
--- a/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_smoothended_rec_120/generate.py
@@ -6,6 +6,18 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
+
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +30,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/305"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/305"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb"
@@ -37,10 +49,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -102,17 +110,16 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
]
-
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -154,7 +161,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -210,7 +217,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -259,7 +266,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/lassen-keras-dbscan_stream_spike/config.yaml b/test/bba/lassen-keras-dbscan_stream_spike/config.yaml
index 7e173734..acccb8cc 100644
--- a/test/bba/lassen-keras-dbscan_stream_spike/config.yaml
+++ b/test/bba/lassen-keras-dbscan_stream_spike/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/spike/system/closed_wrapped.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 720
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 4
@@ -38,11 +38,11 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -115,13 +115,13 @@ agent_stage:
use_random_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -132,11 +132,11 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
skip_aggregation: false
task_config:
@@ -155,13 +155,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 4
@@ -171,11 +171,11 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -240,13 +240,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -257,11 +257,11 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
+ - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2_061222/lib/python3.7/site-packages
- export OMP_PROC_BIND=false
task_config:
adios_xml_file: set_by_deepdrivemd
diff --git a/test/bba/lassen-keras-dbscan_stream_spike/generate.py b/test/bba/lassen-keras-dbscan_stream_spike/generate.py
index 41e8daed..3527dc5f 100755
--- a/test/bba/lassen-keras-dbscan_stream_spike/generate.py
+++ b/test/bba/lassen-keras-dbscan_stream_spike/generate.py
@@ -6,6 +6,17 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +29,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/22"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/22"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/spike/system/closed_wrapped.pdb"
@@ -37,10 +48,6 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
-
-
class CPUReqMD(BaseModel):
processes = 1
process_type: str = None
@@ -103,18 +110,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ f"export PYTHONPATH={PYTHONPATH}",
"export OMP_PROC_BIND=false",
]
-
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -156,7 +162,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -222,7 +228,7 @@ class GPUReqML(BaseModel):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_ml.dict()
@@ -281,7 +287,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_agent.dict()
diff --git a/test/bba/test1_stream/config.yaml b/test/bba/test1_stream/config.yaml
index 0c18b651..f658f61f 100644
--- a/test/bba/test1_stream/config.yaml
+++ b/test/bba/test1_stream/config.yaml
@@ -16,19 +16,19 @@ queue: pbatch
ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
resource: llnl.lassen
schema_: local
-software_directory: /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
title: BBA integration test
walltime_min: 30
agent_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 39
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -39,11 +39,10 @@ agent_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
task_config:
adios_xml_agg: set_by_deepdrivemd
adios_xml_agg_4ml: set_by_deepdrivemd
@@ -114,13 +113,13 @@ agent_stage:
use_random_outliers: false
aggregation_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 0
@@ -131,11 +130,10 @@ aggregation_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
skip_aggregation: false
task_config:
adios_xml_agg: set_by_deepdrivemd
@@ -152,13 +150,13 @@ aggregation_stage:
task_idx: 0
machine_learning_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -169,11 +167,10 @@ machine_learning_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
task_config:
adios_xml_agg: set_by_deepdrivemd
agg_dir: /p/gpfs1/yakushin/Outputs/3/aggregation_runs/
@@ -235,13 +232,13 @@ machine_learning_stage:
use_model_checkpoint: true
molecular_dynamics_stage:
arguments:
- - /usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
cpu_reqs:
process_type: null
processes: 1
thread_type: OpenMP
threads_per_process: 4
- executable: /usr/workspace/cv_ddmd/conda1/powerai/bin/python
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
gpu_reqs:
process_type: null
processes: 1
@@ -252,11 +249,10 @@ molecular_dynamics_stage:
- unset PYTHONPATH
- module load gcc/7.3.1
- . /etc/profile.d/conda.sh
- - conda activate /usr/workspace/cv_ddmd/conda1/powerai
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
- export IBM_POWERAI_LICENSE_ACCEPT=yes
- module use /usr/workspace/cv_ddmd/software1/modules
- - module load adios2
- - export PYTHONPATH=/p/gpfs1/yakushin/DDMD//entk_cvae_md/Outlier_search:/usr/workspace/cv_ddmd/software1/ADIOS2/lib/python3.6/site-packages
+ - module load adios2/2.8.1a
task_config:
adios_xml_file: set_by_deepdrivemd
adios_xml_sim: set_by_deepdrivemd
diff --git a/test/bba/test1_stream/generate.py b/test/bba/test1_stream/generate.py
index 6f8e2d61..ec6aa205 100755
--- a/test/bba/test1_stream/generate.py
+++ b/test/bba/test1_stream/generate.py
@@ -6,6 +6,17 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
title = "BBA integration test"
@@ -18,9 +29,9 @@ class Header(BaseModel):
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/3"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/3"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
init_pdb_file = (
@@ -41,8 +52,8 @@ class Header(BaseModel):
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
+#pythonpath = os.getenv("PYTHONPATH")
+#python = "/usr/workspace/cv_ddmd/conda2/powerai/bin/python"
class CPUReqMD(BaseModel):
@@ -102,17 +113,16 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -153,7 +163,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -207,7 +217,7 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -256,7 +266,7 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
diff --git a/test/bba/test1_stream3/adios_agg.xml b/test/bba/test1_stream3/adios_agg.xml
new file mode 100644
index 00000000..84ef89bd
--- /dev/null
+++ b/test/bba/test1_stream3/adios_agg.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/bba/test1_stream3/adios_agg_4ml.xml b/test/bba/test1_stream3/adios_agg_4ml.xml
new file mode 100644
index 00000000..4c558114
--- /dev/null
+++ b/test/bba/test1_stream3/adios_agg_4ml.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/bba/test1_stream3/adios_file.xml b/test/bba/test1_stream3/adios_file.xml
new file mode 100644
index 00000000..34560017
--- /dev/null
+++ b/test/bba/test1_stream3/adios_file.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
diff --git a/test/bba/test1_stream3/adios_sim.xml b/test/bba/test1_stream3/adios_sim.xml
new file mode 100644
index 00000000..53378413
--- /dev/null
+++ b/test/bba/test1_stream3/adios_sim.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/bba/test1_stream3/config.yaml b/test/bba/test1_stream3/config.yaml
new file mode 100644
index 00000000..32971b9b
--- /dev/null
+++ b/test/bba/test1_stream3/config.yaml
@@ -0,0 +1,290 @@
+adios_xml_agg: set_by_deepdrivemd
+adios_xml_agg_4ml: set_by_deepdrivemd
+adios_xml_file: set_by_deepdrivemd
+adios_xml_sim: set_by_deepdrivemd
+config_directory: set_by_deepdrivemd
+cpus_per_node: 40
+experiment_directory: /p/gpfs1/yakushin/Outputs/3mm
+gpus_per_node: 4
+hardware_threads_per_cpu: 4
+init_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input/1FME-0.pdb
+max_iteration: 4
+model: cvae
+node_local_path: null
+project: cv19-a01
+queue: pbatch
+ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
+resource: llnl.lassen
+schema_: local
+software_directory: /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd
+title: BBA integration test
+walltime_min: 30
+
+agent_stage:
+ arguments:
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/agents/stream/dbscan.py
+ cpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: OpenMP
+ threads_per_process: 39
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
+ gpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: CUDA
+ threads_per_process: 1
+ num_tasks: 1
+ pre_exec:
+ - unset PYTHONPATH
+ - module load gcc/7.3.1
+ - . /etc/profile.d/conda.sh
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
+ - export IBM_POWERAI_LICENSE_ACCEPT=yes
+ - module use /usr/workspace/cv_ddmd/software1/modules
+ - module load adios2/2.8.1a
+ - export DDMD_DEBUG=1
+ task_config:
+ adios_xml_agg: set_by_deepdrivemd
+ adios_xml_agg_4ml: set_by_deepdrivemd
+ agg_dir: /p/gpfs1/yakushin/Outputs/3mm/aggregation_runs
+ best_model: /p/gpfs1/yakushin/Outputs/3mm/machine_learning_runs/stage0000/task0000/published_model/best.h5
+ compute_rmsd: true
+ conv_filter_shapes:
+ - - 3
+ - 3
+ - - 3
+ - 3
+ - - 3
+ - 3
+ - - 3
+ - 3
+ conv_filters:
+ - 64
+ - 64
+ - 64
+ - 64
+ conv_layers: 4
+ conv_strides:
+ - - 1
+ - 1
+ - - 2
+ - 2
+ - - 1
+ - 1
+ - - 1
+ - 1
+ dense_dropouts:
+ - 0.25
+ dense_layers: 1
+ dense_neurons:
+ - 128
+ experiment_directory: set_by_deepdrivemd
+ final_shape:
+ - 28
+ - 28
+ - 1
+ init_eps: 1.3
+ init_min_samples: 10
+ init_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input/1FME-0.pdb
+ initial_shape:
+ - 28
+ - 28
+ lastN: 2000
+ latent_dim: 10
+ min_step_increment: 500
+ model: cvae
+ num_agg: 1
+ num_sim: 12
+ outlier_count: 120
+ outlier_max: 1000
+ outlier_min: 500
+ output_path: set_by_deepdrivemd
+ project_gpu: false
+ project_lastN: 50000
+ read_batch: 2000
+ ref_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
+ shuffle: true
+ split_pct: 0.8
+ stage_idx: 0
+ task_idx: 0
+ timeout1: 30
+ timeout2: 10
+ use_outliers: true
+ use_random_outliers: false
+aggregation_stage:
+ arguments:
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/aggregation/stream/aggregator.py
+ cpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: OpenMP
+ threads_per_process: 4
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
+ gpu_reqs:
+ process_type: null
+ processes: 0
+ thread_type: null
+ threads_per_process: 0
+ num_tasks: 1
+ pre_exec:
+ - unset PYTHONPATH
+ - module load gcc/7.3.1
+ - . /etc/profile.d/conda.sh
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
+ - export IBM_POWERAI_LICENSE_ACCEPT=yes
+ - module use /usr/workspace/cv_ddmd/software1/modules
+ - module load adios2/2.8.1a
+ - export DDMD_DEBUG=1
+ skip_aggregation: false
+ task_config:
+ adios_xml_agg: set_by_deepdrivemd
+ adios_xml_agg_4ml: set_by_deepdrivemd
+ compute_rmsd: true
+ experiment_directory: set_by_deepdrivemd
+ model: cvae
+ n_sim: 12
+ node_local_path: set_by_deepdrivemd
+ num_tasks: 1
+ output_path: set_by_deepdrivemd
+ sleeptime_bpfiles: 30
+ stage_idx: 0
+ task_idx: 0
+machine_learning_stage:
+ arguments:
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/models/keras_cvae_stream/train.py
+ cpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: OpenMP
+ threads_per_process: 4
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
+ gpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: CUDA
+ threads_per_process: 1
+ num_tasks: 1
+ pre_exec:
+ - unset PYTHONPATH
+ - module load gcc/7.3.1
+ - . /etc/profile.d/conda.sh
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
+ - export IBM_POWERAI_LICENSE_ACCEPT=yes
+ - module use /usr/workspace/cv_ddmd/software1/modules
+ - module load adios2/2.8.1a
+ - export DDMD_DEBUG=1
+ task_config:
+ adios_xml_agg: set_by_deepdrivemd
+ agg_dir: /p/gpfs1/yakushin/Outputs/3mm/aggregation_runs/
+ batch_size: 32
+ checkpoint_dir: set_by_deepdrivemd
+ conv_filter_shapes:
+ - - 3
+ - 3
+ - - 3
+ - 3
+ - - 3
+ - 3
+ - - 3
+ - 3
+ conv_filters:
+ - 64
+ - 64
+ - 64
+ - 64
+ conv_layers: 4
+ conv_strides:
+ - - 1
+ - 1
+ - - 2
+ - 2
+ - - 1
+ - 1
+ - - 1
+ - 1
+ dense_dropouts:
+ - 0.25
+ dense_layers: 1
+ dense_neurons:
+ - 128
+ epochs: 1
+ experiment_directory: set_by_deepdrivemd
+ final_shape:
+ - 28
+ - 28
+ - 1
+ initial_shape:
+ - 28
+ - 28
+ latent_dim: 10
+ max_loss: 1000
+ max_steps: 2000
+ min_step_increment: 200
+ num_agg: 1
+ output_path: set_by_deepdrivemd
+ published_model_dir: set_by_deepdrivemd
+ read_batch: 2000
+ reinit: true
+ shuffle: true
+ split_pct: 0.8
+ stage_idx: 0
+ task_idx: 0
+ timeout1: 30
+ timeout2: 10
+ use_model_checkpoint: true
+molecular_dynamics_stage:
+ arguments:
+ - /g/g15/yakushin/tmp2/tmp/DeepDriveMD-pipeline/deepdrivemd/sim/openmm_stream/run_openmm.py
+ cpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: OpenMP
+ threads_per_process: 4
+ executable: /usr/workspace/cv_ddmd/conda3/powerai/bin/python
+ gpu_reqs:
+ process_type: null
+ processes: 1
+ thread_type: CUDA
+ threads_per_process: 1
+ num_tasks: 12
+ pre_exec:
+ - unset PYTHONPATH
+ - module load gcc/7.3.1
+ - . /etc/profile.d/conda.sh
+ - conda activate /usr/workspace/cv_ddmd/conda3/powerai
+ - export IBM_POWERAI_LICENSE_ACCEPT=yes
+ - module use /usr/workspace/cv_ddmd/software1/modules
+ - module load adios2/2.8.1a
+ - export DDMD_DEBUG=1
+ task_config:
+ adios_xml_file: set_by_deepdrivemd
+ adios_xml_sim: set_by_deepdrivemd
+ bp_file: set_by_deepdrivemd
+ compute_rmsd: true
+ copy_velocities_p: 0.5
+ dt_ps: 0.002
+ experiment_directory: set_by_deepdrivemd
+ heat_bath_friction_coef: 1.0
+ in_memory: false
+ init_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input/1FME-0.pdb
+ initial_pdb_dir: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input
+ lock: set_by_deepdrivemd
+ mda_selection: protein and name CA
+ model: cvae
+ node_local_path: set_by_deepdrivemd
+ openmm_selection:
+ - CA
+ outliers_dir: /p/gpfs1/yakushin/Outputs/3mm/agent_runs/stage0000/task0000/published_outliers
+ output_path: set_by_deepdrivemd
+ pdb_file: set_by_deepdrivemd
+ reference_pdb_file: /usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb
+ report_interval_ps: 1
+ simulation_length_ns: 0.1
+ solvent_type: implicit
+ stage_idx: 0
+ task_idx: 0
+ temperature_kelvin: 300.0
+ threshold: 8.0
+ top_suffix: null
+
diff --git a/test/bba/lassen-aae_stream_smoothended_rec/generate.py.old b/test/bba/test1_stream3/generate.py
similarity index 76%
rename from test/bba/lassen-aae_stream_smoothended_rec/generate.py.old
rename to test/bba/test1_stream3/generate.py
index 82ad669b..4f6e6212 100755
--- a/test/bba/lassen-aae_stream_smoothended_rec/generate.py.old
+++ b/test/bba/test1_stream3/generate.py
@@ -6,39 +6,54 @@
import yaml
from pydantic import BaseModel
+import sys
+import subprocess
+
+DDMD="/".join(os.getenv('PWD').split("/")[:-3])
+PATH=os.getenv('PATH')
+USER=os.getenv('USER')
+LD_LIBRARY_PATH=os.getenv('LD_LIBRARY_PATH')
+PYTHONPATH=os.getenv('PYTHONPATH')
+PYTHON=subprocess.getstatusoutput('which python')[1]
+ADIOS2="/".join(subprocess.getstatusoutput('which bpls')[1].split("/")[:-2])
+CONDA="/".join(PYTHON.split("/")[:-2])
class Header(BaseModel):
- title = "smoothended_rec mini"
+ title = "BBA integration test"
resource = "llnl.lassen"
queue = "pbatch"
schema_ = "local"
project = "cv19-a01"
- walltime_min = 60 * 3
+ walltime_min = 30
max_iteration = 4
cpus_per_node = 40
gpus_per_node = 4
hardware_threads_per_cpu = 4
- experiment_directory = "/p/gpfs1/yakushin/Outputs/18m"
+ experiment_directory = f"/p/gpfs1/{USER}/Outputs/3mm"
software_directory = (
- "/usr/workspace/cv_ddmd/yakushin/Integration1/DeepDriveMD-pipeline/deepdrivemd"
+ f"{DDMD}/deepdrivemd"
)
node_local_path: Path = None
- init_pdb_file = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/system/comp.pdb"
- ref_pdb_file: Path = init_pdb_file
+ init_pdb_file = (
+ "/usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input/1FME-0.pdb"
+ )
+ ref_pdb_file = (
+ "/usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_reference/1FME.pdb"
+ )
config_directory = "set_by_deepdrivemd"
adios_xml_sim = "set_by_deepdrivemd"
adios_xml_agg = "set_by_deepdrivemd"
- adios_xml_agg_4ml = "set_by_deepdrivemd"
adios_xml_file = "set_by_deepdrivemd"
- model = "aae"
+ adios_xml_agg_4ml = "set_by_deepdrivemd"
+ model = "cvae"
header = Header()
print(yaml.dump(header.dict()))
-pythonpath = os.getenv("PYTHONPATH")
-python = "/usr/workspace/cv_ddmd/conda1/powerai/bin/python"
+#pythonpath = os.getenv("PYTHONPATH")
+#python = "/usr/workspace/cv_ddmd/conda2/powerai/bin/python"
class CPUReqMD(BaseModel):
@@ -68,11 +83,11 @@ class TaskConfigMD(BaseModel):
output_path = "set_by_deepdrivemd"
node_local_path = "set_by_deepdrivemd"
pdb_file = "set_by_deepdrivemd"
- initial_pdb_dir = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/BigMolecules/smoothended_rec/"
- solvent_type = "explicit"
- top_suffix: str = ".top"
- simulation_length_ns = 10.0 / 20
- report_interval_ps = 50.0 / 2
+ initial_pdb_dir = "/usr/workspace/cv_ddmd/yakushin/Integration1/data/bba/ddmd_input"
+ solvent_type = "implicit"
+ top_suffix: str = None
+ simulation_length_ns = 0.1
+ report_interval_ps = 1
dt_ps = 0.002
temperature_kelvin = 300.0
heat_bath_friction_coef = 1.0
@@ -83,15 +98,12 @@ class TaskConfigMD(BaseModel):
in_memory = False
bp_file = "set_by_deepdrivemd"
outliers_dir = f"{header.experiment_directory}/agent_runs/stage0000/task0000/published_outliers"
- copy_velocities_p = 1.0
- next_outlier_policy = 1
+ copy_velocities_p = 0.5
lock = "set_by_deepdrivemd"
adios_xml_sim = header.adios_xml_sim
adios_xml_file = header.adios_xml_file
compute_rmsd = True
- divisibleby = 32
- zcentroid_atoms = "resname CY8 and not name H*"
- init_pdb_file = f"{header.init_pdb_file}"
+ init_pdb_file = header.init_pdb_file
model = header.model
@@ -101,17 +113,17 @@ class TaskConfigMD(BaseModel):
"unset PYTHONPATH",
"module load gcc/7.3.1",
". /etc/profile.d/conda.sh",
- "conda activate /usr/workspace/cv_ddmd/conda1/powerai",
+ f"conda activate {CONDA}",
"export IBM_POWERAI_LICENSE_ACCEPT=yes",
"module use /usr/workspace/cv_ddmd/software1/modules",
- "module load adios2",
- f"export PYTHONPATH={pythonpath}",
+ "module load adios2/2.8.1a",
+ "export DDMD_DEBUG=1",
]
class MD(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/sim/openmm_stream/run_openmm.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
@@ -142,8 +154,8 @@ class TaskConfigAgg(BaseModel):
n_sim = md.num_tasks
sleeptime_bpfiles = 30
adios_xml_agg = header.adios_xml_agg
- adios_xml_agg_4ml = header.adios_xml_agg_4ml
compute_rmsd = task_config_md.compute_rmsd
+ adios_xml_agg_4ml = header.adios_xml_agg_4ml
model = header.model
@@ -152,7 +164,7 @@ class TaskConfigAgg(BaseModel):
class Aggregator(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/aggregation/stream/aggregator.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_agg.dict()
@@ -165,18 +177,18 @@ class Aggregator(BaseModel):
class CVAE(BaseModel):
- initial_shape = [458, 458]
- final_shape = [458, 458, 1]
+ initial_shape = [28, 28]
+ final_shape = [28, 28, 1]
split_pct = 0.8
shuffle = True
latent_dim = 10
conv_layers = 4
- conv_filters = [32, 64, 64, 64]
- conv_filter_shapes = [[5, 5], [3, 3], [3, 3], [3, 3]]
- conv_strides = [[2, 2], [2, 2], [2, 2], [2, 2]]
+ conv_filters = [64] * 4
+ conv_filter_shapes = [[3, 3]] * 4
+ conv_strides = [[1, 1], [2, 2], [1, 1], [1, 1]]
dense_layers = 1
dense_neurons = [128]
- dense_dropouts = [0.4]
+ dense_dropouts = [0.25]
class TaskConfigML(CVAE):
@@ -184,11 +196,11 @@ class TaskConfigML(CVAE):
stage_idx = 0
task_idx = 0
output_path = "set_by_deepdrivemd"
- epochs = 70
+ epochs = 1
batch_size = 32
- min_step_increment = 600
- max_steps = 600
- max_loss = 1500
+ min_step_increment = 200
+ max_steps = 2000
+ max_loss = 1000
num_agg = agg.num_tasks
timeout1 = 30
timeout2 = 10
@@ -196,11 +208,9 @@ class TaskConfigML(CVAE):
published_model_dir = "set_by_deepdrivemd"
checkpoint_dir = "set_by_deepdrivemd"
adios_xml_agg = header.adios_xml_agg
- adios_xml_agg_4ml = header.adios_xml_agg_4ml
reinit = True
use_model_checkpoint = True
- read_batch = 600
- model = header.model
+ read_batch = 2000
task_config_ml = TaskConfigML()
@@ -208,11 +218,12 @@ class TaskConfigML(CVAE):
class ML(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/models/keras_cvae_stream/train.py"]
cpu_reqs = cpu_req_md.dict()
gpu_reqs = gpu_req_md.dict()
task_config = task_config_ml.dict()
+ num_tasks = 1
cpu_req_agent = cpu_req_md.copy()
@@ -227,27 +238,27 @@ class TaskConfigAgent(CVAE):
agg_dir = f"{header.experiment_directory}/aggregation_runs"
num_agg = agg.num_tasks
- min_step_increment = 200
+ min_step_increment = 500
timeout1 = 30
timeout2 = 10
best_model = f"{header.experiment_directory}/machine_learning_runs/stage0000/task0000/published_model/best.h5"
- lastN = 200
+ lastN = 2000
outlier_count = 120
- outlier_max = 2000
- outlier_min = 120
+ outlier_max = 1000
+ outlier_min = 500
init_pdb_file = f"{header.init_pdb_file}"
ref_pdb_file = f"{header.ref_pdb_file}"
init_eps = 1.3
init_min_samples = 10
- read_batch = 200
+ read_batch = 2000
num_sim = md.num_tasks
project_lastN = 50 * 1000
project_gpu = False
adios_xml_agg = header.adios_xml_agg
use_outliers = True
- use_random_outliers = True
+ use_random_outliers = False
compute_rmsd = task_config_md.compute_rmsd
- outlier_selection = "lof"
+ adios_xml_agg_4ml = header.adios_xml_agg_4ml
model = header.model
@@ -256,11 +267,12 @@ class TaskConfigAgent(CVAE):
class Agent(BaseModel):
pre_exec = pre_exec_md
- executable = python
+ executable = PYTHON
arguments = [f"{header.software_directory}/agents/stream/dbscan.py"]
cpu_reqs = cpu_req_agent.dict()
gpu_reqs = gpu_req_md.dict()
task_config = task_config_agent.dict()
+ num_tasks = 1
class Components(BaseModel):
diff --git a/test/topology/smoothened_receptor_cvae.yml b/test/topology/smoothened_receptor_cvae.yml
index 16e91897..32d6c46a 100644
--- a/test/topology/smoothened_receptor_cvae.yml
+++ b/test/topology/smoothened_receptor_cvae.yml
@@ -8,7 +8,7 @@ max_iteration: 20
cpus_per_node: 40
gpus_per_node: 4
hardware_threads_per_cpu: 4
-experiment_directory: /p/gpfs1/brace3/runs/ddmd_production_runs/smoothened_receptor_ddmd_run-0
+experiment_directory: /p/gpfs1/brace3/runs/ddmd_production_runs/smoothened_receptor_ddmd_run-1
node_local_path: /tmp
molecular_dynamics_stage:
pre_exec:
@@ -41,7 +41,7 @@ molecular_dynamics_stage:
solvent_type: explicit
top_suffix: .top
simulation_length_ns: 2.0
- report_interval_ps: 10.0
+ report_interval_ps: 5.0
dt_ps: 0.002
temperature_kelvin: 300.0
heat_bath_friction_coef: 1.0
@@ -217,7 +217,7 @@ agent_stage:
num_extrinsic_outliers: 120
intrinsic_score: dbscan_lof_outlier
extrinsic_score: null
- n_traj_frames: 200
+ n_traj_frames: 500
n_most_recent_h5_files: 120
k_random_old_h5_files: 120
sklearn_num_jobs: -1