"""
Multisim startup script.
This will launch the driver: chorus_multijob.py.
Copyright Schrodinger, LLC. All rights reserved.
"""
# Contributors: Yujie Wu
# FIXME move this script into the mmshare/python/scripts/startup_scripts/ dir.
import copy
import os
from pathlib import Path
import shutil
import sys
import time
from past.utils import old_div
import schrodinger.application.desmond.cmdline as cmdline
import schrodinger.application.desmond.cmj as cmj
import schrodinger.application.desmond.envir as envir
import schrodinger.application.desmond.util as util
import schrodinger.job.jobcontrol as jobcontrol
import schrodinger.job.launcher as launcher
from schrodinger.application.desmond.stage import workflow
from schrodinger.application.desmond.starter.ui.cmdline import check_jobname
from schrodinger.utils import fileutils
from schrodinger.utils import sea
[docs]class Help(object):
"""
"""
VERSION = "multisim v" + cmj.VERSION
SHORT_USAGE = """
Multisim workflow program
Usage:
* To run a new job:
_APPLICATION_NAME_ [<options>] {-m <file>.msj} {<file>.cms|<file>.mae}
* To restart a previous job:
_APPLICATION_NAME_ [<options>] {-RESTART <multisim_checkpoint_file>}
You normally also need to specify the '-d' option.
Frequently used options:
-m <input_file> input .msj file defining the basic workflow
-d <input_file> stage data file from the previous job and usually with
the .tgz suffix. This option should be used only in
restarting a job with multisim checkpoint file.
-c <input_file> set configuration parameters for a family of stages.
-o <output_file> output .cms or .mae file. This file can be incorporated
into Maestro.
-maxjob <number> maximum number of simultaneous subjobs (default is 1)
-mode {umbrella} run in non-default execution mode. Currently, there is
only 1 valid value: umbrella. In umbrella mode, subjobs
will run on the nodes allocated to the main job.
-notify <email_addr> send a notification email when this job terminates.
-nopref ignore the multisim preference file.
-probe probe a checkpoint file and quit.
-JOBNAME <job_name> name of this multisim job
-HOST <host_name> host machine to run the multisim main job (default is
localhost)
-SUBHOST <host_name> host machine to run subjobs
-RESTART <input_file> restart a previous job from its multisim checkpoint file.
-h [<topic>] show help and quit. User can optionally specify a topic
for help. Valid topics are the following: @all, @doc,
@<flag> (where <flag> represents the flag of any valid
option, e.g. @-help), @examples, and @deprecated.
For more detail see help of the '@-help' topic.
"""
LONG_USAGE = """
Multisim workflow program
Usage:
* To run a new job:
_APPLICATION_NAME_ [<options>] {-m <file>.msj} {<file>.cms|<file>.mae}
* To restart a previous job:
_APPLICATION_NAME_ [<options>] {-RESTART <multisim_checkpoint_file>}
You normally also need to specify the '-d' option.
I/O options:
-i <input file> input structure file in the .cms or .mae format
(This option has been DEPRECATED, please specify the
.cms or .mae file without the -i flag in the future)
-m <input_file> input .msj file defining the basic workflow
-RESTART <input_file> multisim checkpoint file for restarting a job.
-r <input file> multisim checkpoint file for restarting a job.
(This option has been DEPRECATED, please use -RESTART
instead in the future)
-d <input_file> stage data file from the previous job and usually with
the .tgz suffix. This option should be used only in
restarting a job with multisim checkpoint file.
-c <input_file> set configuration parameters for a family of stages.
-ADD_FILE <input_file> Additional input file to copy to the working directory
of this multisim job
-ALT_DIR <directory> Alternative directory where to look for input files
-o <output_file> output .cms or .mae file. This file can be incorporated
into Maestro.
Workflow options:
-m <input_file> input .msj file defining the basic workflow
-c <input_file> set configuration parameters for a family of stages.
-set <string> modify the basic workflow defined by the .msj file.
Job options:
-JOBNAME <job_name> name of this multisim job
-HOST <host_name> host machine to run the multisim main job (default is
localhost)
-SUBHOST <host_name> host machine to run subjobs
-host <host_name> host machine to run subjobs.
(This option has been DEPRECATED, please use -SUBHOST
instead in the future)
-RETRIES <number> maximum times to retry failed subjobs (default is 3).
-max_retries <number> maximum times to retry failed subjobs (default is 3).
(This option has been DEPRECATED, please use -RETRIES
instead in the future)
-TMPDIR <dir> specify a tmpdir setting for this job
-WAIT do not return until the job completes.
-SAVE return zip archive of job directory at job completion.
-NICE run the job at reduced priority.
-maxjob <number> maximum number of simultaneous subjobs (default is 1)
-mode {umbrella} run in non-default execution mode. Currently, there is
only 1 valid value: umbrella. In umbrella mode, subjobs
will run on the nodes allocated to the main job.
-max-walltime <number> number of seconds to run the job and then checkpoint.
Other options:
-description <text> job description to print to the log
-notify <email_addr> send a notification email when this job terminates.
-nopref ignore the multisim preference file.
-probe probe a checkpoint file and quit.
-quiet light log information (default)
-verbose heavy log information
-debug turn on multisim debug mode.
-DEBUG turn on both multisim and jobcontrol debug modes.
-v, -version show the program's version and exit.
-h, -help, -HELP [<topic>]
show help and quit. User can optionally specify a topic
for help. Valid topics are the following: @all, @doc,
@<flag> (where <flag> represents the flag of any valid
option, e.g. @-help), @example, and @deprecated.
For more detail see help of the '@-help' topic.
"""
TOPIC = {
"@deprecated": """
The following options have been deprecated in this release and will be removed
in the future:
-r Use -RESTART instead.
-i Specify the input structure file without the '-i' flag.
-host Use -SUBHOST instead.
-max_retries Use -RETRIES instead.""",
"@examples": """_APPLICATION_NAME_ -JOBNAME my_ligand -m my_protocol.msj my_structure.cms
Launch a new job with the name of 'my_ligand'. The input structure file is
my_structure.cms, and the workflow is defined by the my_protocol.msj file.
_APPLICATION_NAME_ -JOBNAME "my ligand" -m my_protocol.msj my_structure.cms
Similar to the above example, but note the space and the quotes in the value of
the '-JOBNAME' option.
In general, if a value has spaces in it, the value needs to be quoted.
_APPLICATION_NAME_ -JOBNAME my_ligand_restart -RESTART my_ligand-multisim_checkpoint -d my_ligand-6.tgz
Restart a previous job using multisim checkpoint file. The job will be restarted
from the point where the previous job was interrupted. You usually need to
provide the data file of the stage prior to the interrupted one. In this example,
we assume the previous job was interrupted at stage 7, so we specify the -d option
with the my_ligand_6.tgz file.
_APPLICATION_NAME_ -JOBNAME my_ligand_restart -RESTART my_ligand-multisim_checkpoint:5 -d my_ligand-4.tgz
Restart a previous job using multisim checkpoint file. The job will be restarted
from stage 5 (which should be either completed or be the point where the previous
job was interrupted).
""",
"@-m": """
The value after the -m flag should specify a .msj file that defines the basic
workflow for this multisim job. The .msj file must be specified for a new job,
but it is optional for restarting a job from multisim checkpoint file. The
basic workflow defined by the .msj file can be altered using the -set and -c
options.""",
"@-d": """
This option should be used only for restarting a job from multisim
checkpoint file. The value after the -d flag must specify a stage-data file
produced by the previous job. Stage-data files usually have a .tgz extension.
In a restarted job if the stages to run will read the files of the previous
stages, you need to specify the stage-data files of these previous stages
using the -d option. The -d option can be specified multiple times to include
all needed stage-data files.""",
"@-c": """
In general, the value after the -c flag should specify a configuration file,
where the settings will affect a family of stages of the basic workflow (as
defined by the .msj file). Only Desmond stage family is supported so far. So the
value after the -c flag currently must specify a Desmond front-end .cfg file.
The settings from the configuration file will effect unless the same settings
are explicitly mentioned in the stages.""",
"@-o": """
The value after the -o flag specifies the output structure file name. If an
output structure is produced, it will be written to the file with the specified
filename. If no structure will be produced, the -o option will have no effect.""",
"@-max-walltime": """
The number of seconds to run the job before automatically checkpointing.
If this is part of a subjob, it will request that the main job automatically
restart this job. This can be useful for running on a resource that limits
jobs to a certain run time.""",
"@-maxjob": """
Set the maximum number of simultaneously active subjobs. Active jobs are those
in the waiting or running state. Any non-negative integer is a valid value for
this option. The default value is 1, which means at any moment of the main
job's life time there is at most 1 active subjob, and only when this subjob is
completed will another subjob be launched. Value 0 is also a valid value, but
its exact semantics depends on multisim mode (see help of the -mode option). In
the default mode, value 0 means that the number of simulataneously active
subjobs is unlimited. If you run subjobs on a cluster with a queuing system, you
probably want to use 0 in the default mode. In the 'umbrella' mode, value 0 is
equivalent to 1.""",
"@-mode": """
Multisim will run in a predefined but nondefault mode. Only one nondefault mode
is supported so far: umbrella. You can activate this mode by adding
"-mode umbrella" (without quotes) in your command.
The umbrella mode will let all subjobs run on the nodes allocated to the main
job, which avoids submitting the subjobs into the queueing system and thus the
waiting time. The umbrella mode ignores the -SUBHOST option, so specify the
compute host using the -HOST option.""",
"@-notify": """
The value after the flag should be an email address. When the main job
terminates multisim will send an email (with multisim log file attached) to the
email address. You can repeat the option multiple times to specify more than one
email addresses.""",
"@-nopref": """
User can customize default values of certain options in the
$HOME/.schrodinger/multisim_preference file. This -nopref switch causes multisim
to ignore this file.""",
"@-probe": """
Probe a checkpoint file and quit. The value after the flag should specify a
multisim checkpoint file. Multisim will show the previous job's state as
recorded in the checkpoint file. In addition, multisim will indicate whether
the checkpoint file is compatible (and thus restartable) with the current
version of multisim.""",
"@-JOBNAME": """
The value after the flag specifies the name of this multisim job. If this
option is omitted, multisim will automatically generate a unique job name using
your username, current date and time. The automatic name is after the pattern
<username><yyyymmdd>T<hhmmss>, e.g., bgates20101120T193036.""",
"@-HOST": """
Specify the host machine to run the multisim main job. If this option is
omitted, the default host will be localhost. The main job normally requires
only 1 processor to run. In some cases (for example, when you run the job in the
umbrella mode -- see the help of the -mode option), you want the main job to
run on more than 1 processor. For such cases, you can explicitly request the
number of processors for the main job using the host syntax:
<hostname>:<number_of_processors>, e.g., -HOST robin_para:8. Note that you
should use a parallel queue if you want the main job to gain multiple
processors on a queued host.""",
"@-SUBHOST": """
Specify the default host machine to run subjobs. Unless a stage has explicitly
set a different host to run its subjobs, subjobs will run on the machine
specified via the -SUBHOST option. This option does not support the
<hostname>:<number_of_processors> syntax. In umbrella mode, this option will be
ignored, and so the compute host should be specified using the -HOST option.""",
"@-RESTART": """
The value after the -RESTART flag should specify a multisim checkpoint file for
restarting a job. You can explicitly specify the restarting stage using the
syntax: <multisim_checkpoint_file>:<stage_number>. The restarted stage must be
either a previously completed stage or the interrupted stage. If the restarting
stage is not explicitly specified, multisim will restart the job from the
interruption point. The restarted job will run, following the previous workflow.
If you want it to follow an altered workflow, you can specify the workflow via
the -m option.""",
"@-h": """
Let multisim show online help and then quit. Aliases of the -h flag are: -HELP
and -help. A value can be optionally given after the flag to specify a
particular topic for help. If no value is given, multisim will show concise help
for the most frequently used options. Valid values (or topics) are the following:
@all - Show concise help of all valid options.
@doc - Show detailed help of all valid options.
@<flag> - Show detailed help of the <flag> option, e.g. @-h.
@examples - Show a few command examples.
@deprecated - Show a list of deprecated options.""",
"@-help": """
Let multisim show online help and then quit. Aliases of the -help flag are:
-HELP and -h. A value can be optionally given after the flag to specify a
particular topic for help. If no value is given, multisim will show concise help
for the most frequently used options. Valid values (or topics) are the following:
@all - Show concise help of all valid options.
@doc - Show detailed help of all valid options.
@<flag> - Show detailed help of the <flag> option, e.g. @-h.
@examples - Show a few command examples.
@deprecated - Show a list of deprecated options.""",
"@-HELP": """
Let multisim show online help and then quit. Aliases of the -HELP flag are:
-help and -h. A value can be optionally given after the flag to specify a
particular topic for help. If no value is given, multisim will show concise help
for the most frequently used options. Valid values (or topics) are the following:
@all - Show concise help of all valid options.
@doc - Show detailed help of all valid options.
@<flag> - Show detailed help of the <flag> option, e.g. @-h.
@examples - Show a few command examples.
@deprecated - Show a list of deprecated options.""",
"@-i": """
The value after the -i flag should specify a structure input file (either .mae
or .cms file). This option has been deprecated. You can specify the structure
input file without the -i flag.""",
"@-r": """
This option has been deprecated. The -r flag is temporarily an alias of
-RESTART. See the help for the -RESTART option for detail. Please try not to
use this flag, use -RESTART instead.""",
"@-ADD_FILE": """
If your workflow needs additional input files to run, you can specify these
input files after the -ADD_FILE flag. You can repeat the -ADD_FILE flag multiple
times with each one followed by one file so to include all necessary files. All
the input files will be automatically copied to the host of the main job.
Normally, you do not need to use this option since multisim can do a pretty good
job in figuring out the necessary input files as mentioned in either the command
or the .msj file.""",
"@-ALT_DIR": """
When multisim looks for a file with relative path name, it will search the
current directory by default. To let multisim also search other directories,
user can specify the directories with this option. The searching order will be
the current directory first, then user-specified directories. You can set the
-ALT_DIR option multiple times to specify multiple search directories.""",
"@-set": """
This option enables you to mutate the basic workflow through the command line
interface. For example, if you want to change the value of stage 3's time
parameter, you can include this setting in your command:
-set "stage[3].time=2.0"
. You can repeat the -set flag multiple times to include all workflow
mutations.""",
"@-host": """
This option has been deprecated. The -host flag is temporarily an alias of
-SUBHOST. See the help for the latter for detail. Please try not to use this
flag, use -SUBHOST instead.""",
"@-RETRIES": """
Specify the maximum times to retry failed subjobs. By default, multisim retries
for at most 3 times.""",
"@-max_retries": """
This option has been deprecated. The -max_retries flag is temporarily an alias
of -RETRIES. See the help for the latter for detail. Please try not to use this
flag, use -RETRIES instead.""",
"@-USER": """
Use a different username to launch the job. This option has been DEPRECATED, and
this feature will likely be removed in future releases. Please try not to use
this option any more.""",
"@-LOCAL": """
If you want to run main job in the local directory, you can add this option to
your command. Note that the local directory must be accessible on the host of
the main job, otherwise your job will fail. For this reason, this option is
used most times when the main job's host is localhost. Also note that this
option affects the main job only.""",
"@-TMPDIR": """
The value after the flag should specify a scratch dir where this multisim job
will run.""",
"@-WAIT": """
With this option the launch command will not finish/return until this job
terminates.""",
"@-SAVE": """
Let jobcontrol return the zip archive of this job directory at the job
completion.""",
"@-NICE": """
Let the multisim job run at reduced priority.""",
"@-description": """
This value after the flag should be a text, which will be printed to the
main log file. If the text contains spaces, then it must be quoted.""",
"@-quiet": """
With this option multisim will print minimum amount of information to the main
log file. This is the default log level.""",
"@-verbose": """
With this option multisim will print a heavy amount of information to the main
log file.""",
"@-debug": """
Turn on multisim debug mode -- multisim will print not only normal log
information but also degugging information.""",
"@-DEBUG": """
Turn on both multisim and jobcontrol debug modes.""",
"@-v": """
Show the program's version and then quit. The -version flag is an alias of the
-v flag.""",
"@-version": """
Show the program's version and then quit. The -v flag is an alias of the
-version flag.""",
}
[docs] def __init__(self, app_name):
"""
"""
self._app_name = app_name
[docs] def print_version(self):
"""
- Prints version information.
"""
print(self.VERSION)
[docs] def print_help(self, topic=None):
"""
- Print help information.
"""
if (topic == "@doc"):
doc = sorted(self.TOPIC.items())
for t, help in doc:
print("* %s" % t[1:])
print("%s\n" %
help.replace("_APPLICATION_NAME_", self._app_name))
print()
elif (topic is None):
print(self.SHORT_USAGE.replace("_APPLICATION_NAME_",
self._app_name))
elif (topic == "@all"):
print(self.LONG_USAGE.replace("_APPLICATION_NAME_", self._app_name))
else:
try:
print("* %s" % topic[1:])
print("%s\n" % self.TOPIC[topic].replace(
"_APPLICATION_NAME_", self._app_name))
print()
except KeyError:
print("ERROR: Unavailable topic: %s" % topic)
print("""Available topics are the following:
@all - concise description of all options
@examples - command examples
@doc - detailed explanation of all options
@<flag> - detailed explanation of a particular option
Example: @-h.
""")
[docs]class Startup(object):
"""
"""
APPLICATION_NAME = os.path.join("$SCHRODINGER", "utilities", "multisim")
SCRIPT = "chorus_multijob.py"
Help = Help
[docs] def __init__(self, argv, orig_argv=None):
"""
"""
self._argv = argv[1:]
self._orig_argv = copy.copy(orig_argv if (orig_argv) else argv)
self._should_skip = False
# Gets arguments that were swallowed by the shell script from environment variables.
self._get_arg_from_envir("-USER", "JOBUSER")
self._get_arg_from_envir("-TMPDIR", "SCHRODINGER_TMPDIR")
self._get_arg_from_envir("-NICE", "SCHRODINGER_NICE", False)
# Gets HOST.
host = envir.get("JOBHOST")
self._argv = [
"-HOST",
(host if (host) else "localhost"),
] + self._argv
# Gets SUBHOST.
host = envir.get("SCHRODINGER_NODELIST")
self._argv = [
"-SUBHOST",
(host if (host) else "localhost"),
] + self._argv
# Command flags
self.LOCAL = None
self.TMPDIR = None
self.WAIT = None
self.OPLSDIR = None
self.SAVE = None
self.NICE = None
self.JOBNAME = None
self.USER = None
self.HOST = None
self.ADD_FILE = []
self.ALT_DIR = []
self.lic = []
self.notify = []
self.DEBUG = None
self.debug = None
self.verbose = None
self.quiet = None
self.nopref = None
self.prog = "multisim"
self.probe = None
self.mode = None
self.description = ""
self.relay_arg = None
self.cpu = 1
self.ncpu = 1
self.ncpu_main = 1
self.pset = []
self.maxjob = None
self.max_retries = None
self.max_walltime = None
self.cfg = None
self.host = None
self.cms_output = None
self.cms_input = None
self.msj_input = None
self.rst_input = None
self.rst_stage = None
self.dat_input = []
[docs] def print_version(self):
"""
"""
self.Help(self.APPLICATION_NAME).print_version()
sys.exit(0)
[docs] def print_usage(self, i_argv=None, text=None):
"""
"""
topic = None
if (i_argv is not None):
try:
if (self._argv[i_argv + 1][0] == '@'):
topic = self._argv[i_argv + 1]
except IndexError:
pass
self.Help(self.APPLICATION_NAME).print_help(topic)
if (text is not None):
print(text)
sys.exit(0)
def _get_value(self, i):
"""
"""
try:
if (self._argv[i + 1][0] == '-'):
raise IndexError()
else:
self._should_skip = True
return self._argv[i + 1]
except IndexError:
self.print_usage(text="No value for the option: %s" % self._argv[i])
def _get_file(self, fname):
"""
Returns file pathname if we can find 'fname', otherwise returns None.
"""
if (os.path.isfile(fname)):
return fname
if (os.path.isabs(fname)):
return None
for d in self.ALT_DIR:
trial_fname = os.path.join(d, fname)
if (os.path.isfile(trial_fname)):
return util.relpath(trial_fname)
return None
def _add_script_arg(self, arg_vals):
"""
Construct a cmdline from given list of arguments. All arguments are a
2-tuple of argument_name, argument_value. All argument_value are
converted to str, except:
- If argument_value is a boolean, only argument_name appears in
output.
- If argument_value is a list, ("-argname", ["a", "b"]) converts to
"-argname a -argname b"
- If argument_value is None, argname is also dropped.
:param arg_val: List of argument values to append
:type arg_val: tuple(object, object)
:rtype: list(str)
:return: list of cmdline arguments
"""
script_args = []
for argname, argval in arg_vals:
# Ignore arguments that are null
if argval is None:
continue
if isinstance(argval, list):
for subval in argval:
script_args.append(argname)
script_args.append(subval)
else:
script_args.append(argname)
if isinstance(argval, bytes):
raise TypeError(
f"Won't coerce {argval} in cmdline argument, is bytes")
if not isinstance(argval, bool):
script_args.append(str(argval))
return script_args
def _make_ark_str(self, arg_val):
"""
"""
ark_str = ""
for av in arg_val:
arg = av[0]
val = av[1]
if (val is not None):
if (isinstance(val, list)):
ark_str += "" if (val == []) else ("%s = [%s] " % (
arg,
" ".join(val),
))
elif (isinstance(val, bool)):
ark_str += "%s = %s " % (
arg,
str(val).lower(),
)
else:
ark_str += "%s = %s " % (
arg,
cmj.escape_string(str(val)),
)
return ark_str
def _get_arg_from_envir(self, arg, envar, has_val=True):
"""
"""
if (arg not in self._argv):
try:
val = os.environ[envar]
if (has_val):
if (val != ""):
self._argv += [
arg,
val,
]
else:
self._argv.append(arg)
except KeyError:
pass
def _parse_arg(self):
"""
"""
if (len(self._orig_argv) < 1):
self.print_usage()
sys.exit(1)
for i, arg in enumerate(self._argv):
if (self._should_skip):
self._should_skip = False
elif (arg in [
"-h",
"-help",
"-HELP",
]):
self.print_usage(i)
elif (arg in [
"-v",
"-version",
]):
self.print_version()
elif (arg == "-LOCAL"):
self.LOCAL = True
elif (arg == "-TMPDIR"):
self.TMPDIR = self._get_value(i)
elif (arg == "-WAIT"):
self.WAIT = True
elif (arg == "-OPLSDIR"):
self.OPLSDIR = self._get_value(i)
elif (arg == "-SAVE"):
self.SAVE = True
elif (arg == "-NICE"):
self.NICE = True
elif (arg == "-JOBNAME"):
self.JOBNAME = self._get_value(i)
elif (arg == "-HOST"):
self.HOST = self._get_value(i)
elif (arg == "-ADD_FILE"):
self.ADD_FILE.append(self._get_value(i))
elif (arg == "-ALT_DIR"):
self.ALT_DIR.append(self._get_value(i))
elif (arg == "-lic"):
self.lic.append(self._get_value(i))
elif (arg == "-notify"):
self.notify.append(self._get_value(i))
elif (arg == "-nopref"):
self.nopref = True
elif (arg == "-DEBUG"):
self.DEBUG = True
elif (arg == "-debug"):
self.debug = True
elif (arg == "-verbose"):
self.verbose = True
elif (arg == "-quiet"):
self.quiet = True
elif (arg == "-_margorp_"):
self.prog = self._get_value(i)
elif (arg == "-probe"):
self.probe = self._get_value(i)
elif (arg == "-mode"):
self.mode = self._get_value(i)
elif (arg == "-description"):
self.description = self._get_value(i)
elif (arg == "-encoded_description"):
self.description = cmdline.get_b64decoded_str(
self._get_value(i))
elif (arg == "-cpu"):
self.cpu = self._get_value(i)
if self.cpu and int(self.cpu) != 1:
self.cpu = 1
print("WARNING: The -cpu option has been deprecated and "
"the default value of 1 is used instead.")
elif (arg == "-set"):
self.pset.append(self._get_value(i))
elif (arg == "-maxjob"):
self.maxjob = self._get_value(i)
elif (arg == "-RETRIES"):
self.max_retries = self._get_value(i)
elif arg == "-max-walltime":
self.max_walltime = int(self._get_value(i))
elif (arg == "-c"):
self.cfg = self._get_value(i)
elif (arg == "-SUBHOST"):
self.host = self._get_value(i)
elif (arg == "-o"):
self.cms_output = self._get_value(i)
elif (arg == "-m"):
self.msj_input = self._get_value(i)
elif (arg == "-RESTART"):
self.rst_input = self._get_value(i)
elif (arg == "-d"):
self.dat_input.append(self._get_value(i))
elif (arg == "-i"):
print("WARNING: The -i option has been deprecated. Please specify the input .cms or .mae file without\n" \
" the -i flag in the future.")
self.cms_input = self._get_value(i)
elif (arg == "-USER"):
print(
"WARNING: The -USER option has been deprecated. Please do not use this option in the future."
)
self.USER = self._get_value(i)
elif (arg == "-max_retries"):
print(
"WARNING: The -max_retries option has been deprecated. Please use -RETRIES instead in the future."
)
self.max_retries = self._get_value(i)
elif (arg == "-host"):
print(
"WARNING: The -host option has been deprecated. Please use -SUBHOST instead in the future."
)
self.host = self._get_value(i)
elif (arg == "-r"):
print(
"WARNING: The -r option has been deprecated. Please use -RESTART instead in the future."
)
self.rst_input = self._get_value(i)
else:
if ((arg[-4:] not in [
".mae",
".cms",
]) and (arg[-7:] not in [
".mae.gz",
".cms.gz",
]) and (arg[-6:] not in [
".maegz",
".cmsgz",
])):
self.print_usage(text="ERROR: Unrecognized option: %s" %
arg)
else:
self.cms_input = arg
if (self.rst_input):
rst = self.rst_input.split(':')
n = len(rst)
if (n > 2 or n < 1):
print("ERROR: Wrong value for the '-RESTART' option: %s" %
self.rst_input)
print(
"Syntax for values of this option: <file-name> | <file-name:stage-number>"
)
print(
"To restart from the uncompleted stage, use the first syntax."
)
print(
"To restart from a completed stage, use the second syntax.")
sys.exit(1)
self.rst_input = rst[0]
if (n == 2):
self.rst_stage = int(rst[1])
def _check_arg(self):
"""
This checks some argument settings and should be called after
`self._parse_arg` method.
"""
if (self.probe):
if (self._get_file(self.probe)):
try:
cmj.probe_checkpoint(self.probe)
except:
raise
print("ERROR: Probing failed. Bad checkpoint file.")
sys.exit(1)
sys.exit(0)
else:
print("ERROR: File %s not found." % self.probe)
sys.exit(1)
if (self.rst_input is None):
if (self.cms_input is None):
print("ERROR: Input structure file not provided.")
sys.exit(1)
else:
self.ADD_FILE.append(self.cms_input)
if (self.msj_input is None):
print("ERROR: Input file not provided for the '-m' argument.")
sys.exit(1)
else:
if self.JOBNAME:
backup_files(self.JOBNAME)
self.ADD_FILE.append(self.rst_input)
for fname in self.dat_input:
self.ADD_FILE.append(fname)
if self.msj_input:
if self._get_file(self.msj_input) is None:
print("ERROR: Can't find MSJ file %s" % self.msj_input)
sys.exit(1)
self.ADD_FILE.append(self.msj_input)
if (self.cfg is not None):
self.ADD_FILE.append(self.cfg)
# Checks if a nightly build is being used.
if (-1 != os.environ["SCHRODINGER"].find("/NB/")):
print("WARNING: It seems you are using a nightly build (NB):")
print(" %s" % (os.environ["SCHRODINGER"],))
print(
" A NB could be erased before job completion and thus cause your job to fail."
)
if (self.maxjob is not None):
try:
int(self.maxjob)
except ValueError:
print(
"ERROR: Value for the '-maxjob' option must be an integer number."
)
sys.exit(1)
elif (self.mode != "umbrella"):
print(
"WARNING: You did not specify a value for '-maxjob'. Remember its default value is 1."
)
if (self.max_retries is not None):
try:
int(self.max_retries)
except ValueError:
print(
"ERROR: Value for the '-max_retries' option must be an integer number."
)
sys.exit(1)
if self.JOBNAME is not None:
err = check_jobname(self.JOBNAME)
if err:
sys.exit(err)
def _treat_arg(self):
"""
"""
self.pset = None if (self.pset == []) else chr(30).join(self.pset)
if self.description:
# called when fep_plus calls multisim
self.description += "\n"
self.description += cmdline.get_job_command_in_startup()
if self.mode == "umbrella":
self.description += "\nMultisim runs in the umbrella mode."
if (self.host != "localhost"):
# -HOST's value can be assigned to -SUBHOST as if -SUBHOST was specified in the launch command.
# We need to check the original arguments to make sure if -SUBHOST was indeed in the launch command or not.
try:
TOPLEVEL_HOST_ARGS = os.environ["TOPLEVEL_HOST_ARGS"]
except KeyError:
print(
"ERROR: You must specify -HOST when using umbrella mode."
)
sys.exit(1)
if (TOPLEVEL_HOST_ARGS.find("-SUBHOST") >= 0):
print("WARNING: Umbrella mode ignores the -SUBHOST option.")
self.host = None
if (self.maxjob is None):
maxjob = 1
else:
maxjob = int(self.maxjob)
if (maxjob == 0):
maxjob = 1
ncpu_sum = 0
num_none = 0
for (hostname, ncpu) in jobcontrol.get_command_line_host_list():
if (ncpu is None):
ncpu_sum += 1
num_none += 1
else:
ncpu_sum += ncpu
if (ncpu_sum == 1 and num_none == 1):
# User does not explicitly specify the number of CPUs, so we figure out it from the -cpu and -maxjob options.
if (not self.ncpu):
self.ncpu = 1
self.cpu = 1
ncpu = self.ncpu
self.ncpu_main = ncpu * maxjob
else:
self.ncpu_main = ncpu_sum
# Addresses the issue at DESMOND-4363.
if maxjob == 1 and self.msj_input:
maxjob = old_div(self.ncpu_main, (self.ncpu or 1))
try:
stage_list = cmj.parse_msj(self._get_file(self.msj_input))
except cmj.ParseError as e:
print(
"ERROR: " + str(e) +
"\nPlease correct the setting errors and relaunch the job."
)
sys.exit(1)
has_remd = False
for stage in stage_list:
if (stage.NAME in ["replica_exchange", "lambda_hopping"]):
has_remd = True
break
if (has_remd):
if (maxjob > 1):
print("NOTE: Automatically set '-maxjob %d'." % maxjob)
if (self.pset):
self.pset += chr(30)
self.pset += "stage[1].set_family.remd.total_proc=%d" % self.ncpu_main
else:
self.pset = "stage[1].set_family.remd.total_proc=%d" % self.ncpu_main
print(
"NOTE: Automatically set 'stage[1].set_family.remd.total_proc=%d'."
% self.ncpu_main)
# Now we detect whether the host is non-parallel queued host.
host = jobcontrol.get_host(hostname)
is_parallel = True #if (host.qargs and (-1 != host.qargs.lower().find( "-pe" ))) else False
if (not is_parallel and host.isQueue()):
if (self.ncpu_main > 1):
print(
"ERROR: Cannot request multiple processors on a nonparallel queue '%s'"
% hostname)
print(" Use a parallel queue instead.")
sys.exit(1)
self.maxjob = maxjob
# Makes relay arguments.
# N.B.: "-set", "-ADD_FILE", "-description", and "WAIT" options are not relayed.
self.relay_arg = self._make_ark_str([
(
"SUBHOST",
self.host,
),
(
"cfg",
self.cfg,
),
(
"RETRIES",
self.max_retries,
),
(
"maxjob",
self.maxjob,
),
(
"cpu",
self.cpu,
),
(
"mode",
self.mode,
),
("notify", self.notify),
("nopref", self.nopref),
(
"quiet",
self.quiet,
),
(
"verbose",
self.verbose,
),
(
"debug",
self.debug,
),
(
"DEBUG",
self.DEBUG,
),
(
"HOST",
self.HOST,
),
(
"JOBNAME",
self.JOBNAME,
),
(
"NICE",
self.NICE,
),
(
"SAVE",
self.SAVE,
),
(
"TMPDIR",
self.TMPDIR,
),
(
"LOCAL",
self.LOCAL,
),
])
def _treat_preference(self):
"""
"""
pref_item_tier1 = {
"JOBNAME": "JOBNAME",
}
pref_item_tier2 = {
"host": "host",
"max_retries": "max_retries",
"maxjob": "maxjob",
"notify": "notify",
"struct_output": "cms_output",
"quiet": "quiet",
"verbose": "verbose",
"debug": "debug",
"DEBUG": "DEBUG",
}
pref_item_excl = [
"quiet",
"verbose",
"debug",
"DEBUG",
]
preference_file = os.path.join(
envir.CONST.HOME, ".schrodinger",
"multisim_preference") if (envir.CONST.HOME) else None
if (self.JOBNAME is None):
self.JOBNAME = os.environ.get("SCHRODINGER_JOBNAME")
if (preference_file and os.path.isfile(preference_file)):
macro_dict = {
"$USERNAME": self.USER,
"$TIMESTAMP": time.strftime("%Y%m%dT%H%M%S")
}
if (self.msj_input):
macro_dict["$MSJBASENAME"] = os.path.splitext(
os.path.basename(self.msj_input))[0]
if (self.cms_input):
macro_dict["$ARGBASENAME"] = os.path.splitext(
os.path.basename(self.cms_input))[0]
sea.update_macro_dict(macro_dict)
contents = open(preference_file).read()
pref = sea.Map(contents).bval
for e in pref:
k = e.orig_key()
if (k in pref_item_tier1 and not self.__dict__[e]):
print(
"NOTE: Got '%s' setting from multisim preference: %s" %
(
e,
pref[e],
))
self.__dict__[pref_item_tier1[k]] = pref[e].val
if (not self.JOBNAME):
self.JOBNAME = util.getlogin() + time.strftime("%Y%m%dT%H%M%S")
print("Using an automatically-generated job name: %s" %
self.JOBNAME)
# TODO: Here for backwards compability
sea.update_macro_dict({
"$MAINJOBNAME": self.JOBNAME,
"$MASTERJOBNAME": self.JOBNAME
})
pref = sea.Map(contents).bval
for e in pref:
k = e.orig_key()
if ((k in pref_item_excl and not (self.quiet or self.verbose or
self.debug or self.DEBUG)) or
(k in pref_item_tier2 and
not self.__dict__[pref_item_tier2[k]])):
print(
"NOTE: Got '%s' setting from multisim preference: %s" %
(
e,
pref[e],
))
self.__dict__[pref_item_tier2[k]] = pref[e].val
[docs] def get_launcher(self):
"""
"""
self._parse_arg()
if (not self.nopref):
self._treat_preference()
self._check_arg()
self._treat_arg()
if (not self.JOBNAME):
self.JOBNAME = util.getlogin() + time.strftime("%Y%m%dT%H%M%S")
print("Using an automatically-generated job name: %s" %
self.JOBNAME)
# Sets MSJ log level.
if (self.quiet):
cmj.GENERAL_LOGLEVEL = "quiet"
if (self.verbose):
cmj.GENERAL_LOGLEVEL = "verbose"
if (self.debug):
cmj.GENERAL_LOGLEVEL = "debug"
if (self.DEBUG):
cmj.GENERAL_LOGLEVEL = "debug"
scriptlauncher = launcher.Launcher(
script=self.SCRIPT,
jobname=self.JOBNAME,
local=self.LOCAL,
tmpdir=self.TMPDIR,
save=self.SAVE,
wait=self.WAIT,
nproc=self.ncpu_main,
prog=self.prog,
oplsdir=self.OPLSDIR,
stoppable=True,
)
def get_scratch_filename(fname):
if (not fname):
return None
if (os.path.isabs(fname)):
return fname
# For relative pathnames, we need to find the shortest equivalent name. For example, "foo/bar/baz/../../../far"
# should be shorten to "far".
fname = util.relpath(fname)
# Then we know if it is under the current dir.
if (fname[:2] == ".." and not self.LOCAL):
return os.path.basename(fname)
return fname
def get_transfer_filename(fname):
# JC is not very intelligent to properly deal with convoluted pathname like "foo/bar/baz/./../../../far".
# So we have to help it a little by converting the given pathname to the shortest equivalent name.
if (not fname):
return None
if (os.path.isabs(fname)):
return fname
return util.relpath(fname)
cms_input_ = get_scratch_filename(self.cms_input)
msj_input_ = get_scratch_filename(self.msj_input)
rst_input_ = get_scratch_filename(self.rst_input)
script_arg = self._add_script_arg([(
"-i",
cms_input_,
), (
"-m",
msj_input_,
), (
"-r",
rst_input_,
), (
"-o",
self.cms_output,
), (
"-refrom",
self.rst_stage,
), (
"-jobname",
self.JOBNAME,
), (
"-host",
self.host,
), (
"-user",
self.USER,
), (
"-cfg",
self.cfg,
), (
"-maxjob",
self.maxjob,
), (
"-max_retries",
self.max_retries,
), (
"-launch_dir",
os.getcwd(),
), (
"-cpu",
self.cpu,
), (
"-set",
self.pset,
), ("-encoded_description",
cmdline.get_b64encoded_str(self.description)),
("-notify", self.notify),
(
"-relay_arg",
self.relay_arg,
), (
"-quiet",
self.quiet,
), (
"-verbose",
self.verbose,
),
("-max-walltime", self.max_walltime)
])
if (self.dat_input):
for fname in self.dat_input:
script_arg += ["-d", get_scratch_filename(fname)]
if (self.DEBUG or self.debug):
script_arg.append("-debug")
list(map(scriptlauncher.addLicense, self.lic))
DESMOND_STAGE_NAMES = workflow.PREDEFINED_STAGE_FAMILY["desmond"].union(
set(["analysis", "fep_analysis"]))
# TODO: Here for backwards compability
macro_dict = {
"$MAINJOBNAME": self.JOBNAME,
"$MASTERJOBNAME": self.JOBNAME,
"$USERNAME": self.USER
}
sea.update_macro_dict(macro_dict)
if (self.msj_input):
# Checks settings in the .msj file.
if (not self._get_file(self.msj_input)):
print("ERROR: Input .msj file not found: %s" % self.msj_input)
sys.exit(1)
try:
stage_list = cmj.parse_msj(self._get_file(self.msj_input),
pset=self.pset)
except cmj.ParseError as e:
print(
"ERROR: " + str(e) +
"\nPlease correct the setting errors and relaunch the job.")
sys.exit(1)
# Collects additional input files specified in the .msj file.
self.ADD_FILE += cmj.collect_inputfile(stage_list)
invalid_asl = cmj.validate_asl_expr(stage_list)
if (invalid_asl):
print("ERROR: Invalid ASL expressions found in .msj file:")
for e in invalid_asl:
print(" ", e)
sys.exit(1)
if (self.rst_input is not None):
if (not self._get_file(self.rst_input)):
print("ERROR: Checkpoint file not found: %s" % self.rst_input)
sys.exit(1)
fh = open(self._get_file(self.rst_input), "rb")
engine = cmj.Engine.deserialize(fh)
fh.close()
if (not cmj.is_restartable_version(engine.version)):
print("Current version of multisim is %s," % cmj.VERSION)
print(
"whereas this checkpoint file was generated by version %s."
% engine.version)
print(
"This checkpoint file cannot be restarted with the current version of multisim."
)
sys.exit(1)
if not cmj.is_restartable_build(engine):
print(
"This workflow cannot be restarted with the Academic version of Desmond/Maestro."
)
sys.exit(1)
if (not self.msj_input):
try:
stage_list = cmj.parse_msj(None, engine.msj_content,
self.pset)
except cmj.ParseError as e:
print(
"ERROR: " + str(e) +
"Please correct the setting errors and relaunch the job."
)
sys.exit(1)
self.ADD_FILE += cmj.collect_inputfile(stage_list)
if (self.rst_stage is not None):
if (self.rst_stage == 1):
print(
"Why don't you rerun the job from scratch using the original input structure file?"
)
sys.exit(1)
stage_state = [s.__getstate__() for s in engine.stage]
engine.stage = cmj.build_stages(stage_list,
stage_state=stage_state)
engine._find_restart_stage()
if (engine.restart_stage and stage_list.index(
engine.restart_stage) < self.rst_stage):
# Allow restart if the rst_stage is an extend stage and the previous stage completed
if not (stage_list.index(engine.restart_stage) + 1
== self.rst_stage and
stage_list[self.rst_stage].NAME
== 'desmond_extend'):
print("ERROR: Cannot restart from stage %d." %
self.rst_stage)
print(
" You can restart from either the first uncompleted stage or a completed stage."
)
sys.exit(1)
stage_names = set(e.NAME for e in stage_list)
if stage_names.intersection(DESMOND_STAGE_NAMES):
script_arg.extend([
"-FROM",
"desmond",
])
scriptlauncher.addScriptArgs(script_arg)
# Changes the default log file name to be <jobname_multisim.log>.
scriptlauncher.setStdOutErrFile(self.JOBNAME + "_multisim.log")
# Checks whether all input files exist.
for fname in self.ADD_FILE:
if (not self._get_file(fname)):
print("ERROR: Input file not found: " + fname)
sys.exit(1)
# Adds input files to jlaunch input file list.
for fname in self.ADD_FILE:
scriptlauncher.addInputFile(
get_transfer_filename(self._get_file(fname)))
return scriptlauncher
[docs] def launch(self, env=None):
"""
Create and launch multisim job
"""
scriptlauncher = self.get_launcher()
# Sets environment variables.
if env:
for e in env:
scriptlauncher.addEnv(e)
# Launches the script.
return scriptlauncher.launch()
[docs]def get_licenses(cmd):
"""
Get the licenses being used in the given multisim command
:param cmd: the multisim command
:type cmd: List[str]
:return: the licenses being used
:rtype: list[str]
"""
startup = Startup(cmd)
startup.get_launcher() # loads lic
return startup.lic
[docs]def backup_files(jobname: str):
"""
Backs-up the primary output files from multisim into a new backup directory
Only creates the backup directory if one of the files exists
"""
backup_dir = Path(
fileutils.get_next_filename_prefix(f"{jobname}_backup", "_"))
_backup_file(backup_dir, jobname + "-multisim_checkpoint", "checkpoint")
_backup_file(backup_dir, jobname + "_multisim.log", "log")
_backup_file(backup_dir, jobname + "_out.fmp", "output fmp")
_backup_file(backup_dir, jobname + "_out.fmpdb", "output fmpdb")
def _backup_file(backup_dir: Path, file_name: str, file_description: str):
"""
If file_name exists, create the backup directory and copy the file into it
"""
if os.path.isfile(file_name):
if not os.path.isdir(backup_dir):
os.mkdir(backup_dir)
new_path = backup_dir / file_name
print(f"Back up the {file_description} file: {file_name} => {new_path}")
shutil.copy(file_name, new_path)
if (__name__ == "__main__"):
Startup(sys.argv).launch()