Source code for schrodinger.application.licensing.licadmin

import argparse
import codecs
import collections
import filecmp
import getpass
import glob
import json
import logging
import os
import re
import shutil
import socket
import stat
import sys
import tempfile
from collections import Counter
from collections import OrderedDict
from collections import defaultdict
from contextlib import contextmanager
from datetime import date
from datetime import datetime
from datetime import time
from pathlib import Path
from string import Template
from tempfile import TemporaryDirectory
from zipfile import ZipFile

import psutil

from schrodinger.utils import fileutils
from schrodinger.utils import subprocess
from schrodinger.utils import sysinfo

from . import flexlm
from . import licerror
from . import netsuiteapi

# Configure logging
fmt = logging.Formatter('%(message)s')
log = logging.getLogger("licadmin")
console_handler = logging.StreamHandler(stream=sys.stdout)
console_handler.setFormatter(fmt)
log.setLevel(logging.INFO)
log.addHandler(console_handler)

# Environment
SCHRODINGER = os.getenv('SCHRODINGER', "")
if SCHRODINGER == "":
    raise Exception("SCHRODINGER is undefined")
MMSHARE_EXEC = glob.glob(SCHRODINGER + '/mmshare-v*/bin/[A-Z]*')[0]
MMSHARE_DATA = glob.glob(SCHRODINGER + '/mmshare-v*/data')[0]

# Constants
DEFAULT_LMGRD_PORT_RANGE = range(27000, 27010)
SCHROD_DAEMON_CHECK_TIMEOUT = 60  # seconds
LICENSE_DIRNAME = "licenses"
LICENSE_FILENAME = "license"
LMGRD_LOG = "lmgrd.log"
LMGRD_ALTLOG = "lmgrd.%s.log" % getpass.getuser()
VERSION_UNKNOWN = "Unknown"
_EXE = '.exe' if sys.platform == 'win32' else ''
LMUTIL = os.getenv("SCHRODINGER_LMUTIL",
                   os.path.join(MMSHARE_EXEC, "lmutil") + _EXE)
LMGRD = os.path.join(os.path.dirname(LMUTIL), "lmgrd" + _EXE)

if sys.platform == "darwin":
    SHARED_LICENSE_DIR = "/Library/Application Support/Schrodinger"
elif sys.platform == "win32":
    SHARED_LICENSE_DIR = r"C:\ProgramData\Schrodinger"
else:
    SHARED_LICENSE_DIR = "/opt/schrodinger"

JOB_REQS_FILE = os.path.join(MMSHARE_DATA, "license_reqs.json")
JOB_PRODUCTS_FILE = os.path.join(MMSHARE_DATA, "job_products.json")

SCHROD_DAEMON_NOT_STARTED = "License server machine is down"
EXPIRED_MESSAGE = "License has expired"

REQUIRED_MAJOR_VERSION = 11
REQUIRED_MINOR_VERSION = 15
LICENSE_SERVER_UPGRADE_MESSAGE = "License server %s will no longer be " \
                                 "supported in Suite 2019-1"

TEMPORARY_LICENSE = "downloaded.lic"

# Code to represent uncounted tokens or jobs
UNCOUNTED = object()

# Module state

# all license resources
_resources = None

# The local server process
_local_server = None

# License requirements for each type of job
_job_reqs = {}

# Products under which job types are grouped
_job_products = []

_tokens_loaded = defaultdict(dict)
_features_loaded = defaultdict(dict)
_server_refresh = True


[docs]def set_server_refresh(refresh): """ This function allows you to prevent _load_tokens from actually hitting the license servers/files again. This is dangerous unless you're sure you're the only user of licadmin.py, and you really don't want to reload the tokens. PANEL-5146 :param refresh: Whether to refresh tokens when _load_tokens is called :type refresh: bool """ global _server_refresh _server_refresh = refresh
[docs]class Status(object): """ A Status object represents the "health" of a license resource. It's an immutable object, with integer and string values. """ _objects = {} def __new__(cls, *args): if args[0] in cls._objects: instance = cls._objects[args[0]] else: instance = object.__new__(cls) cls._objects[args[0]] = instance return instance
[docs] def __init__(self, code, description): self.code = code self._description = description
def __repr__(self): return self._description def __unicode__(self): return self._description
[docs]class MissingExecutableError(Exception): """ If the requested path doesn't point to a valid executable. Used SCHRODINGER_LMUTIL does not point to a valid lmutil. """
[docs]class MissingLicenseFile(Exception): """If no valid server license can be found."""
RESOURCE_OK = Status(0, "License OK") RESOURCE_PROBLEMS = Status(1, "License may have problems") RESOURCE_HAS_FUTURE_START = Status(2, "License has start date in the future") RESOURCE_UNAVAILABLE = Status(3, "License is unusable") # Named tuples stat_info = collections.namedtuple("stat_info", "total used") ####################################################################### # Classes #######################################################################
[docs]class LicenseFeature(object): """ A LicenseFeature object represents a particular license feature provided by a LicenseResource. It records the license information reported by the lmdiag utility. There may be more than one LicenseFeature for a given feature name. """
[docs] def __init__(self, name, version="", location="", lictype="", hostid="", starts="", tokens=0, expires=None, ok=None, error=None): self.name = name self.version = version self.location = location self.lictype = lictype self.suite = "SUITE" in name self.hostid = hostid self.starts = starts self.expires = expires self.ok = ok self.error = error self.tokens = 0
def __repr__(self): return " ".join((self.name, "v" + self.version, self.lictype))
[docs] def set_starts(self, starts): self.starts = datetime.strptime(starts, "%d-%b-%Y")
[docs] def set_expires(self, expires): self.expires = datetime.strptime(expires, "%d-%b-%Y")
[docs] def set_hostid(self, hostid): self.hostid = hostid
[docs] def set_tokens(self, tokens): self.tokens = tokens
[docs]class LicenseResource(object): """ A LicenseResource object represents a single license file or license server (port@host). The object records how the resource was found and what licenses it provides. """ # Versions with a known memory leak issue. LEAK_VERSIONS = ['v11.12']
[docs] def __init__(self, location, source=None): """ :param location: pathname of license file or port@host address of a license server :type location: pathlib.Path or str :param str source: name of the environment variable that specified this resource, or the pathname of the directory where it was found. """ self.source = source self._pathname = "" self._server = "" self._version = "" self._license = None self._tokens = {} self._features = {} self._file_exists = None self._health = None self._errors = [] if isinstance(location, Path) or "@" not in location: self._pathname = os.path.abspath(str(location)) self._load_license_file(location) else: self._server = location self._serverstart = None if self.is_local_server(): self._logfile = server_log(self.pathname()) else: self._logfile = "" self._last_offset = 0
def __unicode__(self): return "LicenseResource: %s (%s)" % (self.location(), self.source) def __repr__(self): return self.__unicode__()
[docs] def errors(self): result = self._errors upgrade_server_advice = ( f"We recommend upgrading your server to version " f"{REQUIRED_MAJOR_VERSION}.{REQUIRED_MINOR_VERSION}, which is " f"included in our distribution for your convenience.") if self.outdated_server(): error = licerror.LicenseError( 0, LICENSE_SERVER_UPGRADE_MESSAGE % (self._version), licerror.WARNING) error._advice = upgrade_server_advice result.append(error) if self.memory_leak_server(): error = licerror.LicenseError( 0, "Memory issue with FlexNet server version.", licerror.WARNING) error._description = ( "This resource is running FlexNet server version %s. " "Because of a bug in the FlexNet code, the server will " "experience a small memory leak with each Schrodinger license " "checkout." % self._version) error._advice = upgrade_server_advice result.append(error) return result
[docs] def outdated_server(self): return self.is_server() and not version_uptodate(self._version)
[docs] def memory_leak_server(self): return (self.is_server() and self._version.startswith(tuple(self.LEAK_VERSIONS)))
[docs] def location(self): if self._server: return self._server else: return self._pathname
[docs] def pathname(self): """ The pathname for the license file. If the license file location isn't known, return the empty string. """ return self._pathname
[docs] def server_pathname(self): """ The license pathnames to use if we're starting a license server for this license file. If the file is in a licenses/ directory, we start the server using all of the server license files for this machine in the directory, not just this one. The pathnames are returned as a single string, with pathnames separated by the system path separator, e.g., "/licenses/server1.lic:/licenses/license2.lic". """ if not self.pathname(): return "" dirname = os.path.dirname(self.pathname()) log.debug("[server_pathname] pathname = %s" % self.pathname()) log.debug("[server_pathname] dirname = %s" % dirname) log.debug("[server_pathname] basename %s" % os.path.basename(dirname)) if os.path.basename(dirname) == LICENSE_DIRNAME: return os.pathsep.join(local_server_files(dirname)) return self.pathname()
[docs] def server_version(self): if self.is_server(): return self._version else: return ""
[docs] def refresh(self): if self._pathname: location = self._pathname else: location = self._server self.__init__(location, self.source)
[docs] def is_server(self): return bool(self._server)
[docs] def is_stub(self): if self._pathname and self._license: return self._license.is_stub() else: return False
[docs] def logfile(self): return self._logfile
[docs] def server_hostnames(self): """ Returns the list of hostnames for this license server. If we have a license file, the hostnames from the SERVER lines in that file are returned, in an arbitrary order. If we have a port@host address, the list will just include the host name from that address. """ if self._server: if self._file_exists: return [s.host for s in self._license.server.values()] else: host, _ = self.server_host_and_port() return [host] return []
[docs] def server_hostids(self): """ Returns the list of hostids from the SERVER lines in the license file. The order in which the hostids is returned is arbitrary, and shouldn't be expected to match the order of SERVER lines in the file. """ if self._server and self._file_exists: return [s.hostid for s in self._license.server.values()] return []
[docs] def server_hostid(self): """ Returns the set of hostid for the SERVER line in the license file. If there's more than one SERVER line, one of them is arbitrarily chosen, which probably isn't what we want. """ hostids = self.server_hostids() if hostids: return hostids[0] return ""
[docs] def server_host_and_port(self): """ Returns the hostname and port number for the server as a tuple (host, port). (Both values are strings.) The port number may be an empty string, if it's not specified in the license. The host name will be empty if this license resource doesn't represent a server. """ if self._server: (port, host) = self._server.split("@", 1) return (host, port) return ("", "")
[docs] def schrod_port(self): """ Returns the port number for the SCHROD daemon, as a string. The empty string may be returned if the port number isn't specified in the license, or if this license resource doesn't represent a server. """ if self._server: try: return str(self._license.vendor["SCHROD"].port()) except KeyError: return ""
[docs] def canonical_filename(self, preserve_if_possible=True): """ Returns the license object's canonical base filename If the filename already matches the canonical pattern, return it. """ filename = self.pathname() if filename and flexlm.matches_canonical_filename(filename): return os.path.basename(filename) return os.path.basename(self._license.canonical_filename())
[docs] def stub_license(self): host, port = self.server_host_and_port() return create_stub_license(host, port)
[docs] def is_local_server(self): """ Return true if this resource represents a license server that is can be started on the current machine. """ if not self.is_server(): return False return (any([hostid_is_local(h) for h in self.server_hostids()]) or any([hostname_is_local(n) for n in self.server_hostnames()]))
[docs] def lmdiag(self): """ Run lmdiag for this license resource. This is mainly useful for debugging. The raw output from the command is returned. """ return execute_lmutil(["lmdiag", "-n", "-c", self.location()])
[docs] def lmstat(self): """ Run lmstat for this license resource, if it represents a license server. This is mainly useful for debugging. The raw output from the command is returned. """ if self.is_server(): try: return execute_lmutil(["lmstat", "-c", self.location()]) except subprocess.CalledProcessError as exc: return exc.output else: return ""
[docs] def checkout(self, token, version=None, count=None, exists=False): """ Try to check out the specified token from this license resource. A version number and token count can be specified, as well, but this shouldn't be necessary for most purposes. If the exists argument is True, then an existence check is done instead of a checkout. Returns True if the checkout succeeded, otherwise False. """ arg = token try: out = execute_lictest(["-d", "-f", self.location(), arg]) log.debug(out) return True except subprocess.CalledProcessError as exc: log.debug(exc.output) return False
[docs] def health(self): """ Return a status code indicating the general "health" of this resource. RESOURCE_OK means the resource can provide licenses. RESOURCE_PROBLEMS means all licenses that should be available aren't. RESOURCE_UNAVAILABLE means this resource is unable to provide licenses. Because this evaluation requires executing lmdiag and lmstat, it may take a few seconds. """ self._errors = [] self._load_tokens() info = None if self._server and self._logfile: info = parse_lmgrd_log(self._logfile, self._last_offset) if info: if info.version: self._version = info.version self._errors.extend(info.errors) if not self._errors and len(self._features) > 0: self._health = RESOURCE_OK return self._health elif is_future_license_error(info): self._health = RESOURCE_HAS_FUTURE_START self._errors = [ licerror.LicenseError( licerror.SCHROD_NO_FEATURES_CODE, "Features in the license file have start dates " + "in the future .") ] else: self._health = RESOURCE_UNAVAILABLE for error in self._errors: if error.level != licerror.ERROR: self._health = RESOURCE_PROBLEMS return self._health
[docs] def summary(self): """ """ health = self.health() lines = [] if self._license: lines.append(self._license.license_description()) if self._server: lines.append("Server: %s" % self._server) if self._pathname: lines.append("License File: %s" % self._pathname) lines.append("(found in %s)" % self.source) lines.append("Status: %s" % health) if self._errors: for error in self._errors: lines.append(str(error)) return "\n".join(lines)
[docs] def add_error(self, error): """ """ assert type(error) == licerror.LicenseError self._errors.append(error)
def _load_license_file(self, pathname): """ Load the license file. """ try: with open(pathname, "r") as fp: text = fp.read() self._license = flexlm.License(text) self._server = self._license.server_hostport() self._file_exists = True except Exception as exc: message = "License file unreadable" self._file_exists = False self.add_error(licerror.LicenseError(0, message, licerror.ERROR))
[docs] def ident(self): """ Return an identifier that can be used to decide whether two license resources are really the same. Two server-based licenses are considered the same if the server hostnames are the same. (There can only be one server per host.) All non-server-based license files are considered unique. """ if self._server: host, _ = self.server_host_and_port() return "@" + shorthost(host) else: return self._pathname
[docs] def priority(self): """ Return a priority score for this resource. (Lower numbers indicate higher priority, with zero the highest priority.) """ if self.is_server(): (host, port) = self.server_host_and_port() if self.pathname(): # license file (may or may not be for a server) if not self.is_stub(): return 0 # full license file elif port: return 1 # stub license file with port number else: return 2 # stub license without port number elif port: return 3 # port@host address else: return 4 # @host address else: return 0 # node-locked license
def _load_tokens(self): """ Load the information about the total number of tokens available and the number currently in use. For server based licenses, this will query the server. The info is recorded in the tokens attribute, a dict that maps feature names to stat_info tuples. """ # We need to run lmdiag to determine if these features are relevant self._get_features() if self._server: self._get_server_tokens() else: self._get_license_tokens()
[docs] def tokens(self): self._load_tokens() return self._tokens
[docs] def available_tokens(self): """ Return a dict mapping feature names to the number of tokens available for each. """ self._load_tokens() available = defaultdict(int) for name, tokens in self._tokens.items(): if tokens.total == UNCOUNTED: # uncounted feature available[name] = UNCOUNTED elif available[name] != UNCOUNTED: available[name] = tokens.total - tokens.used return available
[docs] def total_tokens(self): """ Return a dict mapping feature names to the number of tokens provided by this resource for each. """ self._load_tokens() return dict([(k, v.total) for (k, v) in self._tokens.items()])
def _get_license_tokens(self): """ Populate the tokens table from the license object. """ tokens = defaultdict(int) if self._license is not None: for name, feats in self._features.items(): for feat in feats: if not feat.ok: continue if tokens[name] == UNCOUNTED: continue elif feat.tokens == UNCOUNTED: tokens[name] = UNCOUNTED else: tokens[name] += feat.tokens self._tokens = dict([(k, stat_info(v, 0)) for k, v in tokens.items()]) def _get_server_tokens(self): """ Get license usage information from a server via "lmutil lmstat". A dict mapping the token name to a named tuple recording the total tokens provided by this resource and the number currently checked out. """ global _server_refresh global _tokens_loaded if not _server_refresh and _tokens_loaded[self._server]: self._tokens = _tokens_loaded[self._server].copy() return self._tokens == {} try: attr = run_lmstat(self._server, parsefeatures=True) self._tokens = attr["tokens"] self._version = attr["version"] _tokens_loaded[self._server] = self._tokens.copy() except licerror.LicenseException as exc: if exc.error.unreachable_hostname: self._errors = [] self.add_error(exc.error)
[docs] def jobs(self, jobtypes=None): """ Return a dict mapping each job type to the number of jobs that can be run given the available tokens. """ _require_job_reqs() tokens = self.available_tokens() if not jobtypes: jobtypes = sorted(list(_job_reqs)) result = {} for jobtype in jobtypes: result[jobtype] = 0 for reqs in _job_reqs[jobtype]: # Uncounted is the most we can get, so bail out # as soon as we hit it if result[jobtype] == UNCOUNTED: break njobs = _single_job_capacity(reqs, tokens) if njobs == UNCOUNTED or njobs > result[jobtype]: result[jobtype] = njobs return result
def _get_features(self): """ Get license feature information via "lmutil lmdiag". A dict mapping the feature name to a list of LicenseFeature objects for that feature is stored in self._features. Errors in the found features are recorded in the list self._errors. """ global _server_refresh global _features_loaded if not _server_refresh and self.location() in _features_loaded: self._features = _features_loaded[self.location()].copy() return self._features = {} location = self.location() if not location: return try: diag = run_lmdiag(location) except licerror.LicenseException as exc: if not self._errors: message = "lmdiag failed: " + str(exc) self.add_error(licerror.LicenseError(0, message, licerror.ERROR)) return try: if location.startswith('@'): keys = [k for k in list(diag) if k.endswith(location)] location = keys[0] self._features = diag[location] _features_loaded[self.location()] = self._features.copy() except Exception as exc: message = "server missing from lmdiag output: " + str(exc) # TODO: Define error code? self.add_error(licerror.LicenseError(0, message, licerror.ERROR)) return # Don't report no tokens if we have an invalid license already if self._server or self._license: self._check_features() def _check_features(self): """ Check the results returned from lmdiag and record errors if there are expired fatures or similar problems. TODO: distinguish between a resource that offers some features and one which is completely unusable. """ error_count = Counter() ok_count = 0 for name, feats in self._features.items(): for feat in feats: if feat.ok: if not feat.suite: ok_count += 1 elif feat.error.level != licerror.IGNORE: error_count[(feat.error.message, feat.error.code)] += 1 if ok_count == 0: level = licerror.ERROR else: level = licerror.WARNING if len(error_count) > 0: for (msg, errcode) in list(error_count): if msg.startswith("Feature has expired"): if ok_count == 0: self.add_error( licerror.LicenseError(errcode, EXPIRED_MESSAGE, level, count=error_count[msg])) else: self.add_error( licerror.LicenseError( errcode, "License has some expired features", level, count=error_count[msg])) else: self.add_error( licerror.LicenseError(errcode, msg, level, count=error_count[msg])) elif ok_count == 0: self.add_error( licerror.LicenseError(0, "No license tokens found", level))
[docs]class LocalServerProcess(LicenseResource):
[docs] def __init__(self): LicenseResource.__init__(self, 'localhost') self._resources = [] self._running = None self._checked_resources = [] self._template_resource = None self._pathname = None self._check_res = LicenseResource('@localhost')
[docs] def errors(self): """ Return the errors of localserver license resources. """ errors = [] for res in self._resources: errors.extend(res.errors()) return errors
[docs] def numResources(self): return len(self._resources)
[docs] def clearResources(self): """ Clears the license resources that are registered with this process. """ self._resources = [] self._pathname = None self._health = None
[docs] def addResource(self, resource): """ Register a new license resource. The resource must be a local server license, and it must be good. Use checkResource() first to determine whether the license is good before adding. :param resource: the local server license resource to add :type resource: LicenseResource """ if self._template_resource is None: self.setTemplateResource(resource) self._resources.append(resource) if self._health != RESOURCE_OK: self._health = resource._health
[docs] def setTemplateResource(self, resource): """ Sets a known good license from which to obtain generic information, such as pathname, version, logfile, etc. :param resource: the local server license to use as a template :type resource: LicenseResource """ self.source = resource.source self._pathname = resource._pathname self._server = resource._server self._version = resource._version self._license = resource._license self._file_exists = resource._file_exists self._logfile = server_log(self.pathname())
[docs] def checkResource(self, resource): """ Check the health of a local server resource by using this local server process. This will by necessity stop and start the server process. If the license server was running before this method was called, it will be restarted. :param resource: the local server license to check :type resource: LicenseResource """ pathname = self._pathname or resource._pathname if self.isRunning(): originally_running = True self.stop(pathname) else: originally_running = False dirname = fileutils.get_directory_path(fileutils.TEMP) logfile = os.path.join(dirname, 'checklic.log') try: resource._logfile, resource._last_offset = self._start( [resource], logfile=logfile, append_log=False) except RuntimeError as e: resource.add_error(licerror.LicenseError(0, str(e), licerror.ERROR)) return RESOURCE_UNAVAILABLE # If the resource is @<host> without a port (for which the syntax # is <port>@<host>), checking health using 'lmutil lmdiag' might # give wrong information from non-SCHROD daemon if the port being # used by it is lesser than the SCHROD daemon. So it is safe to run # 'lmutil lmstat' and get the right port for SCHROD. if resource._server.startswith("@"): try: attr = run_lmstat(pathname) resource._server = attr.get("address", resource._server) except licerror.LicenseException as exc: resource.add_error(exc.error) health = resource.health() self.stop(pathname) self._checked_resources.append(resource) if originally_running: self.start() return health
[docs] def checkRunning(self): """ Checks whether the server process is currently running. """ self._check_res.health() errors = self._check_res.errors() for error in errors: if error.code == licerror.LMGRD_DOWN_ERROR_CODE: self._running = False return False else: self._running = True return self._running
[docs] def isRunning(self): """ Whether the server process is currently running. This will actively check the running state if it has not yet been checked. Otherwise, the running state is passively tracked via an internal variable. """ if self._running is None: self.checkRunning() return self._running
[docs] def health(self): """ Returns the health of server process if there are any license resources to serve. Otherwise returns RESOURCE_UNAVAILABLE and sets no errors. The health of the server process aggregates the health of all added resources. """ if not self._resources: self._health = RESOURCE_UNAVAILABLE self._errors = [ licerror.LicenseError( 0, "No valid licenses found for local license server.", licerror.ERROR) ] return self._health elif self._health == RESOURCE_OK: return self._health return LicenseResource.health(self)
[docs] def waitForSchrodDaemon(self, license_file, timeout=SCHROD_DAEMON_CHECK_TIMEOUT): """ Wait timeout seconds for lmstat to report if SCHROD vendor daemon is UP or DOWN. This will allow further operations to read the server log or get features to be reliable. """ start = datetime.now() elapsed = 0 while elapsed <= timeout: try: attr = run_lmstat(license_file) except licerror.LicenseException: # This can return -15 or -96 while starting up. More accurate # errors are read from the lmgrd log file after this function # is called. pass else: # Existence of 'schrod_up' say lmgrd started vendor daemon # and may be UP or DOWN. if "schrod_up" in attr: return elapsed = (datetime.now() - start).seconds return
def _start(self, resources, logfile=None, append_log=True): """ Start the given server resources - they do not necessarily have to be added to this server process object. It is up to the caller to ensure that the server is stopped before trying to start it. :param resources: licenses to start :type resources: list of LicenseResource :param logfile: the logfile to write lmgrd output :type logfile: str :param append_log: whether to append to the logfile or overwrite it :type append_log: bool """ filenames = [res._pathname for res in resources] if not os.path.exists(LMGRD): msg = f'Could not find an lmgrd executable at path {LMGRD}.' if 'SCHRODINGER_LMUTIL' in os.environ: msg += (' Double check value of SCHRODINGER_LMUTIL (' f"{os.environ['SCHRODINGER_LMUTIL']})") raise MissingExecutableError(msg) licfiles_arg = os.pathsep.join(filenames) if logfile is None: logfile = writable_server_log(licfiles_arg, logfile) if not logfile: error = licerror.LicenseError( 0, "Cannot write server log ('%s')" % logfile, licerror.ERROR) raise licerror.LicenseException(error) cmd = [LMGRD] offset = 0 if append_log: try: offset = os.stat(logfile).st_size except (OSError, IOError): pass logfile_args = '+' + logfile else: logfile_args = logfile cmd += ['-l', logfile_args] cmd += ['-c', licfiles_arg] # This allows the server to restart shortly after it was stopped. # The FlexNet docs say the behavior is undefined on Windows. if not sys.platform == 'win32': cmd.append("-reuseaddr") cmdstr = subprocess.list2cmdline(cmd) log.debug(">> " + cmdstr) try: subprocess.check_call(cmd) except subprocess.CalledProcessError as e: raise RuntimeError('Error starting server process "%s": %d' % (cmdstr, e.returncode)) # Make sure vendor daemon is up before reading log or getting # features. self.waitForSchrodDaemon(licfiles_arg) if self.checkRunning(): self._serverstart = datetime.now() return logfile, offset else: info = parse_lmgrd_log(logfile, offset) for err in info.errors: log.info(f"{err}") raise RuntimeError('Server process failed to start for following ' 'reason(s) -\n ' + '\n'.join(map(str, info.errors)))
[docs] def start(self, logfile=None, append_log=True): """ Starts the server process with all the local license resources that have been added. """ if not self._resources: return self._logfile, self._last_offset = \ self._start(self._resources, logfile, append_log)
[docs] def stop(self, pathname=None, anydaemon=False): """ Stops the server process (both SCHROD and LMGRD). :type anydaemon: bool :param anydaemon: If set to True, this will allow stopping non-SCHROD vendor daemon. This is mostly used for automated testing purpose. """ if not self.isRunning(): return if pathname is None: filenames = [res._pathname for res in self._resources] pathname = os.pathsep.join(filenames) if not pathname: return try: attr = run_lmstat(pathname) if "address" in attr and ("schrod_up" in attr or anydaemon): execute_lmutil(['lmdown', '-c', attr["address"], '-q']) except subprocess.CalledProcessError as e: # handle error from lmdown if lmgrd is already down if not e.returncode == licerror.LMGRD_DOWN_ERROR_CODE: raise RuntimeError( f"Error stopping server process (exit code {e.returncode})" f"\nOutput: {e.output}") except licerror.LicenseException as e: # handle error from lmstat if lmgrd has already gone down if not e.error.code == licerror.LMGRD_DOWN_ERROR_CODE: raise self._running = False
####################################################################### # Module functions ####################################################################### def _single_job_capacity(reqs, tokens): """ Calculate the number of jobs that can be run, for a job with the given token requirements and the given set of available tokens. Returns `UNCOUNTED` if an unlimited number of jobs can be run. The requirements are a dict, mapping token name to the number required. The tokens are also a dict, mapping token names to the number of available tokens. A token count of `UNCOUNTED` can be returned. """ njobs = None for name, reqd in reqs.items(): ntokens = tokens.get(name, 0) if ntokens == UNCOUNTED: # uncounted tokens if njobs is None: njobs = UNCOUNTED else: n = ntokens // reqd if njobs is None or njobs is UNCOUNTED or n < njobs: njobs = n return njobs
[docs]def healthier(health1, health2): """ Return True if health1 is healthier than health2. """ if health1 == RESOURCE_OK and health2 != RESOURCE_OK: return True if health1 == RESOURCE_PROBLEMS and health2 == RESOURCE_UNAVAILABLE: return True return False
[docs]def tokens(licenses): """ Report number of tokens provided by and currently available from the given license resources. Returns a dict mapping token name to stat_info tuple for each token. """ tokens = {} for lic in licenses: for k, v in lic.tokens().items(): try: total, used = tokens[k] if total == UNCOUNTED or v.total == UNCOUNTED: tokens[k] = (UNCOUNTED, used + v.used) else: tokens[k] = (total + v.total, used + v.used) except KeyError: tokens[k] = (v.total, v.used) return dict([(k, stat_info(*v)) for k, v in tokens.items()])
[docs]def job_capacity(resources): """ Report number of jobs that can be run with the tokens available on the given license resources. Returns a dict mapping the jobtype to the number of jobs that can be run, with licenses available from the given resources. """ jobs = defaultdict(int) for lic in resources: for k, v in lic.jobs().items(): if v == UNCOUNTED or jobs[k] == UNCOUNTED: jobs[k] = UNCOUNTED else: jobs[k] += v return jobs
[docs]def get_hostid(): """ Get the hostid for the current machine using "lmutil lmhostid". Generally, this will be the MAC address(es) for the ethernet interface. Returns the hostid string reported for the current host. """ hostid_out = execute_lmutil(["lmhostid"]) return parse_lmhostid_output(hostid_out.splitlines())
[docs]def get_first_hostid(): """ Get the hostid for the current machine using get_hostid(). This will always return the first MAC address found by 'lmutil lmhostid' for the current host. :return: first host id given by get_hostid() :rtype: str """ host_output = get_hostid() return host_output.split()[0]
[docs]def execute_lmutil(args): """ Execute the lmutil program with the given commandline arguments. The arguments must be specified as list. The raw output of the command is returned. If the exit code is non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, the output in the output attribute, and the command in the cmd attribute. If lmutil cannot be executed, raises a MissingExecutableError or an OSError. """ if not os.path.isfile(LMUTIL): msg = ('SCHRODINGER_LMUTIL does not point to a lmutil executable: ' f'"{LMUTIL}"') raise MissingExecutableError(msg) cmd = [LMUTIL] + args log.debug(">> " + subprocess.list2cmdline(cmd)) env = os.environ.copy() if 'FLEXLM_DIAGNOSTICS' in env: del env['FLEXLM_DIAGNOSTICS'] return subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT, errors='backslashreplace', universal_newlines=True)
[docs]def execute_lictest(args): """ Execute the lictest program with the given commandline arguments. The arguments must be specified as list. The raw output of the command is returned. If the exit code is non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, the output in the output attribute, and the command in the cmd attribute. """ lictest = os.path.join(SCHRODINGER, "utilities", "lictest") cmd = [lictest] + args log.debug(">> " + subprocess.list2cmdline(cmd)) return subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
[docs]def parse_lmhostid_output(lines): """ Parse the output from "lmutil lmhostid", provided as a list of output lines. Returns the hostid string for the current host. """ hostid_re = re.compile(r"Flexnet host ID .* is \"+([^\"]+)\"", re.IGNORECASE) for line in lines: match = hostid_re.search(line) if match: return match.group(1) return ""
[docs]def run_lmstat(license_file, parsefeatures=False): """ Get license usage information from a server via "lmutil lmstat". :type license_file: str :param license_file: Path to the license file to use with 'lmutil lmstat'. :type parsefeatures: bool :param parsefeatures: If set to True, the returned dict will have a mapping of the token name to a stat_info named tuple for each license feature. A stat_info tuple has integer fields 'total' and 'used', recording the total tokens provided and the number in use. :returntpe: dict :return: Dictionary containing the metadata of license server and vendor daemon corresponding to the given license file. """ if not license_file: raise RuntimeError("license_file should not be empty") try: lmstat_cmd = ["lmstat", "-c", license_file] if parsefeatures: lmstat_cmd += ["-f"] stat_out = execute_lmutil(lmstat_cmd) except subprocess.CalledProcessError as exc: error = licerror.LicenseError(0, "lmstat failed", licerror.ERROR, cmd=exc.cmd, output=exc.output) stat_out = exc.output try: return parse_lmstat_output(stat_out.splitlines()) except licerror.LicenseException as exc: error = exc.error error.output = stat_out error._level = licerror.WARNING if is_unreachable_hostname(error, license_file): error.unreachable_hostname = True error._message = "License checkouts are successful but license details are unavailable as the hostname of the license server is not resolvable from this machine. " raise licerror.LicenseException(error)
[docs]def is_unreachable_hostname(error, license_file): """ Return True if error matches the case where the hostname on the license server is not an accesible hostname. This manifests as lmstat returning -96, but we are still able to use mmlic3. :param error: from parse_lmstat_output :type error: LicenseError :param license_file: path to the license file to use with licstat :type license_file: str """ if error.code == -96: try: execute_lictest(["-f", license_file, "MMLIBS"]) except subprocess.CalledProcessError: return False else: return True return False
[docs]def is_future_license_error(info): """ Return True if this represents a license with a future start date. :param info: Representation of a Server instance :type info: ServerInfo :rtype: bool """ if not info: return False if info.schrod_exit_code != licerror.SCHROD_NO_FEATURES_CODE: return False if not info.has_future_start_date: return False return True
[docs]def parse_lmstat_output(lines): """ Parse the output from "lmutil lmstat", provided as a list of output lines. :type lines: list(str) :param lines: Output of 'lmutil lmstat'. :returntype: dict :return: Dictionary of license server, vendor daemon metadata Metadata info:: "address" port@host address of the license server. "server_version" lmgrd version number, e.g., "v11.10.2" or VERSION_UNKNOWN if indeterminate. This is only returned in the case of running lmstat on a remote server. "server_up" True, if lmgrd is UP; else, False. "version" SCHROD daemon version number or VERSION_UNKNOWN if indeterminate. "schrod_up" True, if SCHROD is up; False if down. "tokens" tokens dict is a mapping of the feature name to a stat_info named tuple for that feature. A stat_info tuple has integer fields 'total' and 'used', recording the total tokens provided and the number in use. If the output lines doesn't have users of feature(s), this will be empty. """ server_address_re = re.compile(r"License server status:\s+(.*)") lmstat_re = re.compile( r"Users of ([A-Z0-9_]+):.*\s(\d+) licenses? issued.*\s(\d+) licenses? in use", re.IGNORECASE) server_version_re = re.compile(r"license server UP.* (v\d[\d.]+).*") schrod_status_re = re.compile(r"SCHROD: (.*)") schrod_version_re = re.compile(r"UP (v\d[\d.]+).*") status_error_re = re.compile(r"Error getting status:\s*(\S.*)$", re.IGNORECASE) lmgrd_error_re = re.compile(r"lmgrd is not running:\s*(\S.*)$", re.IGNORECASE) attr = { "server_up": False, "server_version": VERSION_UNKNOWN, "version": VERSION_UNKNOWN, "tokens": {} } for line in lines: match = server_address_re.search(line) if match: # Once we get the attributes for SCHROD daemon # we finish parsing. Otherwise, we just reset the # values so we don't endup returning with non-SCHROD # daemon attributes. if "schrod_up" not in attr: attr["server_up"] = False attr["server_version"] = VERSION_UNKNOWN attr["version"] = VERSION_UNKNOWN attr["tokens"] = {} if "address" in attr: del attr["address"] else: break attr["address"] = match.group(1) continue match = lmstat_re.search(line) if match: name, issued, inuse = match.groups() attr["tokens"][name] = stat_info(int(issued), int(inuse)) continue for error_re in (status_error_re, lmgrd_error_re): match = error_re.search(line) if match: message = match.group(1) m = re.search(r"([^()]*)\((-?\d+,-?\d[:\d]*)[^)]*\)", message) if m: error = licerror.LicenseError(m.group(2), m.group(1).strip()) else: error = licerror.LicenseError(0, message) raise licerror.LicenseException(error) match = schrod_status_re.search(line) if match: # lmgrd manages SCHROD daemon. # This can be either up or down or not started yet. if match.group(1) and not match.group(1).startswith( SCHROD_DAEMON_NOT_STARTED): submatch = schrod_version_re.search(match.group(1)) if submatch: attr["schrod_up"] = True if submatch.group(1): attr["version"] = submatch.group(1) else: attr["schrod_up"] = False continue match = server_version_re.search(line) if match: attr["server_up"] = True attr["server_version"] = match.group(1) continue return attr
[docs]def run_lmdiag(locations, feature=""): """ Get diagnostic information regarding one or more license resources via "lmutil lmdiag". The argument 'locations' is a string representing a FLEXLM search path, such as $SCHROD_LICENSE_FILE. Returns a dict with an entry for each license resource that was examined. For each resource, returns a dict mapping the feature name to a list of LicenseFeature objects, each representing the diag info for a single license line for that feature. (There may be more than one.) Raises licerror.LicenseException, otherwise. """ try: cmd = ["lmdiag", "-n", "-c", locations] if feature: cmd.append(feature) diag_out = execute_lmutil(cmd) diag = parse_lmdiag_output(diag_out.splitlines()) except subprocess.CalledProcessError as exc: error = licerror.LicenseError(0, "lmdiag failed", licerror.ERROR, cmd=exc.cmd, output=exc.output) raise licerror.LicenseException(error) except licerror.LicenseException as exc: error = exc.error.output = diag_out raise licerror.LicenseException(error) # TODO: throw out expired LicenseFeatures if unexpired LicenseFeatures # are found for the same feature name. # TODO: Consider we we should take hostid and version into account # when deciding when to ignore errors. if not diag: error = licerror.LicenseError(0, "lmdiag failed", licerror.ERROR, cmd=cmd, output=diag_out) raise licerror.LicenseException(error) return diag
[docs]def parse_lmdiag_output(lines): """ Parse the output from "lmutil lmdiag", provided as a list of output lines. Returns a dict with an entry for each license resource that was examined. For each resource, returns a dict mapping the feature name to a list of LicenseFeature objects, each representing the diag info for a single license line for that feature. (There may be more than one.) """ features = defaultdict(list) licenses = {} license = "" linegroup = [] for line in lines: if line.startswith("--------"): if linegroup: linegroup = join_cont_lines(linegroup) feat = parse_lmdiag_entry(linegroup) # feat is None if it wasn't a valid schrodinger feature # e.g. a non SCHROD daemon if feat: features[feat.name].append(feat) linegroup = [] elif line.startswith("License file:"): if license and features: licenses[license] = features license = "" features = defaultdict(list) label, license = line.split(":", 1) license = license.strip() elif license: linegroup.append(line) if license: licenses[license] = features return licenses
[docs]def join_cont_lines(lines): """ Returns the input list of lines, after joining any logical lines that were split across two (or more) physical lines, and stripping leading and trailing whitespace. A continued line is indicated by a trailing comma. """ result = [] line_out = "" for line in lines: line = line.strip() if line_out: line_out += " " + line else: line_out = line if line.endswith(","): continue result.append(line_out) line_out = "" return result
[docs]def parse_lmdiag_entry(lines): """ Returns a LicenseFeature object, representing the information reported by lmdiag for that feature. """ FEATURE_RE = re.compile(r"\"(\w+)\" v([\d.]+), vendor: (\w+)") VENDOR_RE = re.compile(r"\s*License (?:server|path):\s+(\S.*\S)") TIMESTAMP_RE = re.compile( r"(\S.*\slicense),? locked to (\S.*\S)\s+starts: ([\w-]+),\s+expires: ([\w-]+)" ) SUCCESS_RE = re.compile(r"license (can|cannot) be checked out") ERROR_RE = re.compile(r"FlexNet Licensing error:\s*(\S.*\S)", re.IGNORECASE) feat = None errcode = 0 errmsg = "" for line in join_cont_lines(lines): if feat and feat.ok is not None and errmsg == "": # first line after "is/isnt ok" line is the error message errmsg = line.strip() continue match1 = FEATURE_RE.search(line) if match1: name, version, vendor = match1.groups() if vendor != "SCHROD": return None feat = LicenseFeature(name, version=version) continue match2 = VENDOR_RE.search(line) if match2: # We found features without finding matching vendor information # This is probably not a SCHROD license if not feat: return None feat.location = match2.group(1) continue match3 = TIMESTAMP_RE.search(line) if match3: lictype, hostid, starts, expires = match3.groups() feat.lictype = lictype feat.set_hostid(hostid) feat.set_starts(starts) feat.set_expires(expires) feat.set_tokens(UNCOUNTED) continue match4 = SUCCESS_RE.search(line) if match4: feat.ok = (match4.group(1) == "can") continue match6 = ERROR_RE.search(line) if match6: errcode = match6.group(1) continue if not feat: return feat if feat.ok is None: feat.ok = True if errcode or errmsg: feat.error = licerror.LicenseError(errcode, errmsg) elif not feat.ok: feat.error = licerror.LicenseError(0, "Unspecified error", licerror.ERROR) return feat
[docs]def load_job_reqs(filename=""): """ Read a json document describing the license requirements for each job that a user could launch. The license requirements for a given job are represented as a dict, mapping the token name to the number of tokens required for that job. A job may accept more than one combination of tokens. Return a dict mapping the job type to a list of dicts, each representing an acceptable set of licenses for that job. If the file can't be loaded, an exception will be thrown. """ if not filename: filename = JOB_REQS_FILE with open(filename) as fp: return json.load(fp)
[docs]def load_job_products(filename=""): """ Read a json document listing the product names under which job types should be grouped. Returns a list of product names. If the file can't be loaded, an exception will be thrown. """ if not filename: filename = JOB_PRODUCTS_FILE with open(filename) as fp: return json.load(fp)
def _require_job_reqs(): """ Return the table of license requirements for each job. The requirements are returned as a dict. The keys are the full internal job names. The license requirements for each job are recorded as a list of dicts, where each list item describes a set of requirements. A set of requirements is a dict mapping a license token name to the number of tokens required for that job. """ global _job_reqs, _job_products if not _job_reqs: _job_reqs = load_job_reqs() if not _job_products: _job_products = load_job_products()
[docs]def find_license_resources(unique=True): """ Returns a list of lists of all license resources (license files or servers) that would be available to a job trying to check out a license. License resources may be specified using the environment variables :: $SCHROD_LICENSE_FILE $LM_LICENSE_FILE ... which can each specify a list of license files and/or license servers. License resources will also be found automatically if installed in one of the standard locations that mmlic3 is hard-wired to search. These are :: $SCHRODINGER/licenses/*.lic $SCHRODINGER/license{,.txt} ... and, depending on the platform, :: MacOSX: /Library/Application Support/Schrodinger/licenses/*.lic /Library/Application Support/Schrodinger/license{,.txt} Windows: C:/ProgramData/Schrodinger/licenses/*.lic C:/ProgramData/Schrodinger/license{,.txt} Linux /opt/schrodinger/licenses/*.lic /opt/schrodinger/license{,.txt} Returns a list of LicenseResource objects. :param unique: if False, duplicates of server files. (see _unique_resources) :type unique: bool """ result = [] if "SCHROD_LICENSE_FILE" in os.environ: result.extend( resources_from_path(os.getenv("SCHROD_LICENSE_FILE"), '$SCHROD_LICENSE_FILE')) if "LM_LICENSE_FILE" in os.environ: result.extend( resources_from_path(os.getenv("LM_LICENSE_FILE"), '$LM_LICENSE_FILE')) if "SCHRODINGER_LICENSE_FALLBACK" in os.environ: result.extend( resources_from_path(os.getenv("SCHRODINGER_LICENSE_FALLBACK"), '$SCHRODINGER_LICENSE_FALLBACK')) elif "SCHRODINGER_LICENSE" in os.environ: result.extend( resources_from_path(os.getenv("SCHRODINGER_LICENSE"), '$SCHRODINGER_LICENSE')) else: search_path = [ os.path.join(SCHRODINGER, LICENSE_DIRNAME), os.path.join(SCHRODINGER, LICENSE_FILENAME), os.path.join(SCHRODINGER, LICENSE_FILENAME + ".txt") ] result.extend(resources_from_list(search_path, "$SCHRODINGER")) search_path = [ os.path.join(SHARED_LICENSE_DIR, LICENSE_DIRNAME), os.path.join(SHARED_LICENSE_DIR, LICENSE_FILENAME), os.path.join(SHARED_LICENSE_DIR, LICENSE_FILENAME + ".txt") ] result.extend(resources_from_list(search_path, SHARED_LICENSE_DIR)) if not unique: return result return _unique_resources(result)
def _unique_resources(lics): """ Filter the given list of LicenseResources to remove redundant resources. Rules: 1. If both a port@host reference and license file are found for the same server, we keep the license file. 2. If we have both a port@host and an @host reference to a server, keep the port@host reference. 3. If a stub license and a full license file are found, keep the full license file. 4. If there's more than one full license file for a given server, all are kept. """ priority = {} # track highest priority found for each ident for lic in lics: ident = lic.ident() p = lic.priority() if ident not in priority or p < priority[ident]: priority[ident] = p # For each ident, keep only license resources with the highest priority. # For priority 0 (full license file) all resources are kept, # For priority > 0, only the first resource with that priority is kept. filtered_list = [] idents = set() pathnames = set() for lic in lics: ident = lic.ident() p = lic.priority() if p > priority[ident]: continue if p == 0 and lic.pathname() in pathnames: continue if p > 0 and ident in idents: continue if p == 0: pathnames.add(lic.pathname()) filtered_list.append(lic) idents.add(ident) return filtered_list
[docs]def resources_from_path(search_path, source=""): """ Parse the given search path (PATH-style) into components and return a list with the LicenseResource object for each item. """ return resources_from_list(search_path.split(os.pathsep), source)
[docs]def resources_from_list(license_list, source): """ Return a list with the LicenseResource object for each item (server hostport, license file or license directory) in the given list. """ result = [] for item in license_list: if "@" in item: result.append(LicenseResource(item, source)) elif os.path.isfile(item): result.append(LicenseResource(Path(item), source)) elif os.path.isdir(item): for name in sorted(os.listdir(item)): if has_license_file_suffix(name): result.append( LicenseResource(Path(item).joinpath(name), source)) return result
[docs]def resource_summary(resources): """ Provide a condensed list of license resources. Each license server is listed, but the existence of node-locked licenses. The health of each listed resource is indicated with a status code, which will be one of RESOURCE_OK, RESOURCE_PROBLEMS, RESOURCE_UNAVAILABLE. """ summary = [] nodelocked_licenses = False nodelocked_health = RESOURCE_UNAVAILABLE for lic in resources: health = lic.health() if lic.is_server(): summary.append((lic.location(), health)) else: nodelocked_licenses = True if healthier(health, nodelocked_health): nodelocked_health = health if nodelocked_licenses: summary.append(("Nodelocked License", nodelocked_health)) return summary
[docs]def hostid_is_local(lic_hostid): """ Returns True if the given hostid matches the local hostid. """ lic_hostids = set(lic_hostid.split()) local_hostids = set(get_hostid().split()) local_hostids.add("ANY") local_hostids.add("DEMO") # Use set intersection to see if any hostids match return bool(lic_hostids & local_hostids)
[docs]def hostname_is_local(hostname): """ Determine whether the given hostname refers to the current host. """ ourhost = socket.gethostname() if hostname == 'this_host' or hostname == 'localhost': return True if shorthost(hostname) == shorthost(ourhost): return True if hostname == '127.0.0.1': return True if sys.platform.startswith('darwin'): # This case may come up in Mac if the computer name has been added to # /etc/hosts if hostname == '1.0.0.127.in-addr.arpa': return True if is_ipv4_addr(hostname): try: ouripaddr = socket.gethostbyname(ourhost) if hostname == ouripaddr: return True except socket.gaierror: pass return False
[docs]def is_ipv4_addr(addr): """ Determine whether the given string is an IPv4 address. """ m = re.match(r"(\d+)\.(\d+)\.(\d+)\.(\d+)$", addr) if m: for a in m.groups(): if int(a) > 255: return False return True return False
[docs]def server_log(licfile): """ Return the standard pathname for the lmgrd log file associated with the given license file. If the license is just a port@host server address, the empty string is returned, because we have no idea where the logs would be. """ licdir, licname = os.path.split(licfile) dirparent, dirname = os.path.split(licdir) if dirname == LICENSE_DIRNAME: logfile = os.path.join(dirparent, LMGRD_LOG) else: logfile = os.path.join(licdir, LMGRD_LOG) return logfile
[docs]def writable_server_log(licfile, prefer_logfile=""): """ Return the pathname for the log file to use when launching a server for the given license file. The returned pathname will be writable by the current user. This differs from server_log() in that it will check for alternative locations and filenames if the standard lmgrd.log file can't be written. A suggested logfile pathname can be specified. If this file is writable, it will be returned. """ if os.pathsep in licfile: licfile = licfile.split(os.pathsep)[0] candidates = [] if prefer_logfile: candidates = [prefer_logfile] candidates.extend([ server_log(licfile), os.path.join(fileutils.get_directory_path(fileutils.HOME), LMGRD_LOG), os.path.join(os.getcwd(), LMGRD_LOG) ]) for logfile in candidates: if can_write_file(logfile): log.debug("** CAN write %s" % logfile) return logfile log.debug("!! can't write %s" % logfile) altlogfile = logfile.replace(LMGRD_LOG, LMGRD_ALTLOG) if can_write_file(altlogfile): return altlogfile return ""
[docs]def can_write_file(pathname): """ Returns true if the filesystem permissions appear to allow the specified file to be written. """ if os.path.exists(pathname): result = os.access(pathname, os.W_OK) log.debug(f"&& {result} : FILE {pathname}") else: dirname, filename = os.path.split(pathname) result = os.access(dirname, os.W_OK) log.debug(f"&& {result} : DIR {dirname}") return result
[docs]def create_stub_license(host, port=""): """ Create a stub license file, which points to a remote license server. """ return "SERVER %s ANY %s\nUSE_SERVER\n" % (host, port)
[docs]class ServerInfo(object): """ A ServerInfo object represents the information reported in the lmgrd log when a license server is started. """
[docs] def __init__(self): self.version = "" self.starttime = None self.stoptime = None self.licfile = "" self.host = "" self.port = 0 self.pid = 0 self.schrod_version = "" self.schrod_port = 0 self.schrod_pid = 0 self.exit = None self.has_future_start_date = False self.schrod_exit_code = 0 self.errors = []
[docs] def add_error(self, err): if err is None: return if err.tag in ("SCHROD_EXIT", "SCHROD_SHUTDOWN", "LMGRD_EXIT"): if self.exit: return self.exit = err.tag self.errors.append(err)
[docs]def parse_lmgrd_log(filename, offset=0): try: with codecs.open(filename, "r", encoding="ascii", errors="ignore") as fp: fp.seek(offset) return parse_lmgrd_lines(fp) except IOError as exc: log.debug(exc) return None
[docs]def parse_lmgrd_lines(lines): """ Parse an lmgrd log file for messages related to the health of the lmgrd and SCHROD daemons. The log file is supplied as a list of lines. A ServerInfo object is returned """ info = ServerInfo() licfile = "" error = None for ts, prog, line in lmgrd_lines(lines): if "SLOG" in line: continue if prog == "lmgrd": event, level, fields = parse_lmgrd_event(LMGRD_EVENTS, line) if event == "LMGRD_PRE_START": info = ServerInfo() error = None elif event == "LMGRD_START": info = ServerInfo() info.version = fields["version"] info.host = fields["host"] elif event == "LMGRD_FILE": licfile = fields["file"].strip("'\"") elif event == "LMGRD_FILE2": licfile = fields["file"].strip("'\"") elif event == "LMGRD_PORT": info.port = fields["port"] elif event == "LMGRD_SCHROD_START": info.schrod_pid = fields["pid"] info.schrod_port = fields["port"] elif event == "LMGRD_BAD_HOST": info.add_error(error) error = licerror.LicenseError(0, "Invalid server host", licerror.ERROR, tag=event, output=line) elif event == "LMGRD_BAD_PORT": info.add_error(error) error = licerror.LicenseError( 0, "Unable to open the requested port", licerror.ERROR, tag=event, output=line) elif event == "LMGRD_NO_PORTS": info.add_error(error) error = licerror.LicenseError( 0, "Unable to open any default port (see licadmin documentation for list of those ports)", licerror.ERROR, tag=event, output=line) elif event == "LMGRD_EXIT": info.add_error(error) error = licerror.LicenseError(0, "Server exited due to " + fields["reason"], licerror.ERROR, tag=event, output=line) elif event == "LMGRD_SCHROD_EXIT": info.add_error(error) error = licerror.LicenseError( 0, "SCHROD exit due to %s (status %s)" % (fields["msg"], fields["status"]), licerror.ERROR, tag=event, output=line) info.schrod_exit_code = int(fields["status"]) elif error is not None: error.add_output(line) if event and level != LMGRD_ERROR: info.add_error(error) error = None if event: log.debug(" ".join((str(ts), event))) elif prog == "SCHROD": info.add_error(error) error = None event, level, fields = parse_lmgrd_event(SCHROD_EVENTS, line) if event == "SCHROD_START": info.schrod_version = fields["version"] elif event == "SCHROD_EXIT": info.add_error( licerror.LicenseError(0, "SCHROD exited due to " + fields["reason"], licerror.ERROR, tag=event, output=line)) elif event == "SCHROD_SHUTDOWN": info.add_error( licerror.LicenseError(0, "Server shut down by " + fields["who"], licerror.ERROR, tag=event, output=line)) elif event == "SCHROD_FEATURE_FUTURE_START_DATE": info.has_future_start_date = True info.add_error(error) info.licfile = licfile return info
[docs]def version_uptodate(version): """ Returns True unless the given lmgrd version number is older than the version included in our release, given by the REQUIRED_MAJOR_VERSION and REQUIRED_MINOR_VERSION constants. """ m = re.match(r"v?(\d+)\.(\d+)", version) if m: maj, min = [int(x) for x in m.groups()] if maj < REQUIRED_MAJOR_VERSION: return False elif maj == REQUIRED_MAJOR_VERSION and min < REQUIRED_MINOR_VERSION: return False return True
# Regular expressions for events recognized in the lmgrd log file (LMGRD_INFO, LMGRD_ERROR) = (0, 1) LMGRD_EVENTS = OrderedDict() LMGRD_EVENTS["LMGRD_PRE_START"] = (LMGRD_INFO, re.compile(r" Please Note:")) LMGRD_EVENTS["LMGRD_START"] = ( LMGRD_INFO, re.compile( r"FLEXnet Licensing \((?P<version>v\d[\d.]+).* started on (?P<host>[\w.-]+) ", re.IGNORECASE)) LMGRD_EVENTS["LMGRD_FILE"] = (LMGRD_INFO, re.compile(r"License file[^:]*: (?P<file>.*)$", re.IGNORECASE)) LMGRD_EVENTS["LMGRD_FILE2"] = (LMGRD_INFO, re.compile(r"Using license file (?P<file>.*)$", re.IGNORECASE)) LMGRD_EVENTS["LMGRD_PORT"] = (LMGRD_INFO, re.compile(r"lmgrd tcp.port (?P<port>\d+)")) LMGRD_EVENTS["LMGRD_SCHROD_START"] = ( LMGRD_INFO, re.compile(r"Started SCHROD .* tcp.port (?P<port>\d+) pid (?P<pid>\d+)")) LMGRD_EVENTS["LMGRD_SCHROD_EXIT"] = ( LMGRD_ERROR, re.compile(r"SCHROD exited with status (?P<status>\d+) \((?P<msg>[^)]+)\)")) LMGRD_EVENTS["LMGRD_BAD_HOST"] = ( LMGRD_ERROR, re.compile(r"\"(?P<host>[^\"]*)\": Not a valid server hostname, exiting.$")) LMGRD_EVENTS["LMGRD_BAD_PORT"] = ( LMGRD_ERROR, re.compile(r"Failed to open the TCP port number in the license.")) LMGRD_EVENTS["LMGRD_NO_PORTS"] = ( LMGRD_ERROR, re.compile(r"Failed to open any default TCP port.")) LMGRD_EVENTS["LMGRD_EXIT"] = (LMGRD_ERROR, re.compile(r"EXITING DUE TO (?P<reason>.*)$")) # Regular expressions for SCHROD events recognized in the lmgrd log file (SCHROD_INFO, SCHROD_ERROR) = (0, 1) SCHROD_EVENTS = OrderedDict() SCHROD_EVENTS["SCHROD_START"] = ( SCHROD_INFO, re.compile(r"FlexNet Licensing.*\b(?P<version>v\d[\d.]+).*", re.IGNORECASE)) SCHROD_EVENTS["SCHROD_HOST"] = (SCHROD_INFO, re.compile(r" started on (?P<host>\w+) ")) SCHROD_EVENTS["SCHROD_BAD_KEY"] = (SCHROD_ERROR, re.compile(r"Invalid license key ")) SCHROD_EVENTS["SCHROD_EXITING"] = (SCHROD_ERROR, re.compile(r", exiting.$", re.IGNORECASE)) SCHROD_EVENTS["SCHROD_SHUTDOWN"] = ( SCHROD_ERROR, re.compile(r"Shutdown requested from (?P<who>.*)")) SCHROD_EVENTS["SCHROD_EXIT"] = (SCHROD_ERROR, re.compile(r"EXITING DUE TO (?P<reason>.*)$")) SCHROD_EVENTS["SCHROD_FEATURE_FUTURE_START_DATE"] = ( LMGRD_INFO, re.compile(r"is not enabled yet, starts on"))
[docs]def parse_lmgrd_event(eventdict, line): for event, (level, regexp) in eventdict.items(): m = regexp.search(line) if m: return (event, level, m.groupdict()) return ("", None, None)
[docs]def lmgrd_lines(lines): """ A generator to perform initial parsing of lmgrd logfile lines. A sequence of tuples (timestamp, prog, text) is returned. Lines for any prog other than "lmgrd" or "SCHROD" are discarded. """ last_ts = "" seq = 0 re_lmgrd = re.compile(r"^\s*(\d+:\d\d:\d\d) \((\w+)\) (.*)$") for line in lines: line = line.rstrip() m = re_lmgrd.match(line) if m: (ts, prog, rest) = m.groups() if ts == last_ts: seq += 1 else: seq = 0 hh, mm, ss = [int(item) for item in ts.split(":", 2)] timestamp = time(hh, mm, ss, seq) if prog == "lmgrd" or prog == "SCHROD": log.debug(">> " + line) yield (timestamp, prog, rest) else: log.debug("-- " + line) else: log.debug("(( " + line)
####################################################################### # Public API #######################################################################
[docs]def resources(): """ Return a list of all license resources found. """ global _resources if _resources is None: _resources = find_license_resources() return _resources
[docs]def get_local_server(): global _local_server if _local_server is None: _local_server = LocalServerProcess() return _local_server
[docs]def local_server_resource(): """ Returns the first license resource found for the local license server, if any. There may be multiple license servers, but only one that can run on the current machine. Only a local license server can be started or stopped by the user. Returns None if no local license server is found. NOTE: To start or stop a local license server, you'll need all of the license files for that server - there may be more than one. Use the local_server_resources() function to get the complete list. """ local_server_lics = local_server_resources() if local_server_lics: return local_server_lics[0] return None
[docs]def local_server_resources(): """ Returns the license resources that represent the local license server. There may be multiple license servers, but only one that can run on the current machine. Only a local license server can be started or stopped by the user. Returns the empty list if no local license server files are found. """ local_server_lics = [] for lic in resources(): if lic.is_local_server(): local_server_lics.append(lic) return local_server_lics
[docs]def get_working_resources(license_resources=None): """ Returns list of LicenseResources without any all-expired files. This is useful for user-facing presentation. :param license_resources: list of license resource, if none specified search all available :type license_resources: list(LicenseResource objects) """ if not license_resources: refresh() license_resources = resources() output_resources = [] local_server = get_local_server() if not local_server.isRunning(): local_server.clearResources() for resource in license_resources: # Health of local server licenses can only be checked via the server # process if resource.is_local_server(): if local_server.checkResource(resource) != RESOURCE_UNAVAILABLE: local_server.addResource(resource) continue # Call resource.health() so resource.errors() is populated. resource.health() errs = resource.errors() if errs: for err in errs: if not err.message == EXPIRED_MESSAGE: break else: continue output_resources.append(resource) # Do not resources of local server if the list is empty if local_server.numResources() > 0: output_resources.append(local_server) return output_resources
[docs]def refresh(): """ Force license-resource info to be re-loaded. """ global _resources _resources = find_license_resources()
[docs]def local_server_files(dirname): """ Given the name of a license directory, return a list of the license files found therein that can be used to start a local license server. Because the directory is a license directory, license files need to have a .lic suffix. """ files = [] for filename in sorted(os.listdir(dirname)): if has_license_file_suffix(filename): lic = LicenseResource(os.path.join(dirname, filename), dirname) if lic.is_local_server(): files.append(lic.pathname()) return files
[docs]def job_capacity_by_product(res_list=None): """ Return a dict of lists of tuples, reporting the number of jobs of each type that can be run, grouped by product. The structure of the return value is :: { <product1>: [ (<jobtype1>, <number of jobs>), (<jobtype2>, <number of jobs>), ... ], <product2>: [ (<jobtype1>, <number of jobs>), (<jobtype2>, <number of jobs>), ... ], } By default, checks all available resources, but the res_list parameter can be used to restrict the report to a specified collection of resources. """ if res_list is None: res_list = resources() job_capacity_by_jobtype = job_capacity(res_list) capacity_by_product = defaultdict(list) job_product_list = list(map(get_key_name, job_product_names())) for jobtype in job_capacity_by_jobtype: product = get_product(jobtype, job_product_list) jobtype_name = get_jobtype_name(jobtype, product) capacity_by_product[product].append( (jobtype_name, job_capacity_by_jobtype[jobtype])) return capacity_by_product
[docs]def job_product_names(): ''' Return Schrodinger product names. :rtype: list(str) ''' #cache loading json file to avoid re-reading each time _require_job_reqs() return _job_products
[docs]def get_key_name(product_name): """ Return canonicalized name for string matching. :param: str product_name: Name of a Schrodinger product. """ return product_name.upper().replace(" ", "_")
[docs]def get_product(jobtype, products): """ Return a product name for a job type that is in products list. :param str jobtype: Type of a Schrodinger product job. :param products: List of Schrodinger product names. :type list(str) """ for product in products: pattern = r'^{}(?=_|$)'.format(re.escape(product)) if re.match(pattern, jobtype): return product raise RuntimeError("No product for job type '%s' in _job_products" % jobtype)
[docs]def get_jobtype_name(jobtype, product): """ Return jobname for jobtypes in Schrodinger product list. :param str jobtype: Type of a Schrodinger product job. :param str product: Name of a Schrodinger product. """ jobtype_name = jobtype.replace(product, "") if jobtype_name: jobtype_name = jobtype_name.replace("_", " ") jobtype_name = " ".join([w.capitalize() for w in jobtype_name.split()]) else: jobtype_name = product return jobtype_name
[docs]def license_directory(): """ Return the pathname of the directory into which license files should be installed. The first writable directory found among the following will be returned: 1. the shared license directory, 2. $SCHRODINGER/licenses The returned license directory will be created, as a side effect, if it doesn't already exist. """ std_lic_dirs = (os.path.join(SHARED_LICENSE_DIR, LICENSE_DIRNAME), os.path.join(SCHRODINGER, LICENSE_DIRNAME)) for license_dir in std_lic_dirs: if not os.path.isdir(license_dir): try: os.makedirs(license_dir, 0o755) except OSError: continue if os.path.isdir(license_dir) and os.access(license_dir, os.W_OK): return license_dir raise PermissionError( "Could not find a writable directory from the following standard directories to install license file - {}" .format(std_lic_dirs))
####################################################################### # Commandline utility #######################################################################
[docs]class UsageError(Exception): """ This exception is thrown when unrecognized or incompatible commandline options are used. """
[docs]class NetsuiteError(Exception): """ This exception is raised when there's a problem obtaining a license from the NetSuite server. """
# NetsuiteError messages NSERROR_GET_FAILED = 'Failed to retrieve license' NSERROR_BAD_KEY_TYPE = 'Unexpected key type: %s' NSERROR_KEY_EXHAUSTED = 'The key you entered has already been used %d times and cannot be used again.' NSERROR_KEY_USED = 'The server key you entered has already been used to generate a license file.' # Global parser object to allow help to be printed from toplevel # try-except block. parser = None
[docs]def parse_args(args): """ Parse the commandline arguments for the licadmin utility. If incompatible options are specified a UsageError is thrown. Returns a namespace object with the values of the commandline options. The subcommand ("START", "INFO", etc.) is returned as the "action" attribute of the namespace object. """ global parser parser = argparse.ArgumentParser( usage="$SCHRODINGER/licadmin <action> [<options>]", add_help=False, description=""" Commands: HOSTID ...... report the FLEXlm hostid START ........ start a license server (also, 'SERVERUP') STOP ......... stop the license server (also, 'SERVERDOWN') REREAD ...... have the license server re-read the license file CKSUM ........ report the checksum of the license file STAT ........ report the status of the license server DIAG ......... diagnose license-checkout problems INFO ......... collect useful files and data for Schrodinger tech support INSTALL ...... install a license file INSTALL_LAUNCHD_FILE ...... install flexlm plist to autostart lmgrd and SCHROD on boot Options: -c <license> ...... a license file pathname. -k <license-key> .. a license key. (INSTALL only) -l <log-file> .... the server log file pathname. (START only) -n ............... non-interactive mode (for DIAG only) -q ............... quiet (non-interactive) mode (STOP only) -h ............... print this usage message """, epilog=""" You can use "port@host" syntax with "-c" to specify a license server for those commands that don't require an actual license file. If no license file is specified explicitly, this script will use the license file that an application would have used if run in this terminal session. That is, if $SCHROD_LICENSE_FILE or $LM_LICENSE_FILE are set, the license it points to is used, otherwise the license file(s) in $SCHRODINGER/licenses/. Finally, we look for licenses in the shared license directory: /opt/schrodinger/licenses/ (Linux), /Library/Application Support/Schrodinger/ (MacOSX), or C:/ProgramData/Schrodinger/licenses\ (Windows). By default, the server log file is written to $SCHRODINGER/lmgrd.log. Commands are case-insensitive. For further assistance, contact help@schrodinger.com """, formatter_class=argparse.RawTextHelpFormatter) if not args or args[0].startswith("-"): if args and args[0].startswith("-h"): parser.print_help() sys.exit(0) else: raise UsageError("No action was specified.") parser.add_argument("-c", dest="license", action="store", help=argparse.SUPPRESS) parser.add_argument("-k", dest="license_key", action="store", help=argparse.SUPPRESS) parser.add_argument("-l", dest="logfile", action="store", help=argparse.SUPPRESS) parser.add_argument("-z", dest="background", action="store_true", help=argparse.SUPPRESS) parser.add_argument("-n", dest="noninteractive", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("-q", dest="quiet", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("-debug", dest="debug", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("args", nargs="*", help=argparse.SUPPRESS) (opts, remaining) = parser.parse_known_args(args[1:]) opts.args = remaining opts.action = args[0].upper() if opts.action == "SERVERUP": opts.action = "START" if opts.action == "SERVERDOWN": opts.action = "STOP" if opts.noninteractive and opts.action != "DIAG": raise UsageError("-n option only supported for the DIAG action.") if opts.quiet and opts.action != "STOP": raise UsageError("-q option only supported for the STOP action.") if opts.action == "INSTALL": if not opts.license and not opts.license_key: raise UsageError( "A license file or key is required for the INSTALL action.") elif opts.license and opts.license_key: raise UsageError( "Either a license file OR a license key should be specified.") if opts.action in LICENSE_ACTIONS: _add_licenses(opts) return opts
# Actions that can be handled by handle_generic_action() GENERIC_ACTIONS = ("CKSUM", "NEWLOG", "REREAD", "REMOVE", "SWITCHR") # Actions that require a license file (or server) to act on. LICENSE_ACTIONS = ("CKSUM", "DIAG", "INFO", "NEWLOG", "REREAD", "REMOVE", "START", "STAT", "STOP", "SWITCHR", "VERIFY")
[docs]def handle_start_action(opts): """ Start a local license server. """ res = LicenseResource(opts.license) localserver = get_local_server() localserver.addResource(res) if not opts.logfile: opts.logfile = writable_server_log(opts.license) log.info("Log file: " + opts.logfile) if localserver.isRunning(): localserver.stop() localserver.start(opts.logfile)
[docs]def handle_stop_action(opts): """ Stop a local license server. """ res = LicenseResource(opts.license) localserver = get_local_server() localserver.addResource(res) localserver.stop()
[docs]def handle_stat_action(opts): """ Execute "lmutil lmstat -a" for the specified license. """ args = ["-a"] if opts.license: args.extend(["-c", opts.license]) if opts.args: args.extend(opts.args) try: return execute_lmutil(["lmstat"] + args) except subprocess.CalledProcessError as exc: return exc.output
[docs]def handle_diag_action(opts): """ Execute "lmutil lmdiag -n" for the specified license. """ args = ["-n"] if opts.license: args.extend(["-c", opts.license]) if opts.args: args.extend(opts.args) try: return execute_lmutil(["lmdiag"] + args) except subprocess.CalledProcessError as exc: return exc.output
[docs]def handle_ver_action(opts): """ Execute "lmutil lmver" for the specified executable. Lines containing "lmutil" are stripped from the result. """ args = opts.args if not args: args = [os.path.join(os.environ["MMSHARE_EXEC"], "lmutil")] try: output = execute_lmutil(["lmver"] + args) lines = [] for line in output.splitlines(): if "lmutil" not in line: lines.append(line) return "\n".join(lines) except subprocess.CalledProcessError as exc: return exc.output
[docs]def handle_list_action(opts): """ List all known license resources. """ lines = [] for lic in resources(): lines.append(lic.location()) return "\n".join(lines)
[docs]def validate_connection(host, port, timeout=None): """ Attempt to connect to a specified host and port. :param host: the host to connect to :type host: str :param port: the port to connect to :type port: int :return: a 2-tuple describing whether the connection was successful, and if not, an error message :rtype: tuple(bool, str) """ try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if timeout is not None: s.settimeout(timeout) s.connect((host, port)) except socket.error as e: return False, str(e) else: return True, '' finally: s.close()
[docs]def validate_license_host_port(host, port, timeout=None): """ Validate the given license server host and port. If the port is empty, the default lmgrd port ranges are checked. :param host: the license server host to connect to :type host: str :param port: the license server port to connect to :type port: str :param timeout: How long in seconds to wait for each connection :type timeout: int :return: list of list with each list representing the host, port, status of connection and an error message. """ host_port_status = [] if port == "": iports = DEFAULT_LMGRD_PORT_RANGE else: iports = [int(port)] for iport in iports: status, msg = validate_connection(host, int(iport), timeout) host_port_status.append([host, iport, status, msg]) return host_port_status
[docs]def handle_check_action(opts): """ Report the status of all known license resources. """ lines = [] for lic in find_license_resources(unique=False): lines.append(lic.location()) if lic.location() != lic.pathname(): lines.append(lic.pathname()) lines.append("-- " + str(lic.health())) for err in lic.errors(): lines.append("** " + str(err)) if lic.health() != RESOURCE_OK and lic.is_server(): host, port = lic.server_host_and_port() host_port_status = validate_license_host_port(host, port) lines.append("") lines.append( "** Check the ability to make socket network connections") for host, port, status, msg in host_port_status: if status: lines.append( f'Successfully made a socket connection to port {port} on host "{host}".' ) else: lines.append( f'Failed to make socket connection to port {port} on host "{host}".' ) lines.append(f'\tError: {msg}') lines.append("") return "\n".join(lines)
[docs]def handle_verify_action(opts): """ Verify the signatures in the specified license file, using lictest. """ args = ["-c"] if opts.license: args.extend(["-f", opts.license]) try: return execute_lictest(args) except subprocess.CalledProcessError as exc: return exc.output
[docs]def handle_generic_action(action, opts): """ Execute the lmutil subcommand for the given action. """ args = [] if opts.license: args.extend(["-c", opts.license]) if action == "REREAD": args.extend(["-vendor", "SCHROD"]) elif action == "NEWLOG": args.append("SCHROD") if opts.args: args.extend(opts.args) lmaction = "lm" + action.lower() try: return execute_lmutil([lmaction] + args) except subprocess.CalledProcessError as exc: return exc.output
[docs]def handle_install_action(opts): """ Install a license in the license directory. If a license key is given, download the license from Netsuite, first. """ try: if opts.license_key: installed_file = install_netsuite_license(opts.license_key) elif opts.license: if "@" in opts.license: # opts.license is "port@host" installed_file = install_remote_server_license(opts.license) elif os.path.isdir(opts.license): installed_files = {} for filename in os.listdir(opts.license): filename = os.path.join(opts.license, filename) installed_file = install_license(filename) installed_files[filename] = installed_file if len(installed_files) != 1: msg = f"Installed {len(installed_files)} license files:\n" msg += '\n'.join( (f' {k} as {v}' for (k, v) in installed_files.items())) return msg else: # TODO: Verify that it's a license file installed_file = install_license(opts.license) return "Installed license file:\n" + installed_file except (licerror.LicenseException, NetsuiteError) as exc: return "ERROR: %s" % exc
[docs]def install_remote_server_license(lic_server_address): """ Generate a license file pointing to a remote server, and install it in the license directory. :type lic_server_address: str :param lic_server_address: The address of the license server, in the form "port@host". """ res = LicenseResource(lic_server_address) content = res.stub_license() return install_license_text(content)
[docs]def install_netsuite_license(key): """ Download a license from Netsuite using the given key, and install it in the license directory. """ with temporary_directory(): # Call license_directory for its side effects. It will raise an exception # if a suitable license directory can't be found or created. This prevents # consuming keys in get_netsuite_license without actually writing a license. license_directory() downloaded_file = get_netsuite_license(key) return install_license(downloaded_file)
[docs]def get_netsuite_license(key): """ Download a license from Netsuite, using the given key. Returns the pathname of the downloaded file. A NetsuiteError is raised if there's a problem using the given license key. """ info = netsuiteapi.get_license_request_info(key) if netsuiteapi.ERROR_CODE in info: raise NetsuiteError(info[netsuiteapi.ERROR_MSG]) key_type = info.get(netsuiteapi.LICENSE_TYPE) if key_type == netsuiteapi.SERVER: if not info.get(netsuiteapi.LICENSE_REMAINING): raise NetsuiteError(NSERROR_KEY_USED) return download_server_license(key) elif key_type == netsuiteapi.NODELOCKED: count = int(info.get(netsuiteapi.MAX_COUNT, 0)) remaining = int(info.get(netsuiteapi.LICENSE_REMAINING, 0)) if not remaining: raise NetsuiteError(NSERROR_KEY_EXHAUSTED % count) return download_nodelocked_license(key) else: raise NetsuiteError(NSERROR_BAD_KEY_TYPE % key_type)
[docs]def download_server_license(key, hostname=None, hostid=None, lmgrdport=netsuiteapi.DEFAULT_LMGRD_PORT, schrodport=netsuiteapi.DEFAULT_SCHROD_PORT): """ Retrieves the server license corresponding to the key and saves it to disk, in the license directory. The lmgrd and SCHROD ports to use in the license may be specified. Returns the pathname of the downloaded file. A NetsuiteError is raised if there's a problem using the given license key. :param hostid: If more than one host id is given, this value must be a string of comma-separated host ids. Otherwise NetSuite will throw an error. :type hostid: str """ if hostname is None: hostname = sysinfo.get_hostname() if hostid is None: hostid = get_first_hostid() resp = netsuiteapi.retrieve_license_file(key, [hostname], [hostid], lmgrdport=lmgrdport, schrodport=schrodport) if netsuiteapi.ERROR_CODE in resp: raise NetsuiteError(resp[netsuiteapi.ERROR_MSG]) licstring = resp.get(netsuiteapi.LICENSE_STRING) if not licstring: raise NetsuiteError(NSERROR_GET_FAILED) return write_file(TEMPORARY_LICENSE, licstring)
[docs]def download_nodelocked_license(key, hostname=None, hostid=None): """ Retrieves the license corresponding to the key and saves it to disk, in the licenses directory. Returns the pathname of the downloaded file. A NetsuiteError is raised if there's a problem using the given license key. :param hostid: If more than one host id is given, this value must be a string of comma-separated host ids. Otherwise NetSuite will throw an error. :type hostid: str """ if hostname is None: hostname = socket.gethostname() if hostid is None: hostid = get_first_hostid() resp = netsuiteapi.retrieve_license_file(key, [hostname], [hostid]) if netsuiteapi.ERROR_CODE in resp: raise NetsuiteError(resp[netsuiteapi.ERROR_MSG]) licstring = resp.get(netsuiteapi.LICENSE_STRING) if not licstring: raise NetsuiteError(NSERROR_GET_FAILED) return write_file(TEMPORARY_LICENSE, licstring)
[docs]def install_license_text(text): """ Install the given text as a license file, with the canonical filename. If it's already installed, nothing is done. The pathname of the installed license is returned. """ with temporary_directory(): lic_file = write_file("license.txt", text) inst_path = install_license(lic_file) return inst_path
[docs]def install_license(filename): """ Copy the given license file to the standard licenses directory and give it its canonical filename. If it's already in the licenses directory, it's just renamed. If its name already matches the canonical pattern, it is not renamed. """ filedir = file_dirname(filename) licdir = license_directory() if filedir != licdir: inst_path = installed_pathname(filename) if inst_path is not None: return inst_path res = LicenseResource(Path(filename)) if res.errors(): raise licerror.LicenseException(res.errors()[0]) canon_name = res.canonical_filename() canon_path = os.path.join(licdir, canon_name) if os.path.exists(canon_path): canon_path = fileutils.get_next_filename(canon_path, '_') if file_dirname(filename) == licdir: shutil.move(filename, canon_path) else: shutil.copy(filename, canon_path) st = os.stat(canon_path) os.chmod(canon_path, st.st_mode | stat.S_IRGRP | stat.S_IROTH) return canon_path
[docs]def installed_pathname(filename): """ Check whether the given license file is already installed in the licenses directory under a different name. Return the pathname of the installed license, if it's found there, otherwise, return None. """ licdir = license_directory() for instpath in glob.glob(os.path.join(licdir, '*.lic')): if filecmp.cmp(filename, instpath): return instpath return None
[docs]def file_dirname(pathname): """ Return the name of the directory containing the given file. """ return os.path.dirname(os.path.abspath(pathname))
[docs]def write_file(filename, contents): """ Writes the given content into a file in the CWD. The full pathname of the written file is returned. """ pathname = os.path.abspath(filename) with open(pathname, 'w', newline='\n') as f: f.write(contents) return pathname
####################################################################### # "licadmin INFO" implementation #######################################################################
[docs]@contextmanager def working_directory(new_cwd): """ Change the CWD within some limited context """ orig_cwd = os.getcwd() os.chdir(new_cwd) yield os.chdir(orig_cwd)
[docs]@contextmanager def temporary_directory(): """ Create a temporary directory and make that the current directory in this context. When the context is exited, chdir back to the original directory and delete the temporary directory. """ orig_cwd = os.getcwd() tmpdir = tempfile.mkdtemp() os.chdir(tmpdir) yield os.chdir(orig_cwd) shutil.rmtree(tmpdir)
[docs]def handle_info_action(opts): """ Create an archive of information for troubleshooting license issues. """ user = getpass.getuser() host = shorthost(socket.gethostname()) cwd = os.getcwd() with TemporaryDirectory(prefix='licadmin-', dir=cwd) as archive_dir: licenses = [] for license in opts.license.split(os.pathsep): if os.path.exists(license): license = os.path.abspath(license) licenses.append(license) src_dir = os.path.join(archive_dir, 'src') os.mkdir(src_dir) with working_directory(src_dir): general_info = { "HOST": socket.getfqdn(), "USER": user, "CWD": cwd, "LICENSE": licenses, "LIC_SRC": opts._source, "SHELL": os.getenv("SHELL", ""), "SCHRODINGER": os.getenv("SCHRODINGER", ""), "LM_LICENSE_FILE": os.getenv("LM_LICENSE_FILE", ""), "SCHROD_LICENSE_FILE": os.getenv("SCHROD_LICENSE_FILE", ""), "SCHRODINGER_LICENSE": os.getenv("SCHRODINGER_LICENSE", ""), "FLEXLM_DIAGNOSTICS": os.getenv("FLEXLM_DIAGNOSTICS", ""), "lmhostid": execute_lmutil(["lmhostid"]), "lmpath": get_flexlm_search_path(), "schrodinger_files": get_shared_file_listing(), "shared_files": get_shared_file_listing(), "mmshare_dir": get_mmshare_listing(), "server_processes": get_server_processes(), "network_info": get_network_info() } with open(GENERAL_INFO_FILE, 'w') as fp: fp.write(GENERAL_INFO_TEMPLATE.substitute(general_info)) with open("machid.txt", "w") as fp: fp.write(get_machid_output()) with open("CKSUM.txt", "w") as fp: fp.write(handle_generic_action("CKSUM", opts)) with open("STAT.txt", "w") as fp: fp.write(handle_stat_action(opts)) with open("DIAG.txt", "w") as fp: fp.write(handle_diag_action(opts)) with open("VERIFY.txt", "w") as fp: fp.write(handle_verify_action(opts)) with open("CHECK.txt", "w") as fp: fp.write(handle_check_action(opts)) # In INFO, we want all resources for debugging, it won't matter # if there are dupes all_resources = find_license_resources(unique=False) _copy_files_to_archive(_find_license_files(all_resources), "license_files") _copy_files_to_archive(_find_server_logs(all_resources), "server_logs") shutil.copyfile(system_hosts_file(), 'etc-hosts.txt') try: archive_name = f"{user}-{host}-license-info.zip" archive_file = os.path.join(archive_dir, archive_name) create_archive_file(archive_file, src_dir) os.replace(archive_file, os.path.join(cwd, archive_name)) return f"** Please send '{archive_name}' to help@schrodinger.com **" except OSError as exc: return f"Unable to create '{archive_file}': {str(exc)}"
GENERAL_INFO_FILE = "general-info.txt" GENERAL_INFO_TEMPLATE = Template("""General ------- hostname: $HOST username: $USER curdir: $CWD Hostid ------ ${lmhostid} License ------- license: $LICENSE source: $LIC_SRC License Path Settings --------------------- SCHROD_LICENSE_FILE=${lmpath} Environment variables --------------------- SHELL=$SHELL SCHRODINGER=$SCHRODINGER LM_LICENSE_FILE=$LM_LICENSE_FILE SCHROD_LICENSE_FILE=$SCHROD_LICENSE_FILE SCHRODINGER_LICENSE=$SCHRODINGER_LICENSE FLEXLM_DIAGNOSTICS=$FLEXLM_DIAGNOSTICS Schrodinger installation ------------------------ ${schrodinger_files} ${mmshare_dir} Shared files directory ---------------------- ${shared_files} License daemons --------------- ${server_processes} Network interfaces ------------------ ${network_info} """)
[docs]def options_file(licfile=""): """ Return the pathname of the FLEXlm options file associated with the given license. """ if not licfile: return "" vendor_re = re.compile(r"VENDOR\s+SCHROD") with open(licfile) as fp: for line in fp.readlines(): if vendor_re.match(line): vendor = flexlm.Vendor(line) optfile = vendor.optionfile() if optfile: return optfile else: break licdir = os.path.dirname(licfile) return os.path.join(licdir, "SCHROD.opt")
[docs]def system_hosts_file(): """ Return the pathname of the /etc/hosts file, or its equivalent. """ if sys.platform == "win32": sysroot = os.getenv("SystemRoot", os.environ["SYSTEMROOT"]) return os.path.join(sysroot, "system32", "drivers", "etc", "hosts") else: return "/etc/hosts"
[docs]def get_machid_output(): """ Return the output of the machid utility """ try: return subprocess.check_output([os.path.join(SCHRODINGER, "machid")], stderr=subprocess.STDOUT, universal_newlines=True) except subprocess.CalledProcessError as exc: return exc.output
[docs]def get_network_info(): """ Return network configuration info from ifconfig. """ try: if sys.platform == "win32": cmdline = ["ipconfig", "-all"] else: if os.path.isfile("/sbin/ifconfig"): cmdline = ["/sbin/ifconfig"] else: cmdline = ["/sbin/ip", "address"] return subprocess.check_output(cmdline, stderr=subprocess.STDOUT, universal_newlines=True, errors='backslashreplace') except subprocess.CalledProcessError as exc: return "%% %s\n%s" % (cmdline, exc.output) except OSError as exc: return "%% %s\n%s" % (cmdline, str(exc))
[docs]def get_mmshare_listing(): """ Return info about the mmshare directory. """ cmd = ["ls", "-l"] + glob.glob(os.path.join(SCHRODINGER, "mmshare-v*")) try: schrod_files = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) except subprocess.CalledProcessError as exc: schrod_files = exc.output return schrod_files
[docs]def get_shared_file_listing(): """ Return info about files installed in the shared license directory. """ cmd = [ "ls", "-l", os.path.join(SHARED_LICENSE_DIR, "lic"), os.path.join(SHARED_LICENSE_DIR, "lmgrd"), os.path.join(SHARED_LICENSE_DIR, "SCHROD") ] try: schrod_files = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) except subprocess.CalledProcessError as exc: schrod_files = exc.output return schrod_files
# Server process listings server_ps_attrs = [ 'pid', 'ppid', 'name', 'exe', 'memory_info', 'memory_percent', 'cpu_percent', 'username', 'cmdline' ] server_ps_head_fmt = "%-6s %-8s %6s %6s %4s %4s %6s %7s %s" server_ps_body_fmt = " ".join( ("{name:<6s} {username:<8s} {pid:6d}", "{ppid:6d} {cpu_percent:4.1f} {memory_percent:4.1f} {rss:>6s}", "{vms:>7s} {cmdstr}"))
[docs]def get_server_processes(): """ Return info about any running lmgrd and SCHROD processes. """ lines = [] lines.append(server_ps_head_fmt % ('name', 'user', 'pid', 'ppid', 'cpu%', 'mem%', 'rss(KB)', 'vms(KB)', 'cmdline')) lines.append(server_ps_head_fmt % ('------', '--------', '------', '------', '----', '----', '------', '-------', '------------')) for proc in psutil.process_iter(): try: pathname, ext = os.path.splitext(proc.exe()) exename = os.path.basename(pathname) if exename == "lmgrd" or exename == "SCHROD": pinfo = process_attributes(proc, server_ps_attrs) lines.append(server_ps_body_fmt.format(**pinfo)) except (psutil.AccessDenied, psutil.NoSuchProcess, OSError): pass return "\n".join(lines)
[docs]def process_attributes(proc, attrs): """ Return a dict containing the specified attributes for the given psutil.Process object. Make sure that filler values for unavailable data are of the correct type (LIC-548) """ pinfo = proc.as_dict(attrs=attrs, ad_value="PRIV") for key in ("pid", "ppid"): if pinfo[key] == "PRIV": pinfo[key] = -1 for key in ("cpu_percent", "memory_percent"): if pinfo[key] == "PRIV": pinfo[key] = -1.0 KB = 1024 minfo = pinfo['memory_info'] if minfo is None or minfo == "PRIV": (pinfo['rss'], pinfo['vms']) = ("-", "-") else: (pinfo['rss'], pinfo['vms']) = (str(minfo.rss // KB), str(minfo.vms // KB)) pinfo['cmdstr'] = subprocess.list2cmdline(pinfo['cmdline']) return pinfo
def _copy_files_to_archive(filenames, output_dir): """ Copy given filenames to output directory. They will be ripped out of their location except if there is a duplicate, in which case they will be written by absolute path, with the path separators as underscore. into underscore. :param filenames: list of pathnames to archive :type filenames: list of str :param output_dir: directory where filenames will be copied :type output_dir: str :param index_filename: name of output_dir/filename listing all filenames :type str: str """ if not os.path.exists(output_dir): os.makedirs(output_dir) index = [] for srcfile in filenames: destfile = os.path.join(output_dir, os.path.basename(srcfile)) if os.path.exists(destfile): destfile = os.path.join(output_dir, re.sub(r"[:/\\]+", "_", srcfile)) index.append(srcfile) shutil.copyfile(srcfile, destfile) index_filename = output_dir.upper() + ".txt" index_file = os.path.join(output_dir, index_filename) with open(index_file, 'w') as fp: fp.write("\n".join(index) + "\n") def _find_license_files(resources): """ Return a list of all license files found. Notably, this may return more results than resources, because this is used for licadmin INFO. """ license_files = set() for lic in resources: if lic.pathname(): license_files.add(lic.pathname()) return license_files def _find_server_logs(resources): """ Return a set of all server logfiles found on the standard search path. The standard search path includes... 1. Any directory in which a license is installed 2. The parent directory of a licenses/ directory in which a license is installed 3. The user's home directory 4. The current directory """ logfiles = set() for lic in resources: if lic.pathname(): logfile = server_log(lic.pathname()) if os.path.exists(logfile): logfiles.add(logfile) home_dir = fileutils.get_directory_path(fileutils.HOME) for logfile in (os.path.join(home_dir, LMGRD_LOG), os.path.join(home_dir, LMGRD_ALTLOG), os.path.join(os.getcwd(), LMGRD_LOG), os.path.join(os.getcwd(), LMGRD_ALTLOG)): if os.path.exists(logfile): logfiles.add(logfile) return logfiles
[docs]def create_archive_file(archive_file, archive_dir): """ Create a zip archive with the contents of archive_dir """ with ZipFile(archive_file, "w") as zip: # Place all files in a directory named after the archive minus # the .zip extension toplevel_dir = os.path.splitext(os.path.basename(archive_file))[0] for root, dirs, files in os.walk(archive_dir): relative_path = os.path.relpath(root, archive_dir) for file in files: zip.write(os.path.join(root, file), arcname=os.path.join(toplevel_dir, relative_path, file))
[docs]def shorthost(hostname): """ Return the short hostname for the given hostname, without the domain name. If hostname is an IP address, it is returned unmodified. """ if re.match(r"\d+\.\d+\.\d+\.\d+$", hostname): return hostname return hostname.split(".", 1)[0]
[docs]def get_flexlm_search_path(): """ Execute "lmutil lmpath" to determine the actual search path used for license checkouts. """ try: output = execute_lmutil(["lmpath", "-status"]) for line in output.splitlines(): if line.startswith("schrod:"): return line.split(" ", 1)[1] except subprocess.CalledProcessError as exc: return exc.output
[docs]def has_license_file_suffix(filename): return filename.endswith((".lic"))
[docs]def eth0_is_missing(): if not sys.platform.startswith('linux'): return False return not os.path.exists('/sys/class/net/eth0')
[docs]def install_schrod_plist(): """ Install FLEXlm plist to /Library/LaunchDaemons which allow to autostart lmgrd and SCHROD on boot. This require root privilege to install to the location. """ if not sys.platform.startswith("darwin"): return import plistlib d = { 'Label': 'SchrodLicenseManager', 'ProgramArguments': [ os.path.join(os.environ["SCHRODINGER"], "licadmin"), "START" ], 'RunAtLoad': True, 'AbandonProcessGroup': True } # install to MMSHARE_EXEC schrod_plist = os.path.join("/Library/LaunchDaemons", "Schrod.plist") try: with open(schrod_plist, "wb") as f: plistlib.writePlist(d, f) except OSError: sys.stderr.write( "You need to have root privileges to install Schrod.plist to " "/Library/LaunchDaemons.") raise
[docs]def getLicenseFileList(): """ Get the list of license files to use from available resources. """ resources = find_license_resources() files = [res.location() for res in resources if res.location() != ""] return files
[docs]def feature_expiring(feature, days=30): """ Return False if the given license feature doesn't expire in given number of days. True, otherwise. """ lic_files = getLicenseFileList() errs = [] for lic_file in lic_files: try: diag = run_lmdiag(lic_file, feature=feature) except licerror.LicenseException as exc: errs.append(str(exc)) continue for resource_name, feature_dict in diag.items(): for feature_info in feature_dict[feature]: delta = feature_info.expires.date() - date.today() if delta.days > days: return False if errs: log.debug(f"lmdiag failed with following error(s): {errs}") return True
def _add_licenses(opts): """ Modifies an argparse.Namespace object to include the correct license information. """ if opts.license: opts._source = "cmdline" else: resources = find_license_resources() if not resources: no_lic_msg = "No license was specified" if opts.action != "INFO": raise UsageError(no_lic_msg) else: log.info(no_lic_msg) if opts.action == "START" or opts.action == "STOP": if not opts.license: license = local_server_resource() if license: opts.license = license.server_pathname() opts._source = license.source else: raise MissingLicenseFile( "No license file found for local server") else: # Find unique license by locations, rather than resources # e.g. collapsing all server licenses into one location # This makes INFO or STAT easier to understand license_locations = [] license_sources = [] for r in resources: if r.location() not in license_locations: license_sources.append(r.source) license_locations.append(r.location()) opts.license = os.pathsep.join(license_locations) opts._source = os.pathsep.join(license_sources) log.info("License: %s (%s)" % (opts.license, opts._source))
[docs]def main(args): opts = parse_args(args) if opts.debug: log.setLevel(logging.DEBUG) output = "" if opts.action == "INSTALL_LAUNCHD_FILE": install_schrod_plist() elif opts.action == "HOSTID": output = execute_lmutil(["lmhostid"]) elif opts.action == "LIST": output = handle_list_action(opts) elif opts.action == "CHECK": output = handle_check_action(opts) elif opts.action == "VER": output = handle_ver_action(opts) elif opts.action == "STAT": output = handle_stat_action(opts) elif opts.action == "DIAG": output = handle_diag_action(opts) elif opts.action == "START": output = handle_start_action(opts) elif opts.action == "STOP": output = handle_stop_action(opts) elif opts.action == "INFO": output = handle_info_action(opts) elif opts.action == "VERIFY": output = handle_verify_action(opts) elif opts.action == "INSTALL": output = handle_install_action(opts) elif opts.action in GENERIC_ACTIONS: output = handle_generic_action(opts.action, opts) else: raise UsageError("Unrecognized action: " + opts.action) if output: log.info(output) return 0
[docs]def run(args): try: return main(args) except RuntimeError as err: log.error(err) except subprocess.CalledProcessError as err: log.error(err.output) except (MissingExecutableError, MissingLicenseFile) as err: log.error(err) except (licerror.LicenseException, PermissionError) as err: log.error("ERROR: %s" % str(err)) except UsageError as err: log.error("ERROR: %s" % str(err)) parser.print_usage() print("(Type '$SCHRODINGER/licadmin -h' for help.)") return 1
if __name__ == '__main__': exitcode = run(sys.argv[1:]) sys.exit(exitcode)