Merge branch 'release/0.11.2'

Author: dan-blanchard

.gitattributes

@@ -0,0 +1 @@
+*   text=auto

README.rst

@@ -57,105 +57,4 @@ License
 Changelog
 ~~~~~~~~~
 
--  v0.11.1
-
-   +  Made web front-end for job monitoring a separate script, ``gridmap_web``,
-      since it can be used to talk to any ``JobMonitor`` instance. (Fixes #14)
-   +  Fixed crash if a stalled job comes back from the dead (#15).
-   +  Fixed crash if job's hostname is somehow not in white list and the job
-      needs to be resubmitted (#16).
-   +  Fixed crash from trying to set ``matplotlib`` back-end multiple times.
-   +  Cleaned up some imports and removed some unused variables.
-
--  v0.11.0
-
-   + Vastly more reliable job completion information thanks to switch back to
-     using 0MQ for communication with worker nodes. No more unpickling
-     exceptions because the SGE DRMAA implementation frequently liked to say
-     jobs were finished when they were not.
-   + Add back web monitor to report basic job status.
-   + Switch to using custom fork of drmaa-python until
-     drmaa-python/drmaa-python#4, which fixes Python 3 compatibility issues,
-     gets merged.
-   + Now creates temporary directory for storing log files if it doesn't
-     exist.
-   + Travis-CI SGE installation has been streamlined.
-   + Switch to using sphinx and readthedocs for documentation.
-   + Added detection of stalled jobs. GridMap will also automatically restart
-     any jobs that appear stuck (up to 3 times by default), and email you a
-     report describing their CPU and memory usage over time.
-
--  v0.10.3
-
-   + Fix issue where ``clean_path`` wasn't being called on the working
-     directory, which was causing ETS-specific issues.
-   + Add a couple workarounds for issues with setting environment variables in
-     Python 3.
-   + Made examples into unit tests and added first attempt at getting Travis
-     setup with SGE.
-
--  v0.10.2
-
-   + Working directory is now correctly set for each job.
-   + Simplified handling of environment variables. Should now all be passed on
-     properly.
-
--  v0.10.1
-
-   + Can now import ``JobException`` directly from ``gridmap`` package instead
-     of having to import from ``gridmap.job``.
-
--  v0.10.0
-
-   + Now raise a ``JobException`` instead of an ``Exception`` when one of the
-     jobs has crashed.
-   + Fixed potential pip installation issue from importing package for version
-     number.
-
--  v0.9.9
-
-   + Changed way job results are retrieved to be a bit more efficient in cases
-     of errors.
-   + All job metadata is now retrieved before job output is, which should
-     hopefully alleviate issues where we can't get the metadata because its been
-     flushed too quickly by the grid engine.
-
--  v0.9.8
-
-   + Fixed a bug where only the first error was still showing because of an
-     extra exception caused by job_output being undefined.
-   + Fixed unhandled Exception with error code 24 (since somehow that is not an
-     InvalidJobException, but just an Exception in drmaa-python).
-
--  v0.9.7
-
-   + No longer dies with InvalidJobException when failing to retrieve job
-     metadata from DRMAA service.
-   + Now print all exceptions encountered for jobs submitted instead of just
-     exiting after first one.
-   + Die via exception instead of sys.exit when there were problems with some of
-     the submitted jobs.
-
--  v0.9.6
-
-   + Fixed bug where jobs were being aborted before they ran.
-
--  v0.9.5
-
-   + Fixed bug where ``GRID_MAP_USE_MEM_FREE`` would only be interpretted as true if
-     spelled 'True'.
-   + Added documentation describing how to override constants.
-
--  v0.9.4
-
-   +  Added support for overriding the default queue and other constants via
-      environment variables. For example, to change the default queue, just set
-      the environment variable ``GRID_MAP_DEFAULT_QUEUE``.
-   +  Substantially more information is given about crashing jobs when we fail
-      to unpickle the results from the Redis database.
-
--  v0.9.3
-
-   +  Fixed serious bug where gridmap could not be imported in some instances.
-   +  Refactored things a bit so there is no longer one large module with all of
-      the code in it. (Doesn't change package interface)
+See `GitHub releases <https://github.com/EducationalTestingService/gridmap/releases>`__.

gridmap/conf.py

@@ -75,7 +75,7 @@
 try:
     import drmaa
     DRMAA_PRESENT = True
-except ImportError:
+except (ImportError, RuntimeError):
     logger = logging.getLogger(__name__)
     logger.warning('Could not import drmaa. Only local multiprocessing ' +
                    'supported.')

gridmap/job.py

@@ -52,6 +52,7 @@
 from multiprocessing import Pool
 from socket import gethostname
 
+import psutil
 import zmq
 
 from gridmap.conf import (CHECK_FREQUENCY, CREATE_PLOTS, DEFAULT_QUEUE,
@@ -77,6 +78,12 @@
     import matplotlib.pyplot as plt
 
 
+# Set of "not running" job statuses
+SLEEP_STATUSES = {psutil.STATUS_SLEEPING, psutil.STATUS_DEAD,
+                  psutil.STATUS_IDLE, psutil.STATUS_STOPPED,
+                  psutil.STATUS_ZOMBIE}
+
+
 class JobException(Exception):
     '''
     New exception type for when one of the jobs crashed.
@@ -394,8 +401,8 @@ def check_if_alive(self):
                         logger.error("job died for unknown reason")
                         job.cause_of_death = "unknown"
                     elif (len(job.track_cpu) > MAX_IDLE_HEARTBEATS and
-                          all(cpu_load <= IDLE_THRESHOLD and state == 'S'
-                              for cpu_load, state in
+                          all((cpu_load <= IDLE_THRESHOLD and
+                               state in SLEEP_STATUSES) for cpu_load, state in
                               job.track_cpu[-MAX_IDLE_HEARTBEATS:])):
                         logger.error('Job stalled for unknown reason.')
                         job.cause_of_death = 'stalled'
@@ -417,6 +424,8 @@ def check_if_alive(self):
                 # try to resubmit
                 old_id = job.jobid
                 handle_resubmit(self.session_id, job, temp_dir=self.temp_dir)
+                logging.info('Resubmitted job %s; it now has ID %s', old_id,
+                             job.jobid)
                 if job.jobid is None:
                     logger.error("giving up on job")
                     job.ret = "job dead"
@@ -573,9 +582,9 @@ def handle_resubmit(session_id, job, temp_dir='/scratch/'):
         job.num_resubmits += 1
         job.cause_of_death = ""
 
-        return _resubmit(session_id, job, temp_dir)
+        _resubmit(session_id, job, temp_dir)
     else:
-        return None
+        job.jobid = None
 
 
 def _execute(job):
@@ -701,7 +710,7 @@ def _append_job_to_session(session, job, temp_dir='/scratch/', quiet=True):
 
     if not quiet:
         print('Your job {} has been submitted with id {}'.format(job.name,
-                                                                   jobid),
+                                                                 jobid),
               file=sys.stderr)
 
     session.deleteJobTemplate(jt)

gridmap/runner.py

@@ -42,6 +42,7 @@
 from io import open
 from subprocess import check_output
 
+from psutil import Process
 import zmq
 
 from gridmap.conf import HEARTBEAT_FREQUENCY
@@ -97,31 +98,6 @@ def _heart_beat(job_id, address, parent_pid=-1, log_file="", wait_sec=45):
         time.sleep(wait_sec)
 
 
-def _VmB(VmKey, pid):
-    """
-    get various mem usage properties of process with id pid in MB
-    """
-
-    _proc_status = '/proc/%d/status' % pid
-
-    _scale = {'kB': 1.0/1024.0, 'mB': 1.0,
-              'KB': 1.0/1024.0, 'MB': 1.0}
-
-     # get pseudo file  /proc/<pid>/status
-    try:
-        with open(_proc_status) as t:
-            v = t.read()
-    except:
-        return 0.0  # non-Linux?
-    # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
-    i = v.index(VmKey)
-    v = v[i:].split(None, 3)  # whitespace
-    if len(v) < 3:
-        return 0.0  # invalid format?
-    # convert Vm value to bytes
-    return float(v[1]) * _scale[v[2]]
-
-
 def get_memory_usage(pid):
     """
     :param pid: Process ID for job whose memory usage we'd like to check.
@@ -130,7 +106,8 @@ def get_memory_usage(pid):
     :returns: Memory usage of process in Mb.
     """
 
-    return _VmB('VmSize:', pid)
+    p = Process(pid)
+    return p.get_memory_usage()[0] / (1024.0 ** 2.0)
 
 
 def get_cpu_load(pid):
@@ -143,19 +120,8 @@ def get_cpu_load(pid):
     :rtype: (float, str)
     """
 
-
-    command = ["ps", "h", "-o", "pcpu,state", "-p", "%d" % (pid)]
-
-    try:
-        cpu_load, state = check_output(command).strip().split()
-        cpu_load = float(cpu_load)
-    except:
-        logger = logging.getLogger(__name__)
-        logger.warning('Getting CPU info failed.', exc_info=True)
-        cpu_load = float('NaN')
-        state = '?'
-
-    return cpu_load, state
+    p = Process(pid)
+    return p.get_cpu_percent(), p.status
 
 
 def get_job_status(parent_pid):

gridmap/version.py

@@ -28,5 +28,5 @@
 :organization: ETS
 '''
 
-__version__ = '0.11.1'
+__version__ = '0.11.2'
 VERSION = tuple(int(x) for x in __version__.split('.'))

scripts/gridmap_web gridmap/web.pyscripts/gridmap_web renamed to gridmap/web.py

@@ -124,11 +124,17 @@ def job_to_html(job):
         return body_text.encode()
 
 
-def _main():
+def main(argv=None):
     """
     Parse the command line inputs and start web monitor.
+
+    :param argv: List of arguments, as if specified on the command-line.
+                 If None, ``sys.argv`` is used instead.
+    :type argv: list of str
     """
     # Get command line arguments
+    if argv is None:
+        argv = sys.argv
     parser = argparse.ArgumentParser(description="Provides a web interface to \
                                                   0MQ job monitor.")
     parser.add_argument('module_dir',
@@ -138,7 +144,7 @@ def _main():
     parser.add_argument('-p', '--port',
                         help='Port for server to listen on.', type=int,
                         default=8076)
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
 
     # Make warnings from built-in warnings module get formatted more nicely
     logging.captureWarnings(True)
@@ -161,4 +167,4 @@ def _main():
 
 
 if __name__ == "__main__":
-    _main()
+    main()

requirements.txt

@@ -1,2 +1,3 @@
-git+git://github.com/dan-blanchard/drmaa-python#egg=drmaa
+drmaa
+psutil
 pyzmq

setup.py

@@ -25,11 +25,18 @@
 exec(compile(open('gridmap/version.py').read(), 'gridmap/version.py', 'exec'))
 # (we use the above instead of execfile for Python 3.x compatibility)
 
+
 def readme():
     with open('README.rst') as f:
         return f.read()
 
 
+def requirements():
+    with open('requirements.txt') as f:
+        reqs = f.read().splitlines()
+    return reqs
+
+
 setup(name='gridmap',
       version=__version__,
       description=('Easily map Python functions onto a cluster using a ' +
@@ -41,8 +48,8 @@ def readme():
       author_email='dblanchard@ets.org',
       license='GPL',
       packages=['gridmap'],
-      install_requires=['drmaa', 'pyzmq'],
-      scripts=['scripts/gridmap_web'],
+      install_requires=requirements(),
+      entry_points={'console_scripts': ['gridmap_web = gridmap.web:main']},
       classifiers=['Intended Audience :: Science/Research',
                    'Intended Audience :: Developers',
                    'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',