Skip to content

Commit f2eda18

Browse files
dan-blancharddan-blanchard
committed
Merge branch 'release/0.13.0'
2 parents 39d3b1b + f8a1463 commit f2eda18

14 files changed

Lines changed: 147 additions & 92 deletions

File tree

‎.travis.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
language: python
22
python:
33
- 2.7
4-
- 3.3
4+
- 3.4
55

66
notifications:
77
email: false
88

99
# Install stuff
10-
virtualenv:
11-
system_site_packages: true
1210
before_install:
1311
- travis/install_sge.sh
1412
- export SGE_ROOT=/var/lib/gridengine

‎README.rst‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ A package to allow you to easily create jobs on the cluster directly from
2727
Python. You can directly map Python functions onto the cluster without needing
2828
to write any wrapper code yourself.
2929

30-
This is the ETS fork of an older project called Python Grid. Unlike the older
31-
version, it is Python 2/3 compatible. Another major difference is that you can
32-
change the configuration via environment variables instead of having to modify
33-
a Python file in your ``site-packages`` directory. We've also removed some cruft
34-
and improved the reliability of some of the features.
30+
This is the ETS fork of an older project called `Python Grid <https://github.com/cwidmer/pythongrid>`__. Unlike the older
31+
version, it is Python 2/3 compatible. Another major difference is that you can
32+
change the configuration via environment variables instead of having to modify
33+
a Python file in your ``site-packages`` directory. We've also removed some
34+
cruft and improved the reliability of some of the features.
3535

3636
For some examples of how to use it, check out ``map_reduce.py`` (for a simple
3737
example of how you can map a function onto the cluster) and ``manual.py`` (for
@@ -46,6 +46,7 @@ Requirements
4646
~~~~~~~~~~~~
4747

4848
- `drmaa <https://github.com/drmaa-python/drmaa-python>`__
49+
- `psutil <https://github.com/giampaolo/psutil>`__
4950
- `pyzmq <https://github.com/zeromq/pyzmq>`__
5051
- Python 2.7+
5152

‎docs/conf.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def __getattr__(cls, name):
6767

6868
# General information about the project.
6969
project = u'GridMap'
70-
copyright = u'2008-2012 Max-Planck-Society, 2012-2013 ETS'
70+
copyright = u'2008-2012 Max-Planck-Society, 2012-2014 ETS'
7171

7272
# The version info for the project you're documenting, acts as replacement for
7373
# |version| and |release|, also used in various other places throughout the
@@ -276,7 +276,7 @@ def __getattr__(cls, name):
276276
epub_title = u'GridMap'
277277
epub_author = u'Daniel Blanchard, Cheng Soon Ong, Christian Widmer'
278278
epub_publisher = u'Educational Testing Service'
279-
epub_copyright = u'2008-2012 Max-Planck-Society, 2012-2013 Educational Testing Service'
279+
epub_copyright = u'2008-2012 Max-Planck-Society, 2012-2014 Educational Testing Service'
280280

281281
# The language of the text. It defaults to the language option
282282
# or en if the language is not set.

‎examples/manual.py‎

100644100755
Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
# Written (W) 2008-2012 Christian Widmer
44
# Written (W) 2008-2010 Cheng Soon Ong
5-
# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
6-
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
5+
# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
6+
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
77

88
# This file is part of GridMap.
99

@@ -28,16 +28,29 @@
2828

2929
from __future__ import print_function, unicode_literals
3030

31-
import time
31+
import logging
32+
from datetime import datetime
3233

3334
from gridmap import Job, process_jobs
3435

3536

37+
def sleep_walk(secs):
38+
'''
39+
Pass the time by adding numbers until the specified number of seconds has
40+
elapsed. Intended as a replacement for ``time.sleep`` that doesn't leave the
41+
CPU idle (which will make the job seem like it's stalled).
42+
'''
43+
start_time = datetime.now()
44+
num = 0
45+
while (datetime.now() - start_time).seconds < secs:
46+
num = num + 1
47+
48+
3649
def compute_factorial(n):
3750
"""
3851
computes factorial of n
3952
"""
40-
time.sleep(10)
53+
sleep_walk(10)
4154
ret = 1
4255
for i in range(n):
4356
ret = ret * (i + 1)
@@ -62,7 +75,9 @@ def make_jobs():
6275

6376
# create job objects
6477
for arg in inputvec:
65-
job = Job(compute_factorial, arg)
78+
# The default queue used by the Job class is all.q. You must specify
79+
# the `queue` keyword argument if that is not the name of your queue.
80+
job = Job(compute_factorial, arg, queue='all.q')
6681
jobs.append(job)
6782

6883
return jobs
@@ -73,6 +88,10 @@ def main():
7388
run a set of jobs on cluster
7489
"""
7590

91+
logging.captureWarnings(True)
92+
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
93+
'%(message)s'), level=logging.INFO)
94+
7695
print("=====================================")
7796
print("======== Submit and Wait ========")
7897
print("=====================================")
@@ -83,7 +102,7 @@ def main():
83102
print("sending function jobs to cluster")
84103
print("")
85104

86-
job_outputs = process_jobs(functionJobs)
105+
job_outputs = process_jobs(functionJobs, max_processes=4)
87106

88107
print("results from each job")
89108
for (i, result) in enumerate(job_outputs):

‎examples/map_reduce.py‎

100644100755
Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
# Written (W) 2008-2012 Christian Widmer
44
# Written (W) 2008-2010 Cheng Soon Ong
5-
# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
6-
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
5+
# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
6+
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
77

88
# This file is part of GridMap.
99

@@ -30,13 +30,29 @@
3030

3131
from __future__ import print_function, unicode_literals
3232

33+
import logging
34+
from datetime import datetime
35+
3336
from gridmap import grid_map
3437

3538

39+
def sleep_walk(secs):
40+
'''
41+
Pass the time by adding numbers until the specified number of seconds has
42+
elapsed. Intended as a replacement for ``time.sleep`` that doesn't leave the
43+
CPU idle (which will make the job seem like it's stalled).
44+
'''
45+
start_time = datetime.now()
46+
num = 0
47+
while (datetime.now() - start_time).seconds < secs:
48+
num = num + 1
49+
50+
3651
def computeFactorial(n):
3752
"""
3853
computes factorial of n
3954
"""
55+
sleep_walk(10)
4056
ret = 1
4157
for i in range(n):
4258
ret = ret * (i + 1)
@@ -48,10 +64,18 @@ def main():
4864
execute map example
4965
"""
5066

51-
args = [1, 2, 4, 8, 16]
67+
logging.captureWarnings(True)
68+
logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
69+
'%(message)s'), level=logging.INFO)
70+
71+
args = [3, 5, 10, 20]
5272

53-
intermediate_results = grid_map(computeFactorial, args, quiet=False)
73+
# The default queue used by grid_map is all.q. You must specify
74+
# the `queue` keyword argument if that is not the name of your queue.
75+
intermediate_results = grid_map(computeFactorial, args, quiet=False,
76+
max_processes=4, queue='all.q')
5477

78+
# Just print the items instead of really reducing. We could always sum them.
5579
print("reducing result")
5680
for i, ret in enumerate(intermediate_results):
5781
print("f({0}) = {1}".format(args[i], ret))

‎gridmap/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
# Written (W) 2008-2012 Christian Widmer
44
# Written (W) 2008-2010 Cheng Soon Ong
5-
# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
6-
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
5+
# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
6+
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
77

88
# This file is part of GridMap.
99

‎gridmap/conf.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
# Written (W) 2008-2012 Christian Widmer
44
# Written (W) 2008-2010 Cheng Soon Ong
5-
# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
6-
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
5+
# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
6+
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
77

88
# This file is part of GridMap.
99

‎gridmap/data.py‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
# Written (W) 2008-2012 Christian Widmer
44
# Written (W) 2008-2010 Cheng Soon Ong
5-
# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
6-
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
5+
# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
6+
# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
77

88
# This file is part of GridMap.
99

@@ -39,19 +39,6 @@
3939
import re
4040

4141

42-
def clean_path(path):
43-
'''
44-
Replace all weird SAN paths with normal paths. This is really
45-
ETS-specific, but shouldn't harm anyone else.
46-
'''
47-
48-
path = re.sub(r'/\.automount/\w+/SAN/NLP/(\w+)-(dynamic|static)',
49-
r'/home/nlp-\1/\2', path)
50-
path = re.sub(r'/\.automount/[^/]+/SAN/Research/HomeResearch',
51-
'/home/research', path)
52-
return path
53-
54-
5542
def zdumps(obj):
5643
"""
5744
dumps pickleable object into bz2 compressed string

0 commit comments

Comments
 (0)