Skip to content

Commit 126c171

Browse files
djw8605djw8605
authored
Merge pull request #115 from matyasselmeci/pr/py3
Python 3.6 support for stashcp
2 parents 67c88d8 + 3269fda commit 126c171

5 files changed

Lines changed: 148 additions & 74 deletions

File tree

‎.travis.yml‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,23 @@
11
sudo: required
22
env:
33
matrix:
4-
- OS_TYPE=centos OS_VERSION=6
5-
- OS_TYPE=centos OS_VERSION=7
6-
- OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
7-
- OS_TYPE=centos OS_VERSION=6 XRD_CACHE="root://sc-cache.chtc.wisc.edu"
8-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
9-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://sc-cache.chtc.wisc.edu"
10-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osg-new-york-stashcache.nrp.internet2.edu"
11-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osg-kansas-city-stashcache.nrp.internet2.edu"
12-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osg-chicago-stashcache.nrp.internet2.edu"
13-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://fiona.uvalight.net"
14-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://stashcache.t2.ucsd.edu"
15-
- OS_TYPE=centos OS_VERSION=7 XRD_CACHE="root://osg-gftp.pace.gatech.edu"
16-
- BUILD_TYPE=http OS_TYPE=centos OS_VERSION=7
17-
4+
- OS_TYPE=centos OS_VERSION=6 PYTHON_VERSION=2 XRD_CACHE=
5+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE=
6+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=3 XRD_CACHE=
7+
- OS_TYPE=centos OS_VERSION=6 PYTHON_VERSION=2 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
8+
- OS_TYPE=centos OS_VERSION=6 PYTHON_VERSION=2 XRD_CACHE="root://sc-cache.chtc.wisc.edu"
9+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
10+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://sc-cache.chtc.wisc.edu"
11+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=3 XRD_CACHE="root://its-condor-xrootd1.syr.edu"
12+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=3 XRD_CACHE="root://sc-cache.chtc.wisc.edu"
13+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://osg-new-york-stashcache.nrp.internet2.edu"
14+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://osg-kansas-city-stashcache.nrp.internet2.edu"
15+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://osg-chicago-stashcache.nrp.internet2.edu"
16+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://fiona.uvalight.net"
17+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://stashcache.t2.ucsd.edu"
18+
- OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE="root://osg-gftp.pace.gatech.edu"
19+
- BUILD_TYPE=http OS_TYPE=centos OS_VERSION=7 PYTHON_VERSION=2 XRD_CACHE=
20+
1821

1922
language: python
2023
python:
@@ -39,7 +42,7 @@ before_install:
3942

4043
script:
4144
- if [ "$BUILD_TYPE" = "docs" ]; then bin/stashcp2/tests/build_docs.sh; fi
42-
- bin/stashcp2/tests/setup_tests.sh ${OS_VERSION}
45+
- bin/stashcp2/tests/setup_tests.sh "${OS_VERSION}" "${XRD_CACHE}" "${PYTHON_VERSION}"
4346

4447
deploy:
4548
provider: pypi

‎bin/stashcp2/tests/setup_tests.sh‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,33 @@
44

55
# Version of CentOS/RHEL
66
el_version=$1
7-
cache=$2
7+
XRD_CACHE=$2
8+
PYTHON_VERSION=$3
89

910
if [ "${BUILD_TYPE}" = "http" ]; then
11+
python=python2
12+
if [ "$PYTHON_VERSION" = 3 ]; then
13+
python=python3
14+
fi
15+
1016
# Run the test without a container
1117
# Copy in the .job.ad file:
1218
cp bin/stashcp2/tests/job.ad ./.job.ad
13-
python setup.py install
19+
$python setup.py install
20+
21+
stashcp=$(command -v stashcp)
1422

1523
# Test against a file that is known to not exist
1624
set +e
17-
stashcp --cache=$XRD_CACHE /blah/does/not/exist ./
25+
$python $stashcp --cache=$XRD_CACHE /blah/does/not/exist ./
1826
if [ $? -eq 0 ]; then
1927
echo "Failed to exit with non-zero exit status when it should have"
2028
exit 1
2129
fi
2230
set -e
2331

2432
# Try copying with different destintion filename
25-
stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 query.test
33+
$python $stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 query.test
2634

2735
result=`md5sum query.test | awk '{print $1;}'`
2836

@@ -33,7 +41,7 @@ if [ "${BUILD_TYPE}" = "http" ]; then
3341
rm -f query.test
3442

3543
# Perform tests
36-
stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 ./
44+
$python $stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 ./
3745

3846
result=`md5sum query1 | awk '{print $1;}'`
3947

@@ -44,14 +52,25 @@ if [ "${BUILD_TYPE}" = "http" ]; then
4452
# Run tests in Container
4553
elif [ "$el_version" = "6" ]; then
4654

47-
sudo docker run --privileged --rm=true -v `pwd`:/StashCache:rw centos:centos${OS_VERSION} /bin/bash -c "bash -xe /StashCache/bin/stashcp2/tests/test_inside_docker.sh ${OS_VERSION} ${XRD_CACHE}"
55+
sudo docker run --privileged --rm=true -v `pwd`:/StashCache:rw centos:centos${OS_VERSION} /bin/bash -c "bash -xe /StashCache/bin/stashcp2/tests/test_inside_docker.sh \"${OS_VERSION}\" \"${XRD_CACHE}\" \"${PYTHON_VERSION}\""
4856

4957
elif [ "$el_version" = "7" ]; then
5058

5159
docker run --privileged --cap-add SYS_ADMIN -d -ti -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup -v `pwd`:/StashCache:rw centos:centos${OS_VERSION} /usr/sbin/init
5260
DOCKER_CONTAINER_ID=$(docker ps | grep centos | awk '{print $1}')
5361
docker logs $DOCKER_CONTAINER_ID
54-
docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash -xe /StashCache/bin/stashcp2/tests/test_inside_docker.sh ${OS_VERSION} ${XRD_CACHE};
62+
docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash -xe /StashCache/bin/stashcp2/tests/test_inside_docker.sh \"${OS_VERSION}\" \"${XRD_CACHE}\" \"${PYTHON_VERSION}\";
63+
echo -ne \"------\nEND stashcp TESTS\n\";"
64+
docker ps -a
65+
docker stop $DOCKER_CONTAINER_ID
66+
docker rm -v $DOCKER_CONTAINER_ID
67+
68+
elif [ "$el_version" = "8" ]; then
69+
70+
docker run --privileged --cap-add SYS_ADMIN -d -ti -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup -v `pwd`:/StashCache:rw centos:centos${OS_VERSION} /usr/sbin/init
71+
DOCKER_CONTAINER_ID=$(docker ps | grep centos | awk '{print $1}')
72+
docker logs $DOCKER_CONTAINER_ID
73+
docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash -xe /StashCache/bin/stashcp2/tests/test_inside_docker.sh \"${OS_VERSION}\" \"${XRD_CACHE}\" \"${PYTHON_VERSION}\";
5574
echo -ne \"------\nEND stashcp TESTS\n\";"
5675
docker ps -a
5776
docker stop $DOCKER_CONTAINER_ID

‎bin/stashcp2/tests/test_inside_docker.sh‎

Lines changed: 43 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,55 @@
11
#!/bin/sh -xe
22

3+
if [ $# -ne 3 ]; then
4+
echo >&2 "Usage: $0 OS_VERSION XRD_CACHE PYTHON_VERSION"
5+
exit 2
6+
fi
37
OS_VERSION=$1
48
XRD_CACHE=$2
9+
PYTHON_VERSION=$3
510

611
ls -l /home
712

813
# Clean the yum cache
914
yum -y clean all
10-
yum -y clean expire-cache
1115

1216
# First, install all the needed packages.
1317
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OS_VERSION}.noarch.rpm
1418

15-
yum -y install yum-plugin-priorities
1619
case $OS_VERSION in
17-
6) OSG_VERSION=3.4
20+
6) OSG_VERSION=3.4
21+
yum -y install yum-plugin-priorities
22+
;;
23+
7) OSG_VERSION=3.5
24+
yum -y install yum-plugin-priorities
1825
;;
19-
7|8) OSG_VERSION=3.5
26+
8) OSG_VERSION=3.5
27+
# no priorities
2028
;;
2129
esac
2230
rpm -Uvh https://repo.opensciencegrid.org/osg/${OSG_VERSION}/osg-${OSG_VERSION}-el${OS_VERSION}-release-latest.rpm
2331

2432
yum -y install osg-oasis
2533

26-
yum -y install python-pip
34+
35+
case $PYTHON_VERSION in
36+
2)
37+
pip=pip2
38+
python=python2
39+
if [ "$OS_VERSION" -eq 6 ]; then
40+
yum -y install python-setuptools
41+
export PYTHONDONTWRITEBYTECODE=
42+
easy_install 'pip==9'
43+
else
44+
yum -y install python2-pip
45+
fi
46+
;;
47+
3)
48+
yum -y install python3-pip
49+
pip=pip3
50+
python=python3
51+
;;
52+
esac
2753

2854
echo "user_allow_other" >> /etc/fuse.conf
2955

@@ -46,29 +72,31 @@ module load xrootd
4672
#pylint /StashCache/bin/stashcp || /bin/true
4773

4874
# Install stashcp
49-
pip install StashCache/
75+
$pip install StashCache/
76+
77+
stashcp=$(command -v stashcp)
5078

5179
# Copy in the .job.ad file:
5280
cp /StashCache/bin/stashcp2/tests/job.ad ./.job.ad
5381

5482
# Test against a file that is known to not exist
5583
set +e
56-
stashcp --cache=$XRD_CACHE /blah/does/not/exist ./
84+
$python $stashcp --cache=$XRD_CACHE /blah/does/not/exist ./
5785
if [ $? -eq 0 ]; then
5886
echo "Failed to exit with non-zero exit status when it should have"
5987
exit 1
6088
fi
6189
set -e
6290

6391
# Try copying with no forward slash
64-
stashcp --cache=$XRD_CACHE osgconnect/public/dweitzel/blast/queries/query1 ./
92+
$python $stashcp --cache=$XRD_CACHE osgconnect/public/dweitzel/blast/queries/query1 ./
6593

6694
result=`md5sum query1 | awk '{print $1;}'`
6795

6896
rm query1
6997

7098
# Try copying with different destintion filename
71-
stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 query.test
99+
$python $stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 query.test
72100

73101
result=`md5sum query.test | awk '{print $1;}'`
74102

@@ -79,7 +107,7 @@ fi
79107
rm -f query.test
80108

81109
# Perform tests
82-
stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 ./
110+
$python $stashcp --cache=$XRD_CACHE -d /osgconnect/public/dweitzel/blast/queries/query1 ./
83111

84112
result=`md5sum query1 | awk '{print $1;}'`
85113

@@ -89,7 +117,7 @@ fi
89117
rm -f query.test
90118

91119
# Perform methods test
92-
stashcp --cache=$XRD_CACHE --method=cvmfs,xrootd -d /osgconnect/public/dweitzel/blast/queries/query1 ./
120+
$python $stashcp --cache=$XRD_CACHE --method=cvmfs,xrootd -d /osgconnect/public/dweitzel/blast/queries/query1 ./
93121

94122
result=`md5sum query1 | awk '{print $1;}'`
95123

@@ -99,7 +127,7 @@ fi
99127
rm -f query.test
100128

101129
# Perform methods test
102-
stashcp --cache=$XRD_CACHE --method=xrootd -d /osgconnect/public/dweitzel//blast/queries/query1 ./
130+
$python $stashcp --cache=$XRD_CACHE --method=xrootd -d /osgconnect/public/dweitzel//blast/queries/query1 ./
103131

104132
result=`md5sum query1 | awk '{print $1;}'`
105133

@@ -109,7 +137,7 @@ fi
109137
rm -f query.test
110138

111139
# Perform methods test
112-
stashcp --cache=$XRD_CACHE --method=http,xrootd -d /osgconnect/public/dweitzel/blast/queries/query1 ./
140+
$python $stashcp --cache=$XRD_CACHE --method=http,xrootd -d /osgconnect/public/dweitzel/blast/queries/query1 ./
113141

114142
result=`md5sum query1 | awk '{print $1;}'`
115143

@@ -119,7 +147,7 @@ fi
119147
rm -f query.test
120148

121149
# Scheme test
122-
stashcp --cache=$XRD_CACHE --method=http,xrootd -d stash:///osgconnect/public/dweitzel/blast/queries/query1 file:///tmp
150+
$python $stashcp --cache=$XRD_CACHE --method=http,xrootd -d stash:///osgconnect/public/dweitzel/blast/queries/query1 file:///tmp
123151

124152
result=`md5sum /tmp/query1 | awk '{print $1;}'`
125153

@@ -128,7 +156,7 @@ if [ "$result" != "12bdb9a96cd5e8ca469b727a81593201" ]; then
128156
fi
129157
rm -f query.test
130158

131-
stashcp --cache=$XRD_CACHE -d -r /osgconnect/public/dweitzel/blast/queries ./
159+
$python $stashcp --cache=$XRD_CACHE -d -r /osgconnect/public/dweitzel/blast/queries ./
132160
ls -lah
133161

134162
rm -rf queries

‎setup.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@
106106
# that you indicate whether you support Python 2, Python 3 or both.
107107
'Programming Language :: Python :: 2',
108108
'Programming Language :: Python :: 2.7',
109+
'Programming Language :: Python :: 3',
110+
'Programming Language :: Python :: 3.6',
109111
],
110112

111113
# This field adds keywords for your project which will appear on the

0 commit comments

Comments
 (0)